From b296f71190459da5d2bb589d1bebe50f788d3d14 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 00:40:21 +0200 Subject: [PATCH 01/25] Add high-level client API with per-peer threading Adds dev.onvoid.webrtc.client: a WebRtc runtime that owns the factory and everything created through it, peers with built-in perfect negotiation, futures instead of nested observers, and one event thread per peer that never runs on a WebRTC thread. Pure Java, no native changes. --- design/client-api.md | 794 ++++++++++ .../dev/onvoid/webrtc/client/AudioChunk.java | 74 + .../onvoid/webrtc/client/AudioDispatcher.java | 107 ++ .../dev/onvoid/webrtc/client/AudioMode.java | 40 + .../webrtc/client/BlockingCallException.java | 45 + .../onvoid/webrtc/client/CameraOptions.java | 153 ++ .../onvoid/webrtc/client/ChannelOptions.java | 179 +++ .../onvoid/webrtc/client/ChannelState.java | 51 + .../onvoid/webrtc/client/ClientFuture.java | 88 ++ .../webrtc/client/ConfigurationException.java | 45 + .../dev/onvoid/webrtc/client/CustomAudio.java | 50 + .../onvoid/webrtc/client/CustomAudioImpl.java | 183 +++ .../dev/onvoid/webrtc/client/CustomVideo.java | 44 + .../onvoid/webrtc/client/CustomVideoImpl.java | 56 + .../dev/onvoid/webrtc/client/DataChannel.java | 114 ++ .../onvoid/webrtc/client/DataChannelImpl.java | 448 ++++++ .../webrtc/client/DeferredSubscription.java | 57 + .../dev/onvoid/webrtc/client/Devices.java | 91 ++ .../dev/onvoid/webrtc/client/DevicesImpl.java | 224 +++ .../dev/onvoid/webrtc/client/Encoding.java | 128 ++ .../onvoid/webrtc/client/FrameDelivery.java | 54 + .../onvoid/webrtc/client/FrameDispatcher.java | 118 ++ .../dev/onvoid/webrtc/client/Gathering.java | 30 + .../dev/onvoid/webrtc/client/Handlers.java | 78 + .../java/dev/onvoid/webrtc/client/Json.java | 298 ++++ .../dev/onvoid/webrtc/client/LocalAudio.java | 24 + .../onvoid/webrtc/client/LocalAudioImpl.java | 30 + .../dev/onvoid/webrtc/client/LocalMedia.java | 110 ++ .../onvoid/webrtc/client/LocalMediaImpl.java | 368 +++++ .../dev/onvoid/webrtc/client/LocalTrack.java | 70 + .../onvoid/webrtc/client/LocalTrackImpl.java | 223 +++ .../dev/onvoid/webrtc/client/LocalVideo.java | 40 + .../onvoid/webrtc/client/LocalVideoImpl.java | 87 ++ .../onvoid/webrtc/client/MediaException.java | 45 + .../dev/onvoid/webrtc/client/MediaKind.java | 30 + .../dev/onvoid/webrtc/client/Message.java | 69 + .../dev/onvoid/webrtc/client/Negotiation.java | 36 + .../webrtc/client/NegotiationException.java | 45 + .../dev/onvoid/webrtc/client/Negotiator.java | 85 ++ .../dev/onvoid/webrtc/client/PcmFormat.java | 94 ++ .../java/dev/onvoid/webrtc/client/Peer.java | 215 +++ .../webrtc/client/PeerClosedException.java | 45 + .../dev/onvoid/webrtc/client/PeerImpl.java | 1283 +++++++++++++++++ .../dev/onvoid/webrtc/client/PeerOptions.java | 283 ++++ .../dev/onvoid/webrtc/client/PeerState.java | 61 + .../dev/onvoid/webrtc/client/PeerStats.java | 161 +++ .../dev/onvoid/webrtc/client/RemoteAudio.java | 44 + .../onvoid/webrtc/client/RemoteAudioImpl.java | 94 ++ .../dev/onvoid/webrtc/client/RemoteTrack.java | 100 ++ .../onvoid/webrtc/client/RemoteTrackImpl.java | 193 +++ .../dev/onvoid/webrtc/client/RemoteVideo.java | 52 + .../onvoid/webrtc/client/RemoteVideoImpl.java | 92 ++ .../onvoid/webrtc/client/ScreenOptions.java | 128 ++ .../java/dev/onvoid/webrtc/client/Sender.java | 57 + .../dev/onvoid/webrtc/client/SenderImpl.java | 253 ++++ .../onvoid/webrtc/client/SerialExecutor.java | 190 +++ .../onvoid/webrtc/client/SignalMessage.java | 297 ++++ .../dev/onvoid/webrtc/client/Signaling.java | 65 + .../onvoid/webrtc/client/SignalingPair.java | 93 ++ .../dev/onvoid/webrtc/client/StatsReader.java | 146 ++ .../onvoid/webrtc/client/Subscription.java | 33 + .../dev/onvoid/webrtc/client/VideoStats.java | 96 ++ .../java/dev/onvoid/webrtc/client/WebRtc.java | 560 +++++++ .../onvoid/webrtc/client/WebRtcException.java | 45 + webrtc/src/main/java/module-info.java | 1 + .../onvoid/webrtc/client/ClientTestBase.java | 86 ++ .../dev/onvoid/webrtc/client/MediaTests.java | 227 +++ .../dev/onvoid/webrtc/client/PeerTests.java | 364 +++++ .../webrtc/client/SignalMessageTests.java | 85 ++ 69 files changed, 10254 insertions(+) create mode 100644 design/client-api.md create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/AudioChunk.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/AudioDispatcher.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/AudioMode.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/BlockingCallException.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CameraOptions.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelOptions.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelState.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/ClientFuture.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/ConfigurationException.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudio.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideo.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideoImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/DeferredSubscription.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Encoding.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDelivery.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Handlers.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Json.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrack.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/MediaException.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/MediaKind.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Message.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiation.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/NegotiationException.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiator.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PcmFormat.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Peer.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PeerClosedException.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PeerState.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/PeerStats.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudio.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudioImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrack.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideo.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideoImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/ScreenOptions.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/SerialExecutor.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/SignalMessage.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Signaling.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/SignalingPair.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/StatsReader.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Subscription.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/VideoStats.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtcException.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/SignalMessageTests.java diff --git a/design/client-api.md b/design/client-api.md new file mode 100644 index 00000000..0fc98723 --- /dev/null +++ b/design/client-api.md @@ -0,0 +1,794 @@ +# webrtc-java Client API + +Status: phase 1 implemented on the `client-api` branch, based on `main` at 6cc938ab. See +[Implementation status](#implementation-status) for where the code differs from this design. + +A high-level layer over the existing JNI API. It owns every native object, gives application code +one thread per peer, negotiates on its own, and returns futures instead of nested observers. The +low-level API stays as it is. + +- Package `dev.onvoid.webrtc.client`, in the existing `webrtc-java` jar, exported by the + `webrtc.java` module. +- Java 8, like the rest of the module: no records, no sealed types, no `var`. +- Pure Java. Phase 1 needs no native or JNI changes and changes no existing signature. + +## Decisions + +These were open questions in the mockup. Each went with its recommendation. + +| Question | Decision | Consequence | +|---|---|---| +| Where does it live? | A package in the existing jar, not a separate artifact. | Users get it without another dependency. It follows the core's releases. | +| Default perfect negotiation role? | Impolite by default. | Glare between two impolite peers fails with a `NegotiationException` that names the fix (make one side polite). Every example sets the role explicitly. | +| Plain `CompletableFuture` or an own type? | Plain `CompletableFuture`. | On Java 8, stages derived from a subclass are plain `CompletableFuture`s, so `BlockingCallException` only catches a `join()`/`get()` on the future the API returned. The exception message and the event thread names help with the rest. | +| Does `Peer.close()` block? | No, never. | Safe in handlers and on UI threads. `closeAsync()` returns a future for callers who want to wait. `WebRtc.close()` is the one call that waits for everything. | +| A camera opened twice? | Shared when the options match, `MediaException` when they differ. | One capture per device. Most platforms can't open a device with two capabilities anyway. | +| Entry point name? | `WebRtc`. | `try (WebRtc rtc = WebRtc.create()) { ... }` | + +## What hurts today + +Each row comes from the current code, docs or examples on `main`. + +| Problem | Where it shows | Answer | +|---|---|---| +| Observers run on WebRTC's own threads | `PeerConnectionObserver` is called on the signaling thread. A handler that blocks, or waits for work that needs that thread, stalls every connection of the factory. | Native callbacks only copy and enqueue. Handlers run on a per-peer event thread, where they may block or call `close()`. | +| Threading rules differ per class | Data channel send results, DTLS and injector observers go through `JavaCallbackThread`. Peer connection observers don't. Frame sinks run on capture and decode threads. Custom audio has to be pushed from a single thread. | One rule for all control events. Frames get their own documented thread with a drop policy. Custom audio is serialized by the library. | +| Nested callbacks for offer and answer | `createOffer` β†’ `CreateSessionDescriptionObserver` β†’ `setLocalDescription` β†’ `SetSessionDescriptionObserver`, with the error as a `String`. `WhepExample` uses three `CountDownLatch`es to flatten it. | `CompletableFuture` everywhere, failing with typed exceptions. In automatic mode the application never calls offer or answer. | +| Signaling correctness is left to the app | Glare, rollback, candidates that arrive before the remote description (they fail silently with the plain `addIceCandidate`), and `onRenegotiationNeeded` firing mid-negotiation. | Built-in perfect negotiation with a polite/impolite role, candidate queuing, and a browser-compatible JSON message format. | +| Manual disposal of query results | Every `RTCRtpSender`, `RTCRtpReceiver` and `RTCRtpTransceiver` returned to the app must be disposed, sources are not owned by tracks, and the factory has to outlive its connections. | No `dispose()` in the API. Everything is `AutoCloseable`, owned by its creator, and closed in the right order. | +| Receive-only needs a dummy track | `WhepExample` creates a `VideoDeviceSource` and a `VideoTrack` only to pass them to a `RECV_ONLY` transceiver. | `PeerOptions.receive(VIDEO, AUDIO)`. | +| Audio input mode is found at run time | Mixing device and custom audio in one factory throws `IllegalStateException` on the first track of the other kind. | `AudioMode` is chosen when the runtime is built. The wrong call fails at once with a message naming the mode. | +| Device queries can block the caller | The guides say to query devices on worker threads, never on UI or render threads. | `Devices` is a snapshot kept current in the background, so reading it is instant on any thread. | + +## Before and after + +A video call over a WebSocket. Today the web example spreads this over `PeerConnectionManager` +(460 lines) and `MediaManager` (274 lines). This is the core of it, shortened: + +```java +PeerConnectionFactory factory = new PeerConnectionFactory(); +RTCConfiguration config = new RTCConfiguration(); +RTCIceServer iceServer = new RTCIceServer(); +iceServer.urls.add("stun:stun.l.google.com:19302"); +config.iceServers.add(iceServer); + +RTCPeerConnection pc = factory.createPeerConnection(config, new PeerConnectionObserver() { + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + socket.send(toJson(candidate)); // runs on WebRTC's signaling thread + } + + @Override + public void onRenegotiationNeeded() { + createOffer(); // glare and rollback are up to you + } + + @Override + public void onTrack(RTCRtpTransceiver transceiver) { + RTCRtpReceiver receiver = transceiver.getReceiver(); + MediaStreamTrack track = receiver.getTrack(); + if (track instanceof VideoTrack) { + ((VideoTrack) track).addSink(frame -> view.render(frame)); + } + receiver.dispose(); // query results are yours to dispose + transceiver.dispose(); + } +}); + +void createOffer() { + pc.createOffer(new RTCOfferOptions(), new CreateSessionDescriptionObserver() { + @Override + public void onSuccess(RTCSessionDescription offer) { + pc.setLocalDescription(offer, new SetSessionDescriptionObserver() { + @Override + public void onSuccess() { socket.send(toJson(offer)); } + + @Override + public void onFailure(String error) { log.error(error); } + }); + } + + @Override + public void onFailure(String error) { log.error(error); } + }); +} + +// Not shown: answering, queuing early candidates, JSON parsing, creating and +// disposing sources, tracks and senders, closing pc before disposing factory. +``` + +With the client API, this is the whole thing: + +```java +try (WebRtc rtc = WebRtc.create()) { + Peer peer = rtc.createPeer(PeerOptions.builder() + .iceServer("stun:stun.l.google.com:19302") + .polite(joinedSecond) + .signaling(message -> socket.send(message.toJson())) + .build()); + socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); + + peer.addTrack(rtc.media().microphone()); + peer.addTrack(rtc.media().camera()); + + peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(frame -> view.render(frame)); + } + }); + peer.onStateChange(state -> log.info("Call is {}", state)); + + peer.whenConnected().join(); // fine here: this is not an event thread + hangUp.await(); +} // closes the peer, the camera, the microphone and the factory, in that order +``` + +## Threading model + +The model borrows from Netty's event loop: each peer has one event thread for application code, +and the library never runs application code on a thread WebRTC needs. + +```mermaid +sequenceDiagram + participant App as Your thread + participant Ops as Peer operation queue + participant RTC as WebRTC threads + participant Ev as Peer event thread + participant H as Your handlers + App->>Ops: peer.addTrack(camera), returns at once + Ops->>RTC: addTrack (proxy call, may block Ops) + RTC--)Ev: onRenegotiationNeeded (copied, enqueued) + Ev->>Ops: start offer (perfect negotiation) + Ops->>RTC: setLocalDescription() + RTC--)Ev: local offer set, onIceCandidate ... + Ev->>H: Signaling.send(offer), Signaling.send(candidate) + RTC--)Ev: onConnectionChange(CONNECTED) + Ev->>H: onStateChange(CONNECTED), whenConnected() completes + H->>Ops: peer.close() from a handler, fine +``` + +> **Invariant the implementation must keep:** nothing the library runs on a WebRTC thread ever +> waits for a Java thread. Native observers copy their arguments, enqueue, and return. That rules +> out the deadlock cycle no matter what a handler does. + +| Code | Runs on | May it block? | +|---|---|---| +| Calls into the API | the caller's thread, any thread | They don't block. The only exceptions are `WebRtc.close()`, which waits up to `closeTimeout`, and a `join()` the caller writes. | +| Event handlers, future callbacks, `Signaling.send` | the peer's event thread, or the configured `eventExecutor` | Yes, but it only delays that peer's events. It must not `join()` a future of this API: a direct `join()`/`get()` throws `BlockingCallException`. | +| Frame handlers (`onFrame`, `onAudio`) | one frame thread per subscription | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped and released. | +| Native calls | internal operation threads | Not application code. Each operation has a timeout (default 30 s), so a stuck native call fails with a `TimeoutException` instead of hanging. | +| WebRTC callbacks | signaling, network and capture threads | Not application code. The library copies and enqueues only. | + +### Guarantees + +1. **Every method is thread-safe.** It can be called from any thread, including from inside a + handler of the same peer. +2. **One peer, one thread at a time.** A peer's handlers and future callbacks never run + concurrently, and they run in event order. Different peers may run in parallel unless the + configured executor is serial, such as the Swing EDT. +3. **Order is kept with any executor.** A pool passed as `eventExecutor` is wrapped per peer, so + ordering holds whatever executor is given. +4. **No lost events from registration races.** `onStateChange` is called with the current state + first. `onTrack` and `onChannel` replay what already arrived. A data channel keeps the messages + that arrive before its first `onMessage` handler. +5. **Operations run in order.** Each peer runs its operations one at a time, like the W3C + operations chain. `addTrack` followed by `createChannel` always negotiates both, in that order. +6. **Close is final and quiet.** After `close()` the only event is `onStateChange(CLOSED)`. Pending + futures fail with `PeerClosedException`, and later calls fail the same way instead of crashing + in native code. +7. **Handler exceptions are contained.** An exception thrown by a handler goes to the runtime's + `errorHandler`. It never stops delivery and never crosses into native code. + +## Ownership and lifecycle + +There is no `dispose()` in this API. You close what you created, and closing an owner closes what +it owns. Senders, receivers and transceivers are queried and disposed inside the library and are +never handed out. + +| Object | Created by | Closed by | Closing it also | +|---|---|---|---| +| `WebRtc` | the application | the application, usually try-with-resources | closes all peers, then all local tracks and the device watcher, then disposes the factory and stops its threads | +| `Peer` | `rtc.createPeer` | the application, or `WebRtc` | closes its data channels and ends its remote tracks. Local tracks stay open for other peers. | +| `LocalVideo`, `LocalAudio` | `rtc.media()` | the application, or `WebRtc` | detaches the track from every peer it was sent on (no renegotiation), stops capture, releases the source | +| `Sender` | `peer.addTrack` | `sender.remove()`, or the peer | renegotiates in automatic mode. The track stays open. | +| `DataChannel` | `peer.createChannel`, or the remote side | the application, or the peer | completes pending sends with `PeerClosedException` | +| `RemoteTrack` | the peer | the peer, when the remote side removes it or the peer closes | calls `onEnded` and ends its frame subscriptions | +| `Subscription` | every `on…` method | the application, or the owner of the event | removes the handler | + +## API surface + +Signatures only. Value types reuse the existing ones where they are plain data +(`RTCSessionDescription`, `RTCIceCandidate`, `VideoFrame`, `AudioOptions`, `VideoDevice`, +`AudioDevice`, `DesktopSource`). + +### `WebRtc`, the runtime + +The entry point. It owns the `PeerConnectionFactory`, the audio device module, the library's +threads, and everything created through it. + +```java +public final class WebRtc implements AutoCloseable { + + public static WebRtc create(); // microphone and speakers, built-in codecs + public static Builder builder(); + + public Peer createPeer(PeerOptions options); + public LocalMedia media(); // cameras, microphones, screens, custom sources + public List peers(); // snapshot of the open peers + + public CompletableFuture closeAsync(); + @Override + public void close(); // peers, then tracks, then the factory; waits up to closeTimeout + + public static final class Builder { + public Builder audio(AudioMode mode); // DEVICES by default + public Builder audioProcessing(AudioProcessingConfig config); + public Builder hardwareVideoCodecs(boolean enabled); // GPU encoders and decoders where present + public Builder fieldTrials(Map fieldTrials); + public Builder eventExecutor(Executor executor); // e.g. SwingUtilities::invokeLater, Platform::runLater + public Builder errorHandler(Consumer handler); // handler exceptions, failures nobody observed + public Builder operationTimeout(Duration timeout); // default 30 s + public Builder closeTimeout(Duration timeout); // default 10 s + public Builder configureFactory(Consumer customizer); + public WebRtc build(); + } +} + +public enum AudioMode { + DEVICES, // capture the microphone, play to the speakers + CUSTOM, // send audio you push, play remote audio to the speakers + HEADLESS // no audio hardware: send audio you push, read remote audio through onAudio +} +``` + +- `AudioMode` moves the factory's device-or-custom rule to build time. `media().microphone()` on a + `CUSTOM` runtime throws `ConfigurationException` saying why. +- `HEADLESS` uses `HeadlessAudioDeviceModule`, so servers and bots run without audio hardware. +- Library threads are daemon threads named after their peer (`webrtc-client-events-alice`), so + thread dumps show which peer is busy. + +### `PeerOptions` + +```java +public final class PeerOptions { + + public static Builder builder(); + + public static final class Builder { + public Builder name(String name); // thread names, logs + public Builder iceServer(String... urls); // "stun:host:3478", "turn:host" without credentials + public Builder iceServer(String url, String username, String password); + public Builder relayOnly(boolean relayOnly); // TURN candidates only + public Builder signaling(Signaling signaling); // required with Negotiation.AUTOMATIC + public Builder polite(boolean polite); // perfect negotiation role, default false + public Builder negotiation(Negotiation negotiation); // AUTOMATIC by default, or MANUAL + public Builder receive(MediaKind... kinds); // receive-only audio and video + public Builder configure(Consumer customizer); // anything else RTCConfiguration has + public PeerOptions build(); + } +} + +public enum Negotiation { AUTOMATIC, MANUAL } +public enum MediaKind { AUDIO, VIDEO } +``` + +### `Peer`, one connection + +A connection to one remote peer. Methods that change it return at once and run on the peer's +operation queue, in order. + +```java +public interface Peer extends AutoCloseable { + + String name(); + PeerState state(); // the last state, cached on the Java side + + // Signaling input: hand over what the remote side sent, in the order it was sent. + void signal(SignalMessage message); + + // Sending. In AUTOMATIC mode each change renegotiates by itself. + Sender addTrack(LocalTrack track, String... streamIds); + List senders(); + + // Receiving. Late subscribers get the tracks that already arrived. + Subscription onTrack(Consumer handler); + List remoteTracks(); + + // Data channels. Late subscribers get the channels that already opened. + DataChannel createChannel(String label); + DataChannel createChannel(String label, ChannelOptions options); + Subscription onChannel(Consumer handler); + + // State. onStateChange calls the handler with the current state first. + Subscription onStateChange(Consumer handler); + Subscription onError(Consumer handler); + CompletableFuture whenConnected(); // fails if FAILED or CLOSED comes first + CompletableFuture whenClosed(); + + Negotiator negotiator(); // MANUAL mode only + CompletableFuture stats(); + Subscription onStats(Duration interval, Consumer handler); + void restartIce(); + + // Escape hatch: runs on the operation queue with the underlying connection. + CompletableFuture withNative(Function action); + + CompletableFuture closeAsync(); + @Override + void close(); // never blocks, idempotent, fine inside a handler +} + +public enum PeerState { NEW, CONNECTING, CONNECTED, DISCONNECTED, FAILED, CLOSED } +``` + +- `PeerState` folds `RTCPeerConnectionState` into six values. `DISCONNECTED` can recover by + itself. `FAILED` needs `restartIce()` or a new peer. +- `addTrack` returns the `Sender` handle right away. If the native call fails, the error goes to + `onError` and the sender's futures fail. +- `close()` never blocks, so it is safe in a handler and on the Swing EDT. `closeAsync().join()` + waits, from a thread that is not an event thread. +- Objects obtained inside `withNative` are owned by the caller, as in the low-level API, and must + not be used after the function returns. + +### `Signaling` and `SignalMessage` + +The library has no transport of its own. The application gives it a function that sends, and it +hands over messages that browsers understand. + +```java +@FunctionalInterface +public interface Signaling { + + // Called on the peer's event thread, in the order the remote side must get them. + void send(SignalMessage message); + + // Called once when the peer is created. A transport that receives messages + // can push them into inbound instead of calling peer.signal(...). + default void attach(Consumer inbound) { + } + + // Two linked ends, for tests and same-process demos. + static SignalingPair loopback(); +} + +public final class SignalMessage { + + public enum Type { OFFER, ANSWER, CANDIDATE, END_OF_CANDIDATES } + + public static SignalMessage of(RTCSessionDescription description); + public static SignalMessage of(RTCIceCandidate candidate); + public static SignalMessage endOfCandidates(); + public static SignalMessage fromJson(String json); + + public Type type(); + public RTCSessionDescription description(); // OFFER and ANSWER + public RTCIceCandidate candidate(); // CANDIDATE + public String toJson(); +} +``` + +`toJson()` writes the shapes the browser API uses (`RTCSessionDescriptionInit`, +`RTCIceCandidateInit`), so a JavaScript peer can pass them straight to `setRemoteDescription` and +`addIceCandidate`: + +```json +{"type":"offer","sdp":"v=0\r\no=- 4611731400430051336 2 IN IP4 127.0.0.1\r\n..."} +{"candidate":"candidate:842163049 1 udp 1677729535 203.0.113.7 49203 typ srflx raddr 0.0.0.0 rport 0 generation 0","sdpMid":"0","sdpMLineIndex":0} +{"candidate":"","sdpMid":"0","sdpMLineIndex":0} +``` + +- The last line is the browser's end-of-candidates form. It maps to `END_OF_CANDIDATES`. +- Routing (rooms, sender and receiver ids) stays in the application's own envelope. The JSON is + the payload inside it. +- Candidates that arrive before the remote description are queued and added once it is set. +- The JSON codec is a small built-in one for these fixed shapes, so the module gains no + dependency. + +### `Negotiator`, manual mode + +For WHIP, WHEP, SFUs and anything else that dictates its own offer and answer exchange. Each step +both creates and applies the description, so nothing can change in between. + +```java +public interface Negotiator { + + CompletableFuture createOffer(); // creates and applies the local offer + CompletableFuture createOffer(Gathering gathering); // COMPLETE waits for all candidates + CompletableFuture answer(RTCSessionDescription offer); + CompletableFuture applyAnswer(RTCSessionDescription answer); + CompletableFuture addCandidate(RTCIceCandidate candidate); // queued until the remote description is set +} + +public enum Gathering { TRICKLE, COMPLETE } +``` + +In manual mode a `Signaling`, if one is set, still receives the local candidates for trickle ICE. + +### `LocalMedia`, `Devices`, local tracks + +```java +public interface LocalMedia { + + Devices devices(); + + LocalVideo camera(); // default camera, default capability + LocalVideo camera(CameraOptions options); + LocalAudio microphone(); // AudioMode.DEVICES only + LocalAudio microphone(AudioOptions options); // echo cancellation, noise suppression, gain control + LocalVideo screen(ScreenOptions options); // a screen or a window from devices() + CustomVideo customVideo(String label); + CustomAudio customAudio(String label); // AudioMode.CUSTOM or HEADLESS +} + +public interface Devices { + + List cameras(); // a snapshot, kept current in the background + List microphones(); + List speakers(); + List screens(); + List windows(); + + CompletableFuture useMicrophone(AudioDevice device); + CompletableFuture useSpeaker(AudioDevice device); + Subscription onChange(Consumer handler); +} + +public interface LocalTrack extends AutoCloseable { + + String id(); + MediaKind kind(); + boolean isEnabled(); + void setEnabled(boolean enabled); // false sends black frames or silence, capture stays open + Subscription onEnded(Runnable handler); // device unplugged, window closed, track closed + @Override + void close(); // stops capture, detaches from every peer +} + +public interface LocalVideo extends LocalTrack { + + Subscription onFrame(Consumer handler); // local preview + CompletableFuture switchCamera(VideoDevice device); // peers keep sending, no renegotiation +} + +public interface LocalAudio extends LocalTrack { +} + +public interface CustomVideo extends LocalVideo { + + void push(VideoFrame frame); // captured now + void push(VideoFrame frame, long timestampMicros); // keeps your source's own timing +} + +public interface CustomAudio extends LocalAudio { + + void push(byte[] pcm, PcmFormat format); // any length, any thread +} +``` + +```java +CameraOptions hd = CameraOptions.builder() + .device(rtc.media().devices().cameras().get(0)) + .resolution(1280, 720) + .frameRate(30) + .build(); // the camera's nearest supported capability is used + +ScreenOptions screen = ScreenOptions.builder() + .source(rtc.media().devices().screens().get(0)) + .frameRate(15) + .build(); +``` + +- One `LocalTrack` can be sent on any number of peers. The library keeps one source and one sender + per peer. +- Opening a camera that is already open shares its capture when the options match, and fails with + `MediaException` when they differ. The capture stops when the last track using it is closed. +- `CustomAudio.push` takes any amount of PCM from any thread. The library cuts it into the 10 ms + chunks WebRTC wants and feeds them from one internal thread, which is the single-thread rule + `CustomAudioSource` has today. + +### `Sender`, `RemoteTrack` + +```java +public interface Sender { + + LocalTrack track(); + CompletableFuture replaceTrack(LocalTrack track); // no renegotiation, null sends nothing + CompletableFuture configure(Encoding encoding); // max bitrate, max frame rate, scale down, active + CompletableFuture remove(); // renegotiates in AUTOMATIC mode + DtmfSender dtmf(); // audio senders only +} + +public interface RemoteTrack { + + String id(); + MediaKind kind(); + List streamIds(); + boolean isVideo(); + boolean isAudio(); + RemoteVideo asVideo(); // IllegalStateException for audio + RemoteAudio asAudio(); + Subscription onMute(Consumer handler); + Subscription onEnded(Runnable handler); +} + +public interface RemoteVideo extends RemoteTrack { + + Subscription onFrame(Consumer handler); // FrameDelivery.LATEST + Subscription onFrame(FrameDelivery delivery, Consumer handler); // LATEST, or queue(n) +} + +public interface RemoteAudio extends RemoteTrack { + + Subscription onAudio(Consumer handler); // 10 ms of PCM each, in order + void setVolume(double volume); // playout gain, 0.0 to 10.0 +} +``` + +```java +Encoding.builder().maxBitrate(1_500_000).maxFrameRate(30).scaleDownBy(1.0).build(); +``` + +- A frame passed to `onFrame` is valid while the handler runs, as today. `frame.retain()` keeps it + longer. The library releases the frames it drops. +- Each frame subscription has its own thread, so a slow renderer never holds up WebRTC's decoder + or the peer's events. + +### `DataChannel` + +```java +public interface DataChannel extends AutoCloseable { + + String label(); + ChannelState state(); // CONNECTING, OPEN, CLOSING, CLOSED + CompletableFuture whenOpen(); + + CompletableFuture send(String text); // queued until the channel opens + CompletableFuture send(byte[] data); + CompletableFuture send(ByteBuffer data); // copied before the call returns + long bufferedAmount(); + + Subscription onMessage(Consumer handler); // early messages are kept for the first handler + Subscription onStateChange(Consumer handler); + + @Override + void close(); +} + +public final class Message { + + public boolean isText(); + public String text(); // UTF-8 + public byte[] bytes(); // a copy you own +} +``` + +```java +ChannelOptions.builder().ordered(false).maxRetransmits(0).protocol("json").negotiatedId(3).build(); +``` + +- `send` completes when the message reaches the transport. Above a high-water mark (1 MiB by + default) the library holds sends in Java, so a loop of `send(chunk).join()` slows down by itself + instead of filling native memory. +- The buffer that `RTCDataChannelObserver.onMessage` passes is copied into `Message`. Nothing a + handler gets refers to native memory. + +### `PeerStats` + +The numbers people look for first, computed from `RTCStatsReport`. Rates are taken between two +samples, so `onStats` fills them from its second call on. + +```java +public final class PeerStats { + + public Instant timestamp(); + public String candidateType(); // host, srflx, prflx, relay + public Duration roundTripTime(); + public long bytesSent(); + public long bytesReceived(); + public long outgoingBitrate(); // bits per second since the previous sample + public long incomingBitrate(); + public double packetLoss(); // inbound, 0.0 to 1.0 + public List outboundVideo();// width, height, framesPerSecond, codec, qualityLimitation + public List inboundVideo(); + public RTCStatsReport report(); // everything else +} +``` + +### `Subscription` and errors + +```java +public interface Subscription extends AutoCloseable { + @Override + void close(); // idempotent, no checked exception +} + +public class WebRtcException extends RuntimeException { } +public class NegotiationException extends WebRtcException { } // description rejected, glare between two impolite peers +public class PeerClosedException extends WebRtcException { } // operation pending at, or issued after, close +public class MediaException extends WebRtcException { } // device missing, busy, access denied, or opened with other options +public class ConfigurationException extends WebRtcException { } // e.g. microphone() on an AudioMode.CUSTOM runtime +public class BlockingCallException extends WebRtcException { } // join() or get() on an event thread +``` + +## Scenarios + +### Desktop app with Swing + +With the EDT as the event executor, every handler can touch Swing components directly. + +```java +WebRtc rtc = WebRtc.builder() + .eventExecutor(SwingUtilities::invokeLater) + .build(); + +Peer peer = rtc.createPeer(options); +peer.onStateChange(state -> statusLabel.setText(state.toString())); // on the EDT +peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(remoteView); // VideoView, phase 2: converts off the EDT, paints on it + } +}); + +hangUpButton.addActionListener(e -> peer.close()); // never blocks the EDT +``` + +### Headless bot: speak and listen + +```java +WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); + +CustomAudio voice = rtc.media().customAudio("tts"); +Peer peer = rtc.createPeer(options); +peer.addTrack(voice); + +peer.onTrack(track -> track.asAudio().onAudio(chunk -> recognizer.feed(chunk.samples()))); + +// From any thread, in any size. The library sends it in 10 ms chunks. +voice.push(textToSpeech.render("Hello"), PcmFormat.of(48_000, 1)); +``` + +### Data channel only + +```java +DataChannel chat = peer.createChannel("chat"); +chat.onMessage(message -> System.out.println("peer: " + message.text())); +chat.send("hello"); // sent as soon as the channel opens + +peer.onChannel(channel -> { // channels the other side opens + if (channel.label().equals("files")) { + channel.onMessage(message -> sink.write(message.bytes())); + } +}); +``` + +### WHEP playback, by hand + +This replaces `WhepExample`'s three latches and dummy camera track. A `Whep.play(rtc, uri)` helper +would wrap the same steps (phase 2). + +```java +Peer peer = rtc.createPeer(PeerOptions.builder() + .negotiation(Negotiation.MANUAL) + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .build()); +peer.onTrack(track -> log.info("Receiving {}", track.kind())); + +peer.negotiator().createOffer(Gathering.COMPLETE) + .thenApplyAsync(offer -> http.postSdp(WHEP_URL, offer.sdp), ioPool) // blocking HTTP off the event thread + .thenCompose(sdp -> peer.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, sdp))) + .thenCompose(done -> peer.whenConnected()) + .join(); // the main thread may wait +``` + +### Two peers in one test + +```java +try (WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build()) { + SignalingPair link = Signaling.loopback(); + + Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); + Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); + + CompletableFuture received = new CompletableFuture<>(); + callee.onChannel(channel -> channel.onMessage(message -> received.complete(message.text()))); + caller.createChannel("test").send("ping"); + + assertEquals("ping", received.get(10, TimeUnit.SECONDS)); +} +``` + +## How it maps onto today's classes + +Anything the client API doesn't cover stays reachable through `withNative` and +`configureFactory`. + +| New | Built on | +|---|---| +| `WebRtc` | `PeerConnectionFactory.Builder`, `AudioDeviceModule` or `HeadlessAudioDeviceModule`, `HardwareVideoEncoderFactory` and `HardwareVideoDecoderFactory` | +| `Peer` | `RTCPeerConnection`, an internal `PeerConnectionObserver` that only enqueues | +| automatic negotiation, `Negotiator` | `setLocalDescription(observer)` (implicit offer or answer), `setRemoteDescription`, rollback, `addIceCandidate(candidate, AddIceCandidateObserver)` | +| `Sender` | `RTCRtpSender` and `RTCRtpTransceiver`, disposed inside the library | +| `RemoteTrack` | `RTCRtpReceiver`, `VideoTrack.addSink`, `AudioTrack.addSink`, `AudioTrack.setVolume` | +| `LocalVideo`, `LocalAudio` | `VideoDeviceSource`, `VideoDesktopSource`, `CustomVideoSource`, `AudioTrackSource`, `CustomAudioSource` | +| `Devices` | `MediaDevices`, refreshed on a background thread when `DeviceChangeListener` fires | +| `DataChannel` | `RTCDataChannel`, `RTCDataChannelObserver`, `sendAsync` with `RTCDataChannelSendObserver` | +| `PeerStats` | `getStats(RTCStatsCollectorCallback)`, `RTCStatsReport` | + +Testing: + +- Each guarantee in the threading section gets a test: a handler that closes its own peer, a + handler that sleeps five seconds while a second peer keeps exchanging data, `join()` on an event + thread, event order under a thread pool, glare between two impolite peers, and the whole suite + under `-Pjni-check` for reference leaks. +- The examples and the get-started guide move to the new API once it lands. The low-level guides + stay for the advanced topics. + +## Phases + +1. **Core:** `WebRtc`, `Peer` and the threading model; signaling, JSON and loopback; automatic and + manual negotiation; local media, devices and remote tracks; data channels, errors and stats. +2. **Helpers:** `Whip.publish` and `Whep.play`; `VideoView` for Swing; media files from + `webrtc-java-media` as a `LocalTrack`; recording a `Peer` with `MediaRecorder`. +3. **Adoption:** port the examples, rewrite the get-started guide, and mark the client package as + the recommended entry point. + +## Implementation status + +Phase 1 lives in `webrtc/src/main/java/dev/onvoid/webrtc/client/`, with tests in +`webrtc/src/test/java/dev/onvoid/webrtc/client/`. It changes no native code; the only change +outside the package is the `exports dev.onvoid.webrtc.client` line in `module-info.java`. + +How the threading model is built: + +- Each peer has two `SerialExecutor`s over shared pools. The *control* executor owns the + `RTCPeerConnection` and all negotiation state; every WebRTC callback only posts to it. The + *event* executor runs application code and owns what the handlers have been told so far. +- Operations that the standard puts on the operations chain go through `PeerImpl.enqueue`, which + starts one only after the previous one finished and fails it after `operationTimeout`. +- Native observer completions are posted back to the control executor before anything is chained + on them, so no internal code ever runs on a WebRTC thread. +- Public futures are `ClientFuture`s completed on the event executor. `join()`/`get()` on an + unfinished one from any event thread throws `BlockingCallException`. On Java 9+ derived stages + refuse too, through `newIncompleteFuture`. +- Frame and audio subscriptions each get a `SerialExecutor` over a frame pool, with the drop + policies described above. +- `CustomAudio` is cut into 10 ms chunks and pushed from the runtime's timer thread, every 10 ms. + +Where the code differs from the API above: + +- `Sender.dtmf()` and `LocalVideo.switchCamera()` are left for later. +- `Devices.screens()` and `windows()` ask the system on every call; only the camera, microphone and + speaker lists are snapshots. The first `devices()` call reads them once, synchronously. +- `Peer.addTrack` without stream ids puts every such track of a peer into one shared stream, so a + browser plays its audio and video in sync. +- Additions: `WebRtc.audioMode()`, `RemoteTrack.isMuted()`/`isEnded()`, `LocalTrack.isEnded()`, + `CustomAudio.queuedMillis()`/`clear()`, `Message.size()`, `Negotiator.answer(offer, gathering)`. +- An impolite peer that ignored an offer reports the two-impolite-peers collision after 5 seconds + without an answer. + +Covered by tests (`PeerTests`, `MediaTests`, `SignalMessageTests`): +- data channel round trip; +- early messages kept; +- 2000 messages in order, with no overlapping handler calls; +- a handler closing its own peer; +- `join()` on an event thread refused; +- a blocked handler not delaying another peer; +- collision resolved by the polite peer, and reported between two impolite ones; +- manual negotiation with complete gathering; +- pending work failing on close; +- runtime close; +- stats; +- subscriptions removed; +- pushed audio and video arriving; +- a slow frame handler not slowing the connection; +- a removed sender ending the remote track; +- a closed local track leaving every peer; +- sender configuration; +- audio mode checks. + +Every test also fails if anything reached the runtime's error handler, including while it closed. + +Next: +- a guide page and an example; +- `Whip`/`Whep` and `VideoView`; +- porting the existing examples (phases 2 and 3). diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioChunk.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioChunk.java new file mode 100644 index 00000000..771ced57 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioChunk.java @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A piece of decoded audio, usually 10 ms of 16-bit PCM with interleaved + * channels. + */ +public final class AudioChunk { + + private final byte[] samples; + + private final int sampleRate; + + private final int channels; + + private final int frames; + + + AudioChunk(byte[] samples, int sampleRate, int channels, int frames) { + this.samples = samples; + this.sampleRate = sampleRate; + this.channels = channels; + this.frames = frames; + } + + /** + * @return the samples, 16-bit little-endian PCM with interleaved channels. + * The array belongs to the caller. + */ + public byte[] samples() { + return samples; + } + + /** + * @return the sample rate in Hz. + */ + public int sampleRate() { + return sampleRate; + } + + /** + * @return the number of channels. + */ + public int channels() { + return channels; + } + + /** + * @return the number of samples per channel. + */ + public int frames() { + return frames; + } + + @Override + public String toString() { + return "AudioChunk[" + frames + " frames, " + channels + " channels, " + sampleRate + " Hz]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioDispatcher.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioDispatcher.java new file mode 100644 index 00000000..2a4b26e1 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioDispatcher.java @@ -0,0 +1,107 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayDeque; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.audio.AudioTrackSink; + +/** + * Hands the audio of a track to a handler on a thread of its own, in order. + * When the handler falls more than {@link #CAPACITY} chunks behind, the oldest + * waiting chunk is dropped. + */ +final class AudioDispatcher implements AudioTrackSink { + + /** Five seconds of 10 ms chunks. */ + private static final int CAPACITY = 500; + + private final Consumer handler; + + private final SerialExecutor executor; + + private final Consumer errorHandler; + + private final ArrayDeque queue = new ArrayDeque<>(); + + private boolean scheduled; + + private boolean closed; + + + AudioDispatcher(Consumer handler, SerialExecutor executor, + Consumer errorHandler) { + this.handler = handler; + this.executor = executor; + this.errorHandler = errorHandler; + } + + @Override + public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { + if (bitsPerSample != 16) { + return; + } + + AudioChunk chunk = new AudioChunk(data.clone(), sampleRate, channels, frames); + + synchronized (queue) { + if (closed) { + return; + } + + queue.add(chunk); + + if (queue.size() > CAPACITY) { + queue.poll(); + } + if (!scheduled) { + scheduled = true; + executor.execute(this::drain); + } + } + } + + void close() { + synchronized (queue) { + closed = true; + queue.clear(); + } + } + + private void drain() { + while (true) { + AudioChunk chunk; + + synchronized (queue) { + chunk = queue.poll(); + + if (chunk == null) { + scheduled = false; + return; + } + } + + try { + handler.accept(chunk); + } + catch (Throwable e) { + errorHandler.accept(e); + } + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioMode.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioMode.java new file mode 100644 index 00000000..2b3f9205 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/AudioMode.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * Where the audio a runtime sends comes from. WebRTC feeds the audio a factory + * captures into every audio sender of that factory, so captured and pushed + * audio cannot be mixed within one runtime, and the choice is made when it is + * built. + */ +public enum AudioMode { + + /** Capture the microphone and play to the speakers. */ + DEVICES, + + /** Send audio pushed through {@link CustomAudio} and play remote audio to the speakers. */ + CUSTOM, + + /** + * Use no audio hardware at all. Send audio pushed through {@link CustomAudio} + * and read remote audio through {@link RemoteAudio#onAudio}. Suits servers and + * bots. + */ + HEADLESS + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/BlockingCallException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/BlockingCallException.java new file mode 100644 index 00000000..c30f4112 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/BlockingCallException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A future of this API was waited on from an event thread. That thread is the one that would complete the future, so waiting there would never return. Chain the next step with {@code thenAccept} or {@code thenCompose} instead. + */ +public class BlockingCallException extends WebRtcException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public BlockingCallException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public BlockingCallException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CameraOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CameraOptions.java new file mode 100644 index 00000000..cf4b5aa4 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CameraOptions.java @@ -0,0 +1,153 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.Objects; + +import dev.onvoid.webrtc.media.video.VideoDevice; + +/** + * Which camera to open and how. The camera's supported capability nearest to + * the requested resolution and frame rate is used. + * 
{@code
+ * CameraOptions.builder().device(camera).resolution(1280, 720).frameRate(30).build();
+ * }
+ */ +public final class CameraOptions { + + final VideoDevice device; + + final int width; + + final int height; + + final int frameRate; + + + private CameraOptions(Builder builder) { + this.device = builder.device; + this.width = builder.width; + this.height = builder.height; + this.frameRate = builder.frameRate; + } + + CameraOptions withDevice(VideoDevice camera) { + Builder builder = new Builder(); + builder.device = camera; + builder.width = width; + builder.height = height; + builder.frameRate = frameRate; + + return new CameraOptions(builder); + } + + /** + * @return a builder for the first camera with its first capability. + */ + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof CameraOptions)) { + return false; + } + + CameraOptions other = (CameraOptions) obj; + + return Objects.equals(device, other.device) && width == other.width + && height == other.height && frameRate == other.frameRate; + } + + @Override + public int hashCode() { + return Objects.hash(device, width, height, frameRate); + } + + @Override + public String toString() { + return "CameraOptions[device=" + (device == null ? "first" : device.getName()) + + ", " + width + "x" + height + ", " + frameRate + " fps]"; + } + + + + /** Builds {@link CameraOptions}. */ + public static final class Builder { + + private VideoDevice device; + + private int width; + + private int height; + + private int frameRate; + + + private Builder() { + } + + /** + * @param device one of {@link Devices#cameras()}, or null for the + * first. + * + * @return this builder. + */ + public Builder device(VideoDevice device) { + this.device = device; + return this; + } + + /** + * @param width the width wanted, in pixels. + * @param height the height wanted, in pixels. + * + * @return this builder. + */ + public Builder resolution(int width, int height) { + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("Invalid resolution " + width + "x" + height); + } + + this.width = width; + this.height = height; + return this; + } + + /** + * @param frameRate the frame rate wanted. + * + * @return this builder. + */ + public Builder frameRate(int frameRate) { + if (frameRate <= 0) { + throw new IllegalArgumentException("The frame rate must be positive, got " + frameRate); + } + + this.frameRate = frameRate; + return this; + } + + /** + * @return the options. + */ + public CameraOptions build() { + return new CameraOptions(this); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelOptions.java new file mode 100644 index 00000000..1daae6e1 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelOptions.java @@ -0,0 +1,179 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.time.Duration; + +import dev.onvoid.webrtc.RTCDataChannelInit; + +/** + * How a {@link DataChannel} delivers. The default is reliable and ordered. + * 
{@code
+ * ChannelOptions.builder().ordered(false).maxRetransmits(0).build();
+ * }
+ */ +public final class ChannelOptions { + + private final boolean ordered; + + private final int maxRetransmits; + + private final int maxPacketLifeTime; + + private final String protocol; + + private final int negotiatedId; + + + private ChannelOptions(Builder builder) { + this.ordered = builder.ordered; + this.maxRetransmits = builder.maxRetransmits; + this.maxPacketLifeTime = builder.maxPacketLifeTime; + this.protocol = builder.protocol; + this.negotiatedId = builder.negotiatedId; + } + + /** + * @return a builder for a reliable, ordered channel. + */ + public static Builder builder() { + return new Builder(); + } + + RTCDataChannelInit toInit() { + RTCDataChannelInit init = new RTCDataChannelInit(); + init.ordered = ordered; + init.maxRetransmits = maxRetransmits; + init.maxPacketLifeTime = maxPacketLifeTime; + init.protocol = protocol; + + if (negotiatedId >= 0) { + init.negotiated = true; + init.id = negotiatedId; + } + + return init; + } + + + + /** Builds {@link ChannelOptions}. */ + public static final class Builder { + + private boolean ordered = true; + + private int maxRetransmits = -1; + + private int maxPacketLifeTime = -1; + + private String protocol; + + private int negotiatedId = -1; + + + private Builder() { + } + + /** + * @param ordered false to let messages arrive out of order. + * + * @return this builder. + */ + public Builder ordered(boolean ordered) { + this.ordered = ordered; + return this; + } + + /** + * Makes the channel unreliable: a message is sent at most this many + * more times. Excludes {@link #maxPacketLifeTime(Duration)}. + * + * @param retransmits 0 or more. + * + * @return this builder. + */ + public Builder maxRetransmits(int retransmits) { + if (retransmits < 0) { + throw new IllegalArgumentException("Retransmits must be 0 or more, got " + retransmits); + } + + this.maxRetransmits = retransmits; + return this; + } + + /** + * Makes the channel unreliable: a message is retransmitted only for + * this long. Excludes {@link #maxRetransmits(int)}. + * + * @param lifeTime how long a message may be retransmitted. + * + * @return this builder. + */ + public Builder maxPacketLifeTime(Duration lifeTime) { + long millis = lifeTime.toMillis(); + + if (millis < 0 || millis > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Invalid packet life time " + lifeTime); + } + + this.maxPacketLifeTime = (int) millis; + return this; + } + + /** + * @param protocol the subprotocol the remote peer sees. + * + * @return this builder. + */ + public Builder protocol(String protocol) { + this.protocol = protocol; + return this; + } + + /** + * Creates the channel without announcing it: both peers create it + * with the same id. Such a channel never shows up in {@link + * Peer#onChannel}. + * + * @param id the id both peers use, 0 to 65534. + * + * @return this builder. + */ + public Builder negotiatedId(int id) { + if (id < 0 || id > 65534) { + throw new IllegalArgumentException("The id must be between 0 and 65534, got " + id); + } + + this.negotiatedId = id; + return this; + } + + /** + * @return the options. + * + * @throws IllegalStateException if both limits of an unreliable + * channel are set. + */ + public ChannelOptions build() { + if (maxRetransmits >= 0 && maxPacketLifeTime >= 0) { + throw new IllegalStateException("A channel is limited by retransmits or by packet life time, not both"); + } + + return new ChannelOptions(this); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelState.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelState.java new file mode 100644 index 00000000..abbe1bd0 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/ChannelState.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import dev.onvoid.webrtc.RTCDataChannelState; + +/** + * The state of a {@link DataChannel}. + */ +public enum ChannelState { + + /** Not open yet. Messages sent now are held until it opens. */ + CONNECTING, + + /** Messages can be sent and received. */ + OPEN, + + /** Closing. */ + CLOSING, + + /** Closed. This state is final. */ + CLOSED; + + static ChannelState of(RTCDataChannelState state) { + switch (state) { + case CONNECTING: + return CONNECTING; + case OPEN: + return OPEN; + case CLOSING: + return CLOSING; + default: + return CLOSED; + } + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/ClientFuture.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/ClientFuture.java new file mode 100644 index 00000000..95a4cce0 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/ClientFuture.java @@ -0,0 +1,88 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * The futures this API hands out. They complete on the event thread of their + * peer, so waiting for one there would never return: {@link #join()} and + * {@link #get()} refuse with a {@link BlockingCallException} instead of + * deadlocking. A future that is already done may be read anywhere. + *

+ * On Java 9 and later the futures derived from this one, with {@code + * thenApply} and the like, are of this class too and refuse the same way. + * + * @param the result type. + */ +final class ClientFuture extends CompletableFuture { + + static ClientFuture failed(Throwable error) { + ClientFuture future = new ClientFuture<>(); + future.completeExceptionally(error); + return future; + } + + static ClientFuture completed(T value) { + ClientFuture future = new ClientFuture<>(); + future.complete(value); + return future; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + checkBlocking(); + return super.get(); + } + + @Override + public T get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + checkBlocking(); + return super.get(timeout, unit); + } + + @Override + public T join() { + checkBlocking(); + return super.join(); + } + + /** + * Makes the futures derived from this one, on Java 9 and later, refuse to + * be waited on from an event thread as well. Java 8 has no such method and + * never calls it. + * + * @param the result type of the derived future. + * + * @return a new incomplete future. + */ + public CompletableFuture newIncompleteFuture() { + return new ClientFuture<>(); + } + + private void checkBlocking() { + if (!isDone() && SerialExecutor.onEventThread()) { + throw new BlockingCallException("A future was waited on from the event thread " + + Thread.currentThread().getName() + ", which is the thread that completes it. " + + "Chain the next step with thenAccept or thenCompose instead of calling join() or get()."); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/ConfigurationException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/ConfigurationException.java new file mode 100644 index 00000000..d598ce57 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/ConfigurationException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A call does not fit how the runtime was built, such as asking for the microphone of a runtime whose {@link AudioMode} sends custom audio. + */ +public class ConfigurationException extends WebRtcException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public ConfigurationException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public ConfigurationException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudio.java new file mode 100644 index 00000000..bbeaa2e9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudio.java @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * An audio track fed with audio the application pushes, made by {@link + * LocalMedia#customAudio(String)}. + *

+ * Audio may be pushed in pieces of any length, from any thread. The track + * cuts it into 10 ms chunks and sends one every 10 ms from a thread of its + * own, so pushing a whole second at once is as fine as pushing every 10 ms. + */ +public interface CustomAudio extends LocalAudio { + + /** + * Queues audio to send. + * + * @param pcm 16-bit little-endian PCM with interleaved channels, a whole + * number of samples. The array is copied. + * @param format the sample rate and channel count. A format other than the + * one before starts a new stream: audio still queued in the + * old format is sent first. + */ + void push(byte[] pcm, PcmFormat format); + + /** + * @return the audio queued and not sent yet, in milliseconds. + */ + long queuedMillis(); + + /** + * Drops the audio queued and not sent yet. + */ + void clear(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java new file mode 100644 index 00000000..d24ee024 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomAudioImpl.java @@ -0,0 +1,183 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayDeque; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; + +/** + * An audio track fed with audio the application pushes. Pushed audio is cut + * into 10 ms chunks, which the runtime's timer thread hands to the source one + * every 10 ms. That thread is the only one that pushes into the source, as + * {@link CustomAudioSource} requires. + */ +final class CustomAudioImpl extends LocalAudioImpl implements CustomAudio { + + private final CustomAudioSource source; + + // Guarded by queue. + + private final ArrayDeque queue = new ArrayDeque<>(); + + private PcmFormat format; + + private byte[] partial; + + private int partialLength; + + private ScheduledFuture ticker; + + private boolean stopped; + + + CustomAudioImpl(LocalMediaImpl media, AudioTrack track, CustomAudioSource source) { + super(media, track, source::dispose); + + this.source = source; + } + + @Override + public void push(byte[] pcm, PcmFormat pcmFormat) { + requireNonNull(pcm, "pcm"); + requireNonNull(pcmFormat, "format"); + + int frameSize = pcmFormat.channels() * 2; + + if (pcm.length % frameSize != 0) { + throw new IllegalArgumentException("The audio holds " + pcm.length + " bytes, which is no whole number of " + + frameSize + "-byte samples of " + pcmFormat); + } + + synchronized (queue) { + if (stopped) { + return; + } + + if (!pcmFormat.equals(format)) { + flushPartial(); + format = pcmFormat; + partial = new byte[pcmFormat.bytesPer10Ms()]; + partialLength = 0; + } + + int offset = 0; + + while (offset < pcm.length) { + int count = Math.min(partial.length - partialLength, pcm.length - offset); + + System.arraycopy(pcm, offset, partial, partialLength, count); + partialLength += count; + offset += count; + + if (partialLength == partial.length) { + queue.add(new Chunk(partial, format)); + partial = new byte[format.bytesPer10Ms()]; + partialLength = 0; + } + } + + if (ticker == null) { + ticker = rtc.timer.scheduleAtFixedRate(this::tick, 0, 10, TimeUnit.MILLISECONDS); + } + } + } + + @Override + public long queuedMillis() { + synchronized (queue) { + return queue.size() * 10L; + } + } + + @Override + public void clear() { + synchronized (queue) { + queue.clear(); + partialLength = 0; + } + } + + @Override + void beforeRelease() { + synchronized (queue) { + stopped = true; + queue.clear(); + + if (ticker != null) { + ticker.cancel(false); + ticker = null; + } + } + } + + /** Queues what is left of the previous format, filled with silence. */ + private void flushPartial() { + if (partialLength > 0) { + queue.add(new Chunk(partial, format)); + } + + partial = null; + partialLength = 0; + } + + private void tick() { + Chunk chunk; + + synchronized (queue) { + chunk = queue.poll(); + } + + if (chunk == null) { + return; + } + + synchronized (this) { + if (!isLive()) { + return; + } + + try { + source.pushAudio(chunk.data, 16, chunk.format.sampleRate(), chunk.format.channels(), + chunk.format.framesPer10Ms()); + } + catch (Throwable e) { + rtc.reportError(e); + } + } + } + + + + private static final class Chunk { + + final byte[] data; + + final PcmFormat format; + + + Chunk(byte[] data, PcmFormat format) { + this.data = data; + this.format = format; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideo.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideo.java new file mode 100644 index 00000000..138c9069 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideo.java @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A video track fed with frames the application pushes, made by {@link + * LocalMedia#customVideo(String)}. + */ +public interface CustomVideo extends LocalVideo { + + /** + * Sends a frame, as captured now. The frame is not released. + * + * @param frame the frame. + */ + void push(VideoFrame frame); + + /** + * Sends a frame with a timestamp of its own, for sources with timing of + * their own, such as a video file. The frame is not released. + * + * @param frame the frame. + * @param timestampMicros when the frame was captured, in microseconds on + * a clock of the application. + */ + void push(VideoFrame frame, long timestampMicros); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideoImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideoImpl.java new file mode 100644 index 00000000..7e2146b7 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/CustomVideoImpl.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * A video track fed with frames the application pushes. + */ +final class CustomVideoImpl extends LocalVideoImpl implements CustomVideo { + + private final CustomVideoSource source; + + + CustomVideoImpl(LocalMediaImpl media, VideoTrack track, CustomVideoSource source) { + super(media, track, source::dispose); + + this.source = source; + } + + @Override + public synchronized void push(VideoFrame frame) { + requireNonNull(frame, "frame"); + + if (isLive()) { + source.pushFrame(frame); + } + } + + @Override + public synchronized void push(VideoFrame frame, long timestampMicros) { + requireNonNull(frame, "frame"); + + if (isLive()) { + source.pushFrame(frame, timestampMicros); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java new file mode 100644 index 00000000..66d7fa4e --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * A data channel of a {@link Peer}, created with {@link Peer#createChannel} or + * opened by the remote peer. + *

+ * Handlers run on the event thread of the peer, in order with its other + * events. + */ +public interface DataChannel extends AutoCloseable { + + /** + * @return the label. + */ + String label(); + + /** + * @return the current state. + */ + ChannelState state(); + + /** + * @return a future that completes when the channel is open, or fails if it + * closes first. + */ + CompletableFuture whenOpen(); + + /** + * Sends a text message. Messages sent before the channel opens are held + * until it does. + * + * @param text the message. + * + * @return a future that completes when the message is handed to the + * transport. While a lot of data waits to be sent, the future completes + * only once the backlog shrank, so a loop that waits for each send slows + * down by itself. + */ + CompletableFuture send(String text); + + /** + * Sends a binary message. The array is copied before the call returns. + * + * @param data the message. + * + * @return a future that completes when the message is handed to the + * transport. + */ + CompletableFuture send(byte[] data); + + /** + * Sends the remaining bytes of a buffer as a binary message. They are + * copied before the call returns; the buffer's position does not change. + * + * @param data the message. + * + * @return a future that completes when the message is handed to the + * transport. + */ + CompletableFuture send(ByteBuffer data); + + /** + * @return the bytes sent and not yet passed to the network, including + * those held by this API. + */ + long bufferedAmount(); + + /** + * Calls the handler for each message received. Messages that arrive before + * the first handler is added are kept for it. + * + * @param handler called on the peer's event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onMessage(Consumer handler); + + /** + * Calls the handler with the current state, and then with every change. + * + * @param handler called on the peer's event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onStateChange(Consumer handler); + + /** + * Closes the channel. Messages not yet handed to the transport fail with a + * {@link PeerClosedException}. Never blocks, may be called more than once. + */ + @Override + void close(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java new file mode 100644 index 00000000..102a4820 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java @@ -0,0 +1,448 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.RTCDataChannel; +import dev.onvoid.webrtc.RTCDataChannelBuffer; +import dev.onvoid.webrtc.RTCDataChannelObserver; +import dev.onvoid.webrtc.RTCDataChannelSendObserver; + +/** + * The {@link DataChannel} of a {@link PeerImpl}. + *

+ * The native channel is used on the peer's control executor, where sends wait + * while the channel is not open or holds more than {@link #HIGH_WATER_MARK} + * bytes. Handlers run on the peer's event executor. + */ +final class DataChannelImpl implements DataChannel { + + /** Bytes the native channel may hold before sends wait in Java. */ + static final long HIGH_WATER_MARK = 1024 * 1024; + + private final PeerImpl peer; + + private final String label; + + /** Bytes sent and not yet handed to the transport. */ + private final AtomicLong held = new AtomicLong(); + + private volatile ChannelState state = ChannelState.CONNECTING; + + /** What the native channel holds, as last read on the control executor. */ + private volatile long nativeBuffered; + + // Confined to the peer's control executor. + + private RTCDataChannel channel; + + private final ArrayDeque pending = new ArrayDeque<>(); + + private long inFlight; + + private boolean released; + + // Confined to the peer's event executor. + + private ChannelState deliveredState = ChannelState.CONNECTING; + + private final List early = new ArrayList<>(); + + private final Handlers messageHandlers; + + private final Handlers stateHandlers; + + private final ClientFuture opened = new ClientFuture<>(); + + + /** + * A channel this side creates. It is bound once the control executor + * created the native channel. + */ + DataChannelImpl(PeerImpl peer, String label) { + this.peer = peer; + this.label = label; + this.messageHandlers = new Handlers<>(peer.rtc::reportError); + this.stateHandlers = new Handlers<>(peer.rtc::reportError); + } + + /** + * A channel the remote peer opened, called on WebRTC's signaling thread. + * It is observed at once, so that no message is missed. + */ + DataChannelImpl(PeerImpl peer, RTCDataChannel remote) { + this(peer, remote.getLabel()); + + this.channel = remote; + remote.registerObserver(new Observer(remote)); + + ChannelState current = ChannelState.of(remote.getState()); + + if (current != ChannelState.CONNECTING) { + peer.control.execute(() -> stateChanged(current)); + } + } + + /** Binds the native channel this side created, on the control executor. */ + void bind(RTCDataChannel created) { + channel = created; + created.registerObserver(new Observer(created)); + + ChannelState current = ChannelState.of(created.getState()); + + if (current != ChannelState.CONNECTING) { + stateChanged(current); + } + } + + @Override + public String label() { + return label; + } + + @Override + public ChannelState state() { + return state; + } + + @Override + public CompletableFuture whenOpen() { + return opened; + } + + @Override + public CompletableFuture send(String text) { + requireNonNull(text, "text"); + + return enqueue(text.getBytes(StandardCharsets.UTF_8), false); + } + + @Override + public CompletableFuture send(byte[] data) { + requireNonNull(data, "data"); + + return enqueue(data.clone(), true); + } + + @Override + public CompletableFuture send(ByteBuffer data) { + requireNonNull(data, "data"); + + byte[] copy = new byte[data.remaining()]; + data.duplicate().get(copy); + + return enqueue(copy, true); + } + + @Override + public long bufferedAmount() { + return held.get() + nativeBuffered; + } + + @Override + public Subscription onMessage(Consumer handler) { + requireNonNull(handler, "handler"); + + return peer.subscribe(() -> { + Subscription subscription = messageHandlers.add(handler); + + if (!early.isEmpty()) { + List kept = new ArrayList<>(early); + early.clear(); + + for (Message message : kept) { + peer.call(handler, message); + } + } + + return subscription; + }); + } + + @Override + public Subscription onStateChange(Consumer handler) { + requireNonNull(handler, "handler"); + + return peer.subscribe(() -> { + peer.call(handler, deliveredState); + return stateHandlers.add(handler); + }); + } + + @Override + public void close() { + peer.control.execute(() -> { + if (released) { + return; + } + if (channel == null) { + closeFromPeer(new PeerClosedException("The data channel " + label + " is closed")); + return; + } + + try { + channel.close(); + } + catch (Throwable e) { + closeFromPeer(new PeerClosedException("The data channel " + label + " is closed")); + } + }); + } + + @Override + public String toString() { + return "DataChannel[" + label + ", " + state + "]"; + } + + /** + * Fails what waits, lets go of the native channel and reports the channel + * closed. On the control executor. + */ + void closeFromPeer(WebRtcException error) { + if (released) { + return; + } + + released = true; + failPending(error); + + if (channel != null) { + try { + channel.unregisterObserver(); + channel.close(); + channel.dispose(); + } + catch (Throwable e) { + peer.rtc.reportError(e); + } + + channel = null; + } + + nativeBuffered = 0; + state = ChannelState.CLOSED; + peer.channelClosed(this); + + peer.events.execute(() -> deliverState(ChannelState.CLOSED, error)); + } + + private CompletableFuture enqueue(byte[] data, boolean binary) { + CompletableFuture sent = new CompletableFuture<>(); + held.addAndGet(data.length); + + peer.control.execute(() -> { + if (released || state == ChannelState.CLOSING || state == ChannelState.CLOSED) { + held.addAndGet(-data.length); + sent.completeExceptionally(new PeerClosedException("The data channel " + label + " is closed")); + return; + } + + pending.add(new Pending(data, binary, sent)); + pump(); + }); + + return peer.toPublic(sent); + } + + /** Hands waiting messages to the transport while it has room. */ + private void pump() { + if (released || channel == null || state != ChannelState.OPEN) { + return; + } + + nativeBuffered = channel.getBufferedAmount(); + + while (!pending.isEmpty() && nativeBuffered + inFlight < HIGH_WATER_MARK) { + Pending message = pending.poll(); + int size = message.data.length; + + inFlight += size; + + channel.sendAsync(new RTCDataChannelBuffer(ByteBuffer.wrap(message.data), message.binary), + new RTCDataChannelSendObserver() { + + @Override + public void onSuccess() { + peer.control.execute(() -> sendDone(message, size, null)); + } + + @Override + public void onFailure(String error) { + peer.control.execute(() -> sendDone(message, size, error)); + } + }); + } + } + + private void sendDone(Pending message, int size, String error) { + inFlight -= size; + held.addAndGet(-size); + + if (error == null) { + message.sent.complete(null); + } + else if (released || state != ChannelState.OPEN) { + message.sent.completeExceptionally(new PeerClosedException("The data channel " + label + " closed: " + error)); + } + else { + message.sent.completeExceptionally(new WebRtcException("Sending on the data channel " + label + " failed: " + error)); + } + + pump(); + } + + private void failPending(WebRtcException error) { + for (Pending message : pending) { + held.addAndGet(-message.data.length); + message.sent.completeExceptionally(error); + } + + pending.clear(); + } + + private void stateChanged(ChannelState newState) { + if (released) { + return; + } + + if (newState == ChannelState.CLOSED) { + closeFromPeer(new PeerClosedException("The data channel " + label + " is closed")); + return; + } + + state = newState; + + if (newState == ChannelState.OPEN) { + pump(); + } + else if (newState == ChannelState.CLOSING) { + failPending(new PeerClosedException("The data channel " + label + " is closing")); + } + + peer.events.execute(() -> deliverState(newState, null)); + } + + private void bufferChanged() { + if (!released && channel != null) { + pump(); + } + } + + // On the event executor. + + private void deliverState(ChannelState newState, WebRtcException error) { + if (newState == deliveredState || deliveredState == ChannelState.CLOSED) { + return; + } + + deliveredState = newState; + stateHandlers.accept(newState); + + if (newState == ChannelState.OPEN) { + opened.complete(this); + } + else if (newState == ChannelState.CLOSED) { + opened.completeExceptionally(error != null ? error + : new PeerClosedException("The data channel " + label + " is closed")); + + messageHandlers.clear(); + stateHandlers.clear(); + early.clear(); + } + } + + private void deliverMessage(Message message) { + if (deliveredState == ChannelState.CLOSED) { + return; + } + + if (messageHandlers.isEmpty()) { + early.add(message); + } + else { + messageHandlers.accept(message); + } + } + + + + /** + * Hears what the native channel reports, on a WebRTC thread, and only + * copies and posts it. + */ + private final class Observer implements RTCDataChannelObserver { + + private final RTCDataChannel observed; + + + Observer(RTCDataChannel observed) { + this.observed = observed; + } + + @Override + public void onBufferedAmountChange(long sentDataSize) { + peer.control.execute(DataChannelImpl.this::bufferChanged); + } + + @Override + public void onStateChange() { + // The state is read here, while the native channel surely exists. + ChannelState newState = ChannelState.of(observed.getState()); + + peer.control.execute(() -> stateChanged(newState)); + } + + @Override + public void onMessage(RTCDataChannelBuffer buffer) { + // The buffer is valid only during this call. + byte[] data = new byte[buffer.data.remaining()]; + buffer.data.duplicate().get(data); + + Message message = new Message(data, buffer.binary); + + peer.events.execute(() -> deliverMessage(message)); + } + } + + + + private static final class Pending { + + final byte[] data; + + final boolean binary; + + final CompletableFuture sent; + + + Pending(byte[] data, boolean binary, CompletableFuture sent) { + this.data = data; + this.binary = binary; + this.sent = sent; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DeferredSubscription.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DeferredSubscription.java new file mode 100644 index 00000000..47467d4b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DeferredSubscription.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A subscription handed out before its handler is registered on another + * thread. Closing it before then keeps the handler from being registered. + */ +final class DeferredSubscription implements Subscription { + + private Subscription registered; + + private boolean closed; + + + synchronized void set(Subscription subscription) { + if (closed) { + subscription.close(); + } + else { + registered = subscription; + } + } + + @Override + public void close() { + Subscription subscription; + + synchronized (this) { + if (closed) { + return; + } + + closed = true; + subscription = registered; + registered = null; + } + + if (subscription != null) { + subscription.close(); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java new file mode 100644 index 00000000..b215666e --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.audio.AudioDevice; +import dev.onvoid.webrtc.media.video.VideoDevice; +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; + +/** + * The media devices of the system. The lists are snapshots, kept current in + * the background, so reading them is cheap on any thread. + */ +public interface Devices { + + /** + * @return the cameras. + */ + List cameras(); + + /** + * @return the microphones. + */ + List microphones(); + + /** + * @return the speakers. + */ + List speakers(); + + /** + * Asks the system for its screens. Unlike the device lists, this is not a + * snapshot. + * + * @return the screens that can be captured. + */ + List screens(); + + /** + * Asks the system for its windows. Unlike the device lists, this is not a + * snapshot. + * + * @return the windows that can be captured. + */ + List windows(); + + /** + * Captures from another microphone, also in tracks that are sent now. + * + * @param device one of {@link #microphones()}. + * + * @return a future that completes once the microphone is used. + */ + CompletableFuture useMicrophone(AudioDevice device); + + /** + * Plays to other speakers. + * + * @param device one of {@link #speakers()}. + * + * @return a future that completes once the speakers are used. + */ + CompletableFuture useSpeaker(AudioDevice device); + + /** + * Calls the handler after a device was connected or disconnected. + * + * @param handler called on a background thread with the new snapshot. + * + * @return the subscription that removes the handler. + */ + Subscription onChange(Consumer handler); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java new file mode 100644 index 00000000..308e32d0 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java @@ -0,0 +1,224 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import dev.onvoid.webrtc.media.Device; +import dev.onvoid.webrtc.media.DeviceChangeListener; +import dev.onvoid.webrtc.media.MediaDevices; +import dev.onvoid.webrtc.media.audio.AudioDevice; +import dev.onvoid.webrtc.media.audio.AudioDeviceModuleBase; +import dev.onvoid.webrtc.media.video.VideoDevice; +import dev.onvoid.webrtc.media.video.desktop.DesktopCapturer; +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; +import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer; +import dev.onvoid.webrtc.media.video.desktop.WindowCapturer; + +/** + * The {@link Devices} of a runtime. The device lists are read once, then again + * on the media executor whenever the system reports a change. + */ +final class DevicesImpl implements Devices { + + private final LocalMediaImpl media; + + private final AudioDeviceModuleBase audioModule; + + private final Handlers handlers; + + private final DeviceChangeListener listener; + + private volatile List cameras = Collections.emptyList(); + + private volatile List microphones = Collections.emptyList(); + + private volatile List speakers = Collections.emptyList(); + + + DevicesImpl(LocalMediaImpl media, AudioDeviceModuleBase audioModule) { + this.media = media; + this.audioModule = audioModule; + this.handlers = new Handlers<>(media.rtc::reportError); + + refresh(); + + this.listener = new DeviceChangeListener() { + + @Override + public void deviceConnected(Device device) { + changed(); + } + + @Override + public void deviceDisconnected(Device device) { + changed(); + } + }; + + MediaDevices.addDeviceChangeListener(listener); + } + + @Override + public List cameras() { + return cameras; + } + + @Override + public List microphones() { + return microphones; + } + + @Override + public List speakers() { + return speakers; + } + + @Override + public List screens() { + return desktopSources(new ScreenCapturer()); + } + + @Override + public List windows() { + return desktopSources(new WindowCapturer()); + } + + @Override + public CompletableFuture useMicrophone(AudioDevice device) { + requireNonNull(device, "device"); + + return onMediaExecutor(() -> { + if (media.rtc.audioMode != AudioMode.DEVICES) { + throw new ConfigurationException("A runtime built with AudioMode." + media.rtc.audioMode + + " captures no microphone"); + } + + try { + audioModule.setRecordingDevice(device); + } + catch (Throwable e) { + // Most platforms switch only while not recording. + audioModule.stopRecording(); + audioModule.setRecordingDevice(device); + audioModule.initRecording(); + audioModule.startRecording(); + } + return null; + }); + } + + @Override + public CompletableFuture useSpeaker(AudioDevice device) { + requireNonNull(device, "device"); + + return onMediaExecutor(() -> { + if (media.rtc.audioMode == AudioMode.HEADLESS) { + throw new ConfigurationException("A runtime built with AudioMode.HEADLESS plays to no speakers"); + } + + try { + audioModule.setPlayoutDevice(device); + } + catch (Throwable e) { + // Most platforms switch only while not playing. + audioModule.stopPlayout(); + audioModule.setPlayoutDevice(device); + audioModule.initPlayout(); + audioModule.startPlayout(); + } + return null; + }); + } + + @Override + public Subscription onChange(Consumer handler) { + return handlers.add(requireNonNull(handler, "handler")); + } + + void dispose() { + MediaDevices.removeDeviceChangeListener(listener); + handlers.clear(); + } + + /** Called on a thread of the system, so it only posts. */ + private void changed() { + media.executor.execute(() -> { + refresh(); + handlers.accept(this); + }); + } + + private void refresh() { + cameras = read(MediaDevices::getVideoCaptureDevices); + microphones = read(MediaDevices::getAudioCaptureDevices); + speakers = read(MediaDevices::getAudioRenderDevices); + } + + private List read(Supplier> query) { + try { + List devices = query.get(); + + return devices == null + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(devices)); + } + catch (Throwable e) { + media.rtc.reportError(new MediaException("Reading the devices failed", e)); + return Collections.emptyList(); + } + } + + private static List desktopSources(DesktopCapturer capturer) { + try { + List sources = capturer.getDesktopSources(); + + return sources == null + ? Collections.emptyList() + : Collections.unmodifiableList(new ArrayList<>(sources)); + } + catch (Throwable e) { + throw new MediaException("Reading the capture sources failed: " + e.getMessage(), e); + } + finally { + capturer.dispose(); + } + } + + private CompletableFuture onMediaExecutor(Supplier action) { + CompletableFuture result = new CompletableFuture<>(); + + media.executor.execute(() -> { + try { + result.complete(action.get()); + } + catch (Throwable e) { + result.completeExceptionally(e instanceof WebRtcException + ? e : new MediaException("Switching the device failed: " + e.getMessage(), e)); + } + }); + + return result; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Encoding.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Encoding.java new file mode 100644 index 00000000..bbb5278f --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Encoding.java @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * Values that change how a {@link Sender} encodes. Values left unset keep + * what the sender has. + * 

{@code
+ * sender.configure(Encoding.builder().maxBitrate(1_500_000).maxFrameRate(30).build());
+ * }
+ */ +public final class Encoding { + + final Boolean active; + + final Integer maxBitrate; + + final Double maxFrameRate; + + final Double scaleDownBy; + + + private Encoding(Builder builder) { + this.active = builder.active; + this.maxBitrate = builder.maxBitrate; + this.maxFrameRate = builder.maxFrameRate; + this.scaleDownBy = builder.scaleDownBy; + } + + /** + * @return a builder with no values set. + */ + public static Builder builder() { + return new Builder(); + } + + + + /** Builds an {@link Encoding}. */ + public static final class Builder { + + private Boolean active; + + private Integer maxBitrate; + + private Double maxFrameRate; + + private Double scaleDownBy; + + + private Builder() { + } + + /** + * @param active false to pause sending without renegotiating. + * + * @return this builder. + */ + public Builder active(boolean active) { + this.active = active; + return this; + } + + /** + * @param bitsPerSecond the highest bitrate to send at. + * + * @return this builder. + */ + public Builder maxBitrate(int bitsPerSecond) { + if (bitsPerSecond <= 0) { + throw new IllegalArgumentException("The bitrate must be positive, got " + bitsPerSecond); + } + + this.maxBitrate = bitsPerSecond; + return this; + } + + /** + * @param framesPerSecond the highest frame rate of a video track. + * + * @return this builder. + */ + public Builder maxFrameRate(double framesPerSecond) { + if (framesPerSecond <= 0) { + throw new IllegalArgumentException("The frame rate must be positive, got " + framesPerSecond); + } + + this.maxFrameRate = framesPerSecond; + return this; + } + + /** + * @param factor how much smaller than captured a video track is sent, + * 1 or more. + * + * @return this builder. + */ + public Builder scaleDownBy(double factor) { + if (factor < 1) { + throw new IllegalArgumentException("The factor must be 1 or more, got " + factor); + } + + this.scaleDownBy = factor; + return this; + } + + /** + * @return the encoding values. + */ + public Encoding build() { + return new Encoding(this); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDelivery.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDelivery.java new file mode 100644 index 00000000..6ca60bb1 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDelivery.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * What a video frame subscription does with frames when its handler falls + * behind. Frames that are dropped are released. + */ +public final class FrameDelivery { + + /** Keeps only the newest frame: the handler always gets the latest picture. */ + public static final FrameDelivery LATEST = new FrameDelivery(1); + + final int capacity; + + + private FrameDelivery(int capacity) { + this.capacity = capacity; + } + + /** + * Queues frames, and drops the oldest one when the queue is full. + * + * @param capacity the number of frames the queue holds. + * + * @return the delivery. + */ + public static FrameDelivery queue(int capacity) { + if (capacity < 1) { + throw new IllegalArgumentException("The capacity must be 1 or more, got " + capacity); + } + + return capacity == 1 ? LATEST : new FrameDelivery(capacity); + } + + @Override + public String toString() { + return capacity == 1 ? "FrameDelivery.LATEST" : "FrameDelivery.queue(" + capacity + ")"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java new file mode 100644 index 00000000..f48e09a6 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayDeque; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +/** + * Hands the frames of a video track to a handler on a thread of its own, so + * that a slow handler holds up neither WebRTC nor any other subscription. + * Frames that wait are retained; those dropped when the queue is full, and + * those the handler got, are released. + */ +final class FrameDispatcher implements VideoTrackSink { + + private final Consumer handler; + + private final int capacity; + + private final SerialExecutor executor; + + private final Consumer errorHandler; + + private final ArrayDeque queue = new ArrayDeque<>(); + + private boolean scheduled; + + private boolean closed; + + + FrameDispatcher(Consumer handler, FrameDelivery delivery, + SerialExecutor executor, Consumer errorHandler) { + this.handler = handler; + this.capacity = delivery.capacity; + this.executor = executor; + this.errorHandler = errorHandler; + } + + @Override + public void onVideoFrame(VideoFrame frame) { + VideoFrame dropped = null; + + synchronized (queue) { + if (closed) { + return; + } + + frame.retain(); + queue.add(frame); + + if (queue.size() > capacity) { + dropped = queue.poll(); + } + if (!scheduled) { + scheduled = true; + executor.execute(this::drain); + } + } + + if (dropped != null) { + dropped.release(); + } + } + + void close() { + synchronized (queue) { + closed = true; + + for (VideoFrame frame : queue) { + frame.release(); + } + + queue.clear(); + } + } + + private void drain() { + while (true) { + VideoFrame frame; + + synchronized (queue) { + frame = queue.poll(); + + if (frame == null) { + scheduled = false; + return; + } + } + + try { + handler.accept(frame); + } + catch (Throwable e) { + errorHandler.accept(e); + } + finally { + frame.release(); + } + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java new file mode 100644 index 00000000..86fb7d54 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * When a description created by the {@link Negotiator} is handed back. + */ +public enum Gathering { + + /** At once. Candidates follow through the {@link Signaling}, if there is one. */ + TRICKLE, + + /** Once ICE gathering is complete, with all local candidates in the description. */ + COMPLETE + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Handlers.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Handlers.java new file mode 100644 index 00000000..30cc9c75 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Handlers.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +/** + * The handlers registered for one kind of event. They are called in the order + * they were added, each in isolation: one that throws does not keep the others + * from being called. + * + * @param the event type. + */ +final class Handlers { + + private final CopyOnWriteArrayList> entries = new CopyOnWriteArrayList<>(); + + private final Consumer errorHandler; + + + Handlers(Consumer errorHandler) { + this.errorHandler = errorHandler; + } + + Subscription add(Consumer handler) { + // Wrapped, so that the same handler added twice is two subscriptions. + Entry entry = new Entry<>(handler); + entries.add(entry); + + return () -> entries.remove(entry); + } + + void accept(T value) { + for (Entry entry : entries) { + try { + entry.handler.accept(value); + } + catch (Throwable e) { + errorHandler.accept(e); + } + } + } + + boolean isEmpty() { + return entries.isEmpty(); + } + + void clear() { + entries.clear(); + } + + + + private static final class Entry { + + final Consumer handler; + + + Entry(Consumer handler) { + this.handler = handler; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Json.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Json.java new file mode 100644 index 00000000..4b60096b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Json.java @@ -0,0 +1,298 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Just enough JSON for signaling messages, so that the module needs no JSON + * library. Objects become {@code Map}s, arrays {@code List}s, numbers {@code + * Double}s. + */ +final class Json { + + private final String text; + + private int pos; + + + private Json(String text) { + this.text = text; + } + + static Object parse(String text) { + Json json = new Json(text); + json.skipWhitespace(); + + Object value = json.readValue(); + + json.skipWhitespace(); + + if (json.pos != text.length()) { + throw json.error("Unexpected text after the JSON value"); + } + + return value; + } + + static void writeString(StringBuilder out, String value) { + out.append('"'); + + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + + switch (c) { + case '"': + out.append("\\\""); + break; + case '\\': + out.append("\\\\"); + break; + case '\n': + out.append("\\n"); + break; + case '\r': + out.append("\\r"); + break; + case '\t': + out.append("\\t"); + break; + case '\b': + out.append("\\b"); + break; + case '\f': + out.append("\\f"); + break; + default: + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } + else { + out.append(c); + } + } + } + + out.append('"'); + } + + private Object readValue() { + if (pos >= text.length()) { + throw error("Unexpected end of JSON"); + } + + char c = text.charAt(pos); + + switch (c) { + case '{': + return readObject(); + case '[': + return readArray(); + case '"': + return readString(); + case 't': + expect("true"); + return Boolean.TRUE; + case 'f': + expect("false"); + return Boolean.FALSE; + case 'n': + expect("null"); + return null; + default: + if (c == '-' || (c >= '0' && c <= '9')) { + return readNumber(); + } + throw error("Unexpected character '" + c + "'"); + } + } + + private Map readObject() { + Map object = new LinkedHashMap<>(); + pos++; + skipWhitespace(); + + if (peek() == '}') { + pos++; + return object; + } + + while (true) { + skipWhitespace(); + + if (peek() != '"') { + throw error("Expected a member name"); + } + + String name = readString(); + skipWhitespace(); + + if (peek() != ':') { + throw error("Expected ':'"); + } + + pos++; + skipWhitespace(); + object.put(name, readValue()); + skipWhitespace(); + + char c = peek(); + pos++; + + if (c == '}') { + return object; + } + if (c != ',') { + throw error("Expected ',' or '}'"); + } + } + } + + private List readArray() { + List array = new ArrayList<>(); + pos++; + skipWhitespace(); + + if (peek() == ']') { + pos++; + return array; + } + + while (true) { + skipWhitespace(); + array.add(readValue()); + skipWhitespace(); + + char c = peek(); + pos++; + + if (c == ']') { + return array; + } + if (c != ',') { + throw error("Expected ',' or ']'"); + } + } + } + + private String readString() { + StringBuilder out = new StringBuilder(); + pos++; + + while (true) { + if (pos >= text.length()) { + throw error("Unterminated string"); + } + + char c = text.charAt(pos++); + + if (c == '"') { + return out.toString(); + } + if (c != '\\') { + out.append(c); + continue; + } + if (pos >= text.length()) { + throw error("Unterminated escape"); + } + + char e = text.charAt(pos++); + + switch (e) { + case '"': + case '\\': + case '/': + out.append(e); + break; + case 'n': + out.append('\n'); + break; + case 'r': + out.append('\r'); + break; + case 't': + out.append('\t'); + break; + case 'b': + out.append('\b'); + break; + case 'f': + out.append('\f'); + break; + case 'u': + if (pos + 4 > text.length()) { + throw error("Incomplete unicode escape"); + } + try { + out.append((char) Integer.parseInt(text.substring(pos, pos + 4), 16)); + } + catch (NumberFormatException ex) { + throw error("Invalid unicode escape"); + } + pos += 4; + break; + default: + throw error("Invalid escape '\\" + e + "'"); + } + } + } + + private Double readNumber() { + int start = pos; + + while (pos < text.length() && "+-0123456789.eE".indexOf(text.charAt(pos)) >= 0) { + pos++; + } + + try { + return Double.valueOf(text.substring(start, pos)); + } + catch (NumberFormatException e) { + throw error("Invalid number"); + } + } + + private void expect(String word) { + if (!text.startsWith(word, pos)) { + throw error("Expected '" + word + "'"); + } + + pos += word.length(); + } + + private char peek() { + if (pos >= text.length()) { + throw error("Unexpected end of JSON"); + } + + return text.charAt(pos); + } + + private void skipWhitespace() { + while (pos < text.length() && Character.isWhitespace(text.charAt(pos))) { + pos++; + } + } + + private IllegalArgumentException error(String message) { + return new IllegalArgumentException(message + " at position " + pos); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java new file mode 100644 index 00000000..fda582a2 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudio.java @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * An audio track this side sends. + */ +public interface LocalAudio extends LocalTrack { + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java new file mode 100644 index 00000000..3d628718 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalAudioImpl.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import dev.onvoid.webrtc.media.audio.AudioTrack; + +/** + * A local audio track, from the microphone or pushed audio. + */ +class LocalAudioImpl extends LocalTrackImpl implements LocalAudio { + + LocalAudioImpl(LocalMediaImpl media, AudioTrack track, Runnable releaseSource) { + super(media, track, MediaKind.AUDIO, releaseSource); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java new file mode 100644 index 00000000..90858337 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import dev.onvoid.webrtc.media.audio.AudioOptions; + +/** + * Creates the tracks a runtime sends, from {@link WebRtc#media()}. Every track + * belongs to the runtime and is closed with it, unless it was closed before. + */ +public interface LocalMedia { + + /** + * Returns the devices. The first call reads them from the system; later + * calls return a snapshot that is kept current in the background. + * + * @return the cameras, microphones and speakers. + */ + Devices devices(); + + /** + * Opens the first camera with its first capability. + * + * @return the camera's track, capturing. + * + * @throws MediaException if there is no camera or it cannot be opened. + */ + LocalVideo camera(); + + /** + * Opens a camera. A camera that is already open is shared when the options + * match. + * + * @param options the camera, resolution and frame rate. + * + * @return the camera's track, capturing. + * + * @throws MediaException if the camera cannot be opened, or is already open + * with other options. + */ + LocalVideo camera(CameraOptions options); + + /** + * Captures the microphone selected in {@link Devices}, with echo + * cancellation, noise suppression and gain control. + * + * @return the microphone's track. + * + * @throws ConfigurationException if the runtime's {@link AudioMode} is not + * {@link AudioMode#DEVICES}. + */ + LocalAudio microphone(); + + /** + * Captures the microphone selected in {@link Devices}. + * + * @param options the audio processing to apply. + * + * @return the microphone's track. + * + * @throws ConfigurationException if the runtime's {@link AudioMode} is not + * {@link AudioMode#DEVICES}. + */ + LocalAudio microphone(AudioOptions options); + + /** + * Captures a screen or a window. + * + * @param options the screen or window and the frame rate. + * + * @return the capture's track. + */ + LocalVideo screen(ScreenOptions options); + + /** + * Creates a video track fed with frames the application pushes. + * + * @param label the track label. + * + * @return the track. + */ + CustomVideo customVideo(String label); + + /** + * Creates an audio track fed with audio the application pushes. + * + * @param label the track label. + * + * @return the track. + * + * @throws ConfigurationException if the runtime's {@link AudioMode} is + * {@link AudioMode#DEVICES}. + */ + CustomAudio customAudio(String label); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java new file mode 100644 index 00000000..6d62ec09 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java @@ -0,0 +1,368 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; + +import dev.onvoid.webrtc.media.MediaDevices; +import dev.onvoid.webrtc.media.audio.AudioDeviceModuleBase; +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.VideoCaptureCapability; +import dev.onvoid.webrtc.media.video.VideoDesktopSource; +import dev.onvoid.webrtc.media.video.VideoDevice; +import dev.onvoid.webrtc.media.video.VideoDeviceSource; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * The {@link LocalMedia} of a {@link WebRtc} runtime. It keeps every track it + * made, to close them with the runtime, and shares one capture per camera. + */ +final class LocalMediaImpl implements LocalMedia { + + final WebRtc rtc; + + /** Releases tracks and runs their ended handlers, off WebRTC's and the application's threads. */ + final SerialExecutor executor; + + private final AudioDeviceModuleBase audioModule; + + private final Set tracks = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + // Guarded by cameras. + + private final Map cameras = new HashMap<>(); + + // Guarded by this. + + private DevicesImpl devices; + + + LocalMediaImpl(WebRtc rtc, AudioDeviceModuleBase audioModule) { + this.rtc = rtc; + this.audioModule = audioModule; + this.executor = new SerialExecutor(rtc.controlPool, rtc::reportError, false, null); + } + + @Override + public synchronized Devices devices() { + rtc.checkOpen(); + + if (devices == null) { + devices = new DevicesImpl(this, audioModule); + } + + return devices; + } + + @Override + public LocalVideo camera() { + return camera(CameraOptions.builder().build()); + } + + @Override + public LocalVideo camera(CameraOptions options) { + requireNonNull(options, "options"); + rtc.checkOpen(); + + VideoDevice device = options.device; + + if (device == null) { + List available = devices().cameras(); + + if (available.isEmpty()) { + throw new MediaException("No camera was found"); + } + + device = available.get(0); + } + + CameraOptions resolved = options.withDevice(device); + SharedCamera camera = openCamera(resolved); + + try { + VideoTrack track = rtc.factory.createVideoTrack(newId("camera"), camera.source); + + return register(new LocalVideoImpl(this, track, () -> releaseCamera(camera))); + } + catch (RuntimeException e) { + releaseCamera(camera); + throw new MediaException("The track of the camera " + device.getName() + " could not be created", e); + } + } + + @Override + public LocalAudio microphone() { + AudioOptions options = new AudioOptions(); + options.echoCancellation = true; + options.noiseSuppression = true; + options.autoGainControl = true; + options.highpassFilter = true; + + return microphone(options); + } + + @Override + public LocalAudio microphone(AudioOptions options) { + requireNonNull(options, "options"); + rtc.checkOpen(); + + if (rtc.audioMode != AudioMode.DEVICES) { + throw new ConfigurationException("The runtime was built with AudioMode." + rtc.audioMode + + ", which sends pushed audio only. Build it with AudioMode.DEVICES to capture the microphone, " + + "or use customAudio() to push audio"); + } + + AudioTrackSource source = rtc.factory.createAudioSource(options); + + try { + AudioTrack track = rtc.factory.createAudioTrack(newId("microphone"), source); + + return register(new LocalAudioImpl(this, track, source::dispose)); + } + catch (RuntimeException e) { + source.dispose(); + throw new MediaException("The microphone track could not be created", e); + } + } + + @Override + public LocalVideo screen(ScreenOptions options) { + requireNonNull(options, "options"); + rtc.checkOpen(); + + VideoDesktopSource source = new VideoDesktopSource(); + + try { + source.setSourceId(options.source.id, options.window); + source.setFrameRate(options.frameRate); + + if (options.maxWidth > 0) { + source.setMaxFrameSize(options.maxWidth, options.maxHeight); + } + + source.start(); + + VideoTrack track = rtc.factory.createVideoTrack(newId(options.window ? "window" : "screen"), source); + + return register(new LocalVideoImpl(this, track, () -> { + source.stop(); + source.dispose(); + })); + } + catch (RuntimeException e) { + source.dispose(); + throw new MediaException("Capturing " + options.source.title + " failed: " + e.getMessage(), e); + } + } + + @Override + public CustomVideo customVideo(String label) { + requireNonNull(label, "label"); + rtc.checkOpen(); + + CustomVideoSource source = new CustomVideoSource(); + + try { + VideoTrack track = rtc.factory.createVideoTrack(label, source); + + return register(new CustomVideoImpl(this, track, source)); + } + catch (RuntimeException e) { + source.dispose(); + throw e; + } + } + + @Override + public CustomAudio customAudio(String label) { + requireNonNull(label, "label"); + rtc.checkOpen(); + + if (rtc.audioMode == AudioMode.DEVICES) { + throw new ConfigurationException("The runtime was built with AudioMode.DEVICES, which sends the " + + "microphone. WebRTC feeds captured audio into every audio sender of a runtime, so pushed " + + "audio needs a runtime built with AudioMode.CUSTOM or AudioMode.HEADLESS"); + } + + CustomAudioSource source = new CustomAudioSource(); + + try { + AudioTrack track = rtc.factory.createAudioTrack(label, source); + + return register(new CustomAudioImpl(this, track, source)); + } + catch (RuntimeException e) { + source.dispose(); + throw e; + } + } + + List> closeAll() { + List> closing = new ArrayList<>(); + + for (LocalTrackImpl track : new ArrayList<>(tracks)) { + closing.add(track.closeInternal()); + } + + return closing; + } + + synchronized void dispose() { + if (devices != null) { + devices.dispose(); + devices = null; + } + } + + void forget(LocalTrackImpl track) { + tracks.remove(track); + } + + private T register(T track) { + tracks.add(track); + + if (rtc.isClosing()) { + // The runtime began to close meanwhile and may have missed it. + track.closeInternal(); + } + + return track; + } + + private SharedCamera openCamera(CameraOptions options) { + String key = options.device.getDescriptor(); + + synchronized (cameras) { + SharedCamera camera = cameras.get(key); + + if (camera != null) { + if (!camera.options.equals(options)) { + throw new MediaException("The camera " + options.device.getName() + " is already open with " + + camera.options + ". Open it with the same options to share it"); + } + + camera.users++; + return camera; + } + + VideoDeviceSource source = new VideoDeviceSource(); + + try { + source.setVideoCaptureDevice(options.device); + + VideoCaptureCapability capability = nearest( + MediaDevices.getVideoCaptureCapabilities(options.device), options); + + if (capability != null) { + source.setVideoCaptureCapability(capability); + } + + source.start(); + } + catch (Throwable e) { + source.dispose(); + throw new MediaException("The camera " + options.device.getName() + " could not be opened: " + + e.getMessage(), e); + } + + camera = new SharedCamera(key, options, source); + camera.users = 1; + cameras.put(key, camera); + + return camera; + } + } + + private void releaseCamera(SharedCamera camera) { + synchronized (cameras) { + if (--camera.users > 0) { + return; + } + + cameras.remove(camera.key); + } + + camera.source.stop(); + camera.source.dispose(); + } + + private static VideoCaptureCapability nearest(List capabilities, + CameraOptions options) { + if (capabilities == null || capabilities.isEmpty()) { + return null; + } + + VideoCaptureCapability best = capabilities.get(0); + long bestScore = Long.MAX_VALUE; + + for (VideoCaptureCapability capability : capabilities) { + long score = 0; + + if (options.width > 0) { + score += 1000L * (Math.abs(capability.width - options.width) + + Math.abs(capability.height - options.height)); + } + if (options.frameRate > 0) { + score += Math.abs(capability.frameRate - options.frameRate); + } + if (score < bestScore) { + best = capability; + bestScore = score; + } + } + + return best; + } + + private static String newId(String kind) { + return kind + "-" + UUID.randomUUID(); + } + + + + private static final class SharedCamera { + + final String key; + + final CameraOptions options; + + final VideoDeviceSource source; + + int users; + + + SharedCamera(String key, CameraOptions options, VideoDeviceSource source) { + this.key = key; + this.options = options; + this.source = source; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrack.java new file mode 100644 index 00000000..a5ccd193 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrack.java @@ -0,0 +1,70 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A track this side sends, made by {@link WebRtc#media()}. One track may be sent + * on any number of peers. + */ +public interface LocalTrack extends AutoCloseable { + + /** + * @return the track id. + */ + String id(); + + /** + * @return audio or video. + */ + MediaKind kind(); + + /** + * @return false while the track sends black frames or silence. + */ + boolean isEnabled(); + + /** + * Mutes or unmutes the track. A muted track sends black frames or silence, + * while its capture stays open. + * + * @param enabled false to mute. + */ + void setEnabled(boolean enabled); + + /** + * Calls the handler once the track ended. A handler added after that is + * called at once. + * + * @param handler called on a background thread. + * + * @return the subscription that removes the handler. + */ + Subscription onEnded(Runnable handler); + + /** + * @return true once the track is closed. + */ + boolean isEnded(); + + /** + * Stops sending the track on every peer, without renegotiating, and stops + * its capture. Never blocks, may be called more than once. + */ + @Override + void close(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java new file mode 100644 index 00000000..c71bae0b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import dev.onvoid.webrtc.media.MediaStreamTrack; + +/** + * A {@link LocalTrack} of a {@link WebRtc} runtime. + *

+ * Calls on the native track and source happen under this object's lock and + * only while the track has not ended, so that closing never races with them. + * Closing first makes every sender let go of the track, each on its peer's + * control executor, and only then releases the native track and source. + */ +abstract class LocalTrackImpl implements LocalTrack { + + final WebRtc rtc; + + final LocalMediaImpl media; + + private final MediaStreamTrack track; + + private final MediaKind kind; + + private final String id; + + private final Runnable releaseSource; + + private final CompletableFuture closed = new CompletableFuture<>(); + + private volatile boolean enabled = true; + + // Guarded by this. + + private final List senders = new ArrayList<>(); + + private final List endedHandlers = new ArrayList<>(); + + private boolean ended; + + private boolean endFired; + + + LocalTrackImpl(LocalMediaImpl media, MediaStreamTrack track, MediaKind kind, + Runnable releaseSource) { + this.rtc = media.rtc; + this.media = media; + this.track = track; + this.kind = kind; + this.id = track.getId(); + this.releaseSource = releaseSource; + } + + @Override + public String id() { + return id; + } + + @Override + public MediaKind kind() { + return kind; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public synchronized void setEnabled(boolean enabled) { + if (ended) { + return; + } + + track.setEnabled(enabled); + this.enabled = enabled; + } + + @Override + public Subscription onEnded(Runnable handler) { + requireNonNull(handler, "handler"); + + synchronized (this) { + if (!endFired) { + endedHandlers.add(handler); + + return () -> { + synchronized (LocalTrackImpl.this) { + endedHandlers.remove(handler); + } + }; + } + } + + media.executor.execute(handler); + + return () -> { }; + } + + @Override + public synchronized boolean isEnded() { + return ended; + } + + @Override + public void close() { + closeInternal(); + } + + @Override + public String toString() { + return getClass().getSimpleName().replace("Impl", "") + "[" + id + "]"; + } + + MediaStreamTrack nativeTrack() { + return track; + } + + synchronized boolean register(SenderImpl sender) { + if (ended) { + return false; + } + + senders.add(sender); + return true; + } + + synchronized void unregister(SenderImpl sender) { + senders.remove(sender); + } + + /** + * @return true while the native track may be used, which callers check + * under this object's lock. + */ + boolean isLive() { + return !ended; + } + + CompletableFuture closeInternal() { + List using; + + synchronized (this) { + if (ended) { + return closed; + } + + ended = true; + using = new ArrayList<>(senders); + senders.clear(); + } + + List> detached = new ArrayList<>(); + + for (SenderImpl sender : using) { + detached.add(sender.trackClosed(this)); + } + + CompletableFuture.allOf(detached.toArray(new CompletableFuture[0])) + .whenComplete((ignored, error) -> media.executor.execute(this::release)); + + return closed; + } + + /** + * Called once, before the native track is released, while no other call + * uses it. + */ + void beforeRelease() { + } + + private void release() { + try { + beforeRelease(); + track.dispose(); + releaseSource.run(); + } + catch (Throwable e) { + rtc.reportError(e); + } + + media.forget(this); + + List handlers; + + synchronized (this) { + endFired = true; + handlers = new ArrayList<>(endedHandlers); + endedHandlers.clear(); + } + + for (Runnable handler : handlers) { + try { + handler.run(); + } + catch (Throwable e) { + rtc.reportError(e); + } + } + + closed.complete(null); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java new file mode 100644 index 00000000..ed9d2458 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A video track this side sends. + */ +public interface LocalVideo extends LocalTrack { + + /** + * Calls the handler with the frames the track sends, for a local preview. + * When the handler falls behind, it gets the newest frame next. + * + * @param handler called on a thread of this subscription. A frame is valid + * while the handler runs; call {@code retain()} to keep it + * longer and {@code release()} when done. + * + * @return the subscription that stops the frames. + */ + Subscription onFrame(Consumer handler); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java new file mode 100644 index 00000000..fcb09530 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * A local video track, from a camera, a screen or pushed frames. + */ +class LocalVideoImpl extends LocalTrackImpl implements LocalVideo { + + private final VideoTrack track; + + // Guarded by this. + + private final List dispatchers = new ArrayList<>(); + + + LocalVideoImpl(LocalMediaImpl media, VideoTrack track, Runnable releaseSource) { + super(media, track, MediaKind.VIDEO, releaseSource); + + this.track = track; + } + + @Override + public Subscription onFrame(Consumer handler) { + requireNonNull(handler, "handler"); + + SerialExecutor executor = new SerialExecutor(rtc.framePool, rtc::reportError, false, null); + FrameDispatcher dispatcher = new FrameDispatcher(handler, FrameDelivery.LATEST, executor, rtc::reportError); + + synchronized (this) { + if (!isLive()) { + return () -> { }; + } + + dispatchers.add(dispatcher); + track.addSink(dispatcher); + } + + return () -> { + dispatcher.close(); + + synchronized (LocalVideoImpl.this) { + if (dispatchers.remove(dispatcher) && isLive()) { + track.removeSink(dispatcher); + } + } + }; + } + + @Override + void beforeRelease() { + List attached; + + synchronized (this) { + attached = new ArrayList<>(dispatchers); + dispatchers.clear(); + } + + for (FrameDispatcher dispatcher : attached) { + track.removeSink(dispatcher); + dispatcher.close(); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaException.java new file mode 100644 index 00000000..c2e8ee7a --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A capture device is missing, busy, refused access, or already open with other options. + */ +public class MediaException extends WebRtcException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public MediaException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public MediaException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaKind.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaKind.java new file mode 100644 index 00000000..048f88d6 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/MediaKind.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * The kind of a media track. + */ +public enum MediaKind { + + /** An audio track. */ + AUDIO, + + /** A video track. */ + VIDEO + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Message.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Message.java new file mode 100644 index 00000000..81a382fd --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Message.java @@ -0,0 +1,69 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.nio.charset.StandardCharsets; + +/** + * A message received on a {@link DataChannel}. It holds a copy of the data and + * may be kept as long as needed. + */ +public final class Message { + + private final byte[] data; + + private final boolean binary; + + + Message(byte[] data, boolean binary) { + this.data = data; + this.binary = binary; + } + + /** + * @return true when the message was sent as text. + */ + public boolean isText() { + return !binary; + } + + /** + * @return the message decoded as UTF-8. + */ + public String text() { + return new String(data, StandardCharsets.UTF_8); + } + + /** + * @return a copy of the message's bytes. + */ + public byte[] bytes() { + return data.clone(); + } + + /** + * @return the number of bytes. + */ + public int size() { + return data.length; + } + + @Override + public String toString() { + return binary ? "Message[" + data.length + " bytes]" : "Message[" + text() + "]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiation.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiation.java new file mode 100644 index 00000000..8a17ac55 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiation.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * Who drives the offer and answer exchange of a peer. + */ +public enum Negotiation { + + /** + * The peer negotiates by itself, with perfect negotiation, whenever tracks + * or channels change. Requires a {@link Signaling}. + */ + AUTOMATIC, + + /** + * The application drives the exchange through {@link Peer#negotiator()}, + * as WHIP, WHEP and most SFUs require. + */ + MANUAL + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/NegotiationException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/NegotiationException.java new file mode 100644 index 00000000..0eb55ea9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/NegotiationException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A session description or an ICE candidate was rejected, or an offer collision between two impolite peers could not be resolved. + */ +public class NegotiationException extends WebRtcException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public NegotiationException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public NegotiationException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiator.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiator.java new file mode 100644 index 00000000..ec97e162 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Negotiator.java @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.concurrent.CompletableFuture; + +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCSessionDescription; + +/** + * The offer and answer exchange by hand, for peers built with {@link + * Negotiation#MANUAL}. Each step creates and applies a description in one go, + * so nothing can change in between. Steps run in the order they were called. + */ +public interface Negotiator { + + /** + * Creates the local offer and applies it. + * + * @return a future with the offer, at once. + */ + CompletableFuture createOffer(); + + /** + * Creates the local offer and applies it. + * + * @param gathering {@link Gathering#COMPLETE} to wait for all candidates. + * + * @return a future with the offer. + */ + CompletableFuture createOffer(Gathering gathering); + + /** + * Applies the remote offer, then creates the answer and applies it. + * + * @param offer the remote offer. + * + * @return a future with the answer, at once. + */ + CompletableFuture answer(RTCSessionDescription offer); + + /** + * Applies the remote offer, then creates the answer and applies it. + * + * @param offer the remote offer. + * @param gathering {@link Gathering#COMPLETE} to wait for all candidates. + * + * @return a future with the answer. + */ + CompletableFuture answer(RTCSessionDescription offer, Gathering gathering); + + /** + * Applies the remote answer to the local offer. + * + * @param answer the remote answer. + * + * @return a future that completes once the answer is applied. + */ + CompletableFuture applyAnswer(RTCSessionDescription answer); + + /** + * Adds a remote candidate. One that arrives before the remote description + * is held until the description is applied. + * + * @param candidate the remote candidate. + * + * @return a future that completes once the candidate is added. + */ + CompletableFuture addCandidate(RTCIceCandidate candidate); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PcmFormat.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PcmFormat.java new file mode 100644 index 00000000..c352d474 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PcmFormat.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * The format of 16-bit PCM audio: sample rate and channel count. + */ +public final class PcmFormat { + + private final int sampleRate; + + private final int channels; + + + private PcmFormat(int sampleRate, int channels) { + this.sampleRate = sampleRate; + this.channels = channels; + } + + /** + * @param sampleRate the sample rate in Hz, a multiple of 100 so that 10 ms + * hold a whole number of samples. + * @param channels the channel count, 1 to 8. + * + * @return the format. + */ + public static PcmFormat of(int sampleRate, int channels) { + if (sampleRate < 8000 || sampleRate > 384000 || sampleRate % 100 != 0) { + throw new IllegalArgumentException("The sample rate must be a multiple of 100 between 8000 and 384000 Hz, got " + sampleRate); + } + if (channels < 1 || channels > 8) { + throw new IllegalArgumentException("The channel count must be between 1 and 8, got " + channels); + } + + return new PcmFormat(sampleRate, channels); + } + + /** + * @return the sample rate in Hz. + */ + public int sampleRate() { + return sampleRate; + } + + /** + * @return the channel count. + */ + public int channels() { + return channels; + } + + int framesPer10Ms() { + return sampleRate / 100; + } + + int bytesPer10Ms() { + return framesPer10Ms() * channels * 2; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PcmFormat)) { + return false; + } + + PcmFormat other = (PcmFormat) obj; + + return sampleRate == other.sampleRate && channels == other.channels; + } + + @Override + public int hashCode() { + return sampleRate * 31 + channels; + } + + @Override + public String toString() { + return "PcmFormat[" + sampleRate + " Hz, " + channels + " channels]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Peer.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Peer.java new file mode 100644 index 00000000..0c13463b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Peer.java @@ -0,0 +1,215 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.function.Function; + +import dev.onvoid.webrtc.RTCPeerConnection; + +/** + * A connection to one remote peer, made by {@link WebRtc#createPeer}. + *

+ * Every method may be called from any thread, including from a handler of + * this peer, and none of them blocks. Methods that change the connection + * return at once and take effect in the order they were called. + *

+ * The handlers and the callbacks of the futures of one peer run on its event + * thread, one at a time and in the order of the events. They may block, which + * delays only the events of this peer, and they may call any method, including + * {@link #close()}. They must not wait for a future of this API: chain the + * next step instead. + */ +public interface Peer extends AutoCloseable { + + /** + * @return the name given in {@link PeerOptions.Builder#name(String)}, or + * one made up. + */ + String name(); + + /** + * @return the last state the connection reported. + */ + PeerState state(); + + /** + * Hands over a message the remote peer sent. Messages must be handed over + * in the order they were sent. A message that cannot be applied is + * reported to {@link #onError}. + * + * @param message the message the remote peer sent. + */ + void signal(SignalMessage message); + + /** + * Sends a track to the remote peer. With {@link Negotiation#AUTOMATIC} the + * peer renegotiates by itself. The same track may be sent on many peers. + * + * @param track a track of the same runtime. + * @param streamIds the streams the remote peer sees the track in. Tracks + * added without any share one stream, so that a browser + * plays their audio and video together. + * + * @return the handle of the new sender, at once. + */ + Sender addTrack(LocalTrack track, String... streamIds); + + /** + * @return the senders that were added and not removed. + */ + List senders(); + + /** + * Calls the handler for every track the remote peer sends. Tracks that + * already arrived are handed to the new handler first. + * + * @param handler called on the event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onTrack(Consumer handler); + + /** + * @return the tracks the remote peer sends now. + */ + List remoteTracks(); + + /** + * Creates a reliable, ordered data channel. With {@link + * Negotiation#AUTOMATIC} the first channel makes the peer renegotiate. + * + * @param label the label the remote peer sees. + * + * @return the channel, at once. Messages sent before it opens are held. + */ + DataChannel createChannel(String label); + + /** + * Creates a data channel. + * + * @param label the label the remote peer sees. + * @param options how the channel delivers. + * + * @return the channel, at once. Messages sent before it opens are held. + */ + DataChannel createChannel(String label, ChannelOptions options); + + /** + * Calls the handler for every channel the remote peer opens. Channels that + * already arrived are handed to the new handler first. + * + * @param handler called on the event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onChannel(Consumer handler); + + /** + * Calls the handler with the current state, and then with every change. + * + * @param handler called on the event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onStateChange(Consumer handler); + + /** + * Calls the handler with failures nobody else hears of, such as a remote + * description that was rejected during automatic negotiation. Without a + * handler they go to the runtime's error handler. + * + * @param handler called on the event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onError(Consumer handler); + + /** + * @return a future that completes when the peer is connected, or fails if + * the connection fails or closes first. + */ + CompletableFuture whenConnected(); + + /** + * @return a future that completes when the peer is closed. + */ + CompletableFuture whenClosed(); + + /** + * @return the offer and answer exchange by hand. + * + * @throws IllegalStateException if the peer negotiates automatically. + */ + Negotiator negotiator(); + + /** + * @return a future with the current statistics of the connection. + */ + CompletableFuture stats(); + + /** + * Calls the handler with the statistics of the connection, at a fixed + * interval. Bitrates are filled from the second call on. + * + * @param interval the time between two calls. + * @param handler called on the event thread. + * + * @return the subscription that stops the calls. + */ + Subscription onStats(Duration interval, Consumer handler); + + /** + * Gathers new ICE candidates and renegotiates, to recover a failed + * connection. + */ + void restartIce(); + + /** + * Runs a function with the underlying connection, for what this API does + * not cover. It runs after the operations called before it, and before + * those called after it. Objects the function obtains belong to it, as in + * the low-level API, and must not be used after it returns. + * + * @param action the function, which must not block. + * @param the result type. + * + * @return a future with what the function returned. + */ + CompletableFuture withNative(Function action); + + /** + * Closes the peer. + * + * @return a future that completes when the peer is closed. + */ + CompletableFuture closeAsync(); + + /** + * Closes the peer, its channels and its senders, and ends its remote + * tracks. Local tracks stay open. Never blocks, may be called more than + * once and from a handler. Afterwards the only event is the {@link + * PeerState#CLOSED} state, and pending operations fail with a {@link + * PeerClosedException}. + */ + @Override + void close(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerClosedException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerClosedException.java new file mode 100644 index 00000000..7da63a6c --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerClosedException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * An operation was pending when its peer, channel or runtime closed, or was issued after that. + */ +public class PeerClosedException extends WebRtcException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public PeerClosedException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public PeerClosedException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java new file mode 100644 index 00000000..9385da45 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java @@ -0,0 +1,1283 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +import dev.onvoid.webrtc.AddIceCandidateObserver; +import dev.onvoid.webrtc.CreateSessionDescriptionObserver; +import dev.onvoid.webrtc.PeerConnectionObserver; +import dev.onvoid.webrtc.RTCAnswerOptions; +import dev.onvoid.webrtc.RTCDataChannel; +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCIceGatheringState; +import dev.onvoid.webrtc.RTCOfferOptions; +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.RTCPeerConnectionState; +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.RTCRtpTransceiver; +import dev.onvoid.webrtc.RTCRtpTransceiverDirection; +import dev.onvoid.webrtc.RTCRtpTransceiverInit; +import dev.onvoid.webrtc.RTCSdpType; +import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.RTCSignalingState; +import dev.onvoid.webrtc.SetSessionDescriptionObserver; +import dev.onvoid.webrtc.media.MediaStream; +import dev.onvoid.webrtc.media.MediaStreamTrack; +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * The {@link Peer} of a {@link WebRtc} runtime. + *

+ * The state of a peer lives on two serial executors. The control executor + * owns the connection and everything negotiation needs; WebRTC's callbacks + * only post to it, and it is the only place that calls the connection. The + * event executor runs application code, and owns what the handlers have been + * told so far. Neither ever waits for the other, and nothing on a WebRTC + * thread waits for either, so no handler can deadlock the connection. + *

+ * Changes that the standard runs on its operations chain go through {@link + * #enqueue}, which starts one only once the one before finished. + */ +final class PeerImpl implements Peer { + + /** How long an impolite peer waits for an answer after it ignored an offer. */ + private static final long COLLISION_TIMEOUT_MS = 5000; + + final WebRtc rtc; + + final String name; + + final SerialExecutor control; + + final SerialExecutor events; + + private final PeerOptions options; + + private final Signaling signaling; + + /** The stream the tracks added without stream ids share. */ + private final String defaultStreamId = UUID.randomUUID().toString(); + + private final AtomicBoolean closing = new AtomicBoolean(); + + private final CompletableFuture closedInternal = new CompletableFuture<>(); + + private final CopyOnWriteArrayList senders = new CopyOnWriteArrayList<>(); + + private final CopyOnWriteArrayList remoteTracks = new CopyOnWriteArrayList<>(); + + private volatile PeerState state = PeerState.NEW; + + // Confined to the control executor. + + private RTCPeerConnection connection; + + private boolean closed; + + private final ArrayDeque operations = new ArrayDeque<>(); + + private boolean operationRunning; + + private CompletableFuture runningOperation; + + private boolean negotiationQueued; + + private boolean ignoreOffer; + + private int remoteAnswers; + + private ScheduledFuture collisionCheck; + + private final List heldCandidates = new ArrayList<>(); + + private final List> gatheringWaiters = new ArrayList<>(); + + private final Map remoteTrackById = new HashMap<>(); + + private final List channels = new ArrayList<>(); + + private final StatsReader statsReader = new StatsReader(); + + // Confined to the event executor. + + private PeerState deliveredState = PeerState.NEW; + + private final List deliveredTracks = new ArrayList<>(); + + private final List deliveredChannels = new ArrayList<>(); + + private final Handlers stateHandlers; + + private final Handlers trackHandlers; + + private final Handlers channelHandlers; + + private final Handlers errorHandlers; + + private final ClientFuture connected = new ClientFuture<>(); + + private final ClientFuture closedFuture = new ClientFuture<>(); + + private final NegotiatorImpl negotiator = new NegotiatorImpl(); + + + PeerImpl(WebRtc rtc, PeerOptions options, String name) { + this.rtc = rtc; + this.options = options; + this.name = name; + this.signaling = options.signaling; + this.control = new SerialExecutor(rtc.controlPool, rtc::reportError, false, null); + this.events = new SerialExecutor(rtc.eventExecutor, rtc::reportError, true, + rtc.ownsEventExecutor ? "webrtc-client-events-" + name : null); + this.stateHandlers = new Handlers<>(rtc::reportError); + this.trackHandlers = new Handlers<>(rtc::reportError); + this.channelHandlers = new Handlers<>(rtc::reportError); + this.errorHandlers = new Handlers<>(rtc::reportError); + } + + /** + * Creates the connection on the control executor and waits for it, so that + * a configuration WebRTC refuses fails the caller. + */ + void open() { + CompletableFuture opened = new CompletableFuture<>(); + + control.execute(() -> { + try { + connection = rtc.factory.createPeerConnection(options.toConfiguration(), new Observer()); + + if (connection == null) { + throw new ConfigurationException("WebRTC refused the configuration of " + name + + ". Check the ICE server URLs and the values set through PeerOptions.Builder.configure"); + } + + for (MediaKind kind : options.receive) { + RTCRtpTransceiverInit init = new RTCRtpTransceiverInit(); + init.direction = RTCRtpTransceiverDirection.RECV_ONLY; + + RTCRtpTransceiver transceiver = connection.addTransceiver( + kind == MediaKind.AUDIO ? MediaType.AUDIO : MediaType.VIDEO, init); + transceiver.dispose(); + } + + opened.complete(null); + } + catch (Throwable e) { + closed = true; + + if (connection != null) { + connection.close(); + connection = null; + } + + opened.completeExceptionally(e); + } + }); + + try { + opened.get(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new WebRtcException("Interrupted while creating " + name, e); + } + catch (ExecutionException e) { + Throwable cause = e.getCause(); + + if (cause instanceof WebRtcException) { + throw (WebRtcException) cause; + } + + throw new ConfigurationException("Creating " + name + " failed: " + cause.getMessage(), cause); + } + + if (signaling != null) { + signaling.attach(this::signal); + } + } + + @Override + public String name() { + return name; + } + + @Override + public PeerState state() { + return state; + } + + @Override + public void signal(SignalMessage message) { + requireNonNull(message, "message"); + + control.execute(() -> handleSignal(message)); + } + + @Override + public Sender addTrack(LocalTrack track, String... streamIds) { + LocalTrackImpl local = checkTrack(track); + + List ids = new ArrayList<>(); + + for (String id : streamIds) { + ids.add(requireNonNull(id, "streamId")); + } + if (ids.isEmpty()) { + ids.add(defaultStreamId); + } + + SenderImpl sender = new SenderImpl(this, local); + senders.add(sender); + + control.execute(() -> { + if (closed) { + senders.remove(sender); + sender.failAttach(closedError()); + return; + } + + try { + sender.attach(connection, ids); + } + catch (Throwable e) { + senders.remove(sender); + sender.failAttach(e); + deliverError(asWebRtcException(e, "Adding the track failed")); + } + }); + + return sender; + } + + @Override + public List senders() { + return Collections.unmodifiableList(new ArrayList(senders)); + } + + @Override + public Subscription onTrack(Consumer handler) { + requireNonNull(handler, "handler"); + + return subscribe(() -> { + for (RemoteTrack track : new ArrayList<>(deliveredTracks)) { + call(handler, track); + } + return trackHandlers.add(handler); + }); + } + + @Override + public List remoteTracks() { + return Collections.unmodifiableList(new ArrayList(remoteTracks)); + } + + @Override + public DataChannel createChannel(String label) { + return createChannel(label, ChannelOptions.builder().build()); + } + + @Override + public DataChannel createChannel(String label, ChannelOptions options) { + requireNonNull(label, "label"); + requireNonNull(options, "options"); + + DataChannelImpl channel = new DataChannelImpl(this, label); + + control.execute(() -> { + if (closed) { + channel.closeFromPeer(new PeerClosedException("The peer " + name + " is closed")); + return; + } + + try { + RTCDataChannel nativeChannel = connection.createDataChannel(label, options.toInit()); + + if (nativeChannel == null) { + throw new ConfigurationException("WebRTC refused the data channel " + label); + } + + channels.add(channel); + channel.bind(nativeChannel); + } + catch (Throwable e) { + WebRtcException error = asWebRtcException(e, "Creating the data channel " + label + " failed"); + channel.closeFromPeer(error); + deliverError(error); + } + }); + + return channel; + } + + @Override + public Subscription onChannel(Consumer handler) { + requireNonNull(handler, "handler"); + + return subscribe(() -> { + for (DataChannel channel : new ArrayList<>(deliveredChannels)) { + call(handler, channel); + } + return channelHandlers.add(handler); + }); + } + + @Override + public Subscription onStateChange(Consumer handler) { + requireNonNull(handler, "handler"); + + return subscribe(() -> { + call(handler, deliveredState); + return stateHandlers.add(handler); + }); + } + + @Override + public Subscription onError(Consumer handler) { + requireNonNull(handler, "handler"); + + return subscribe(() -> errorHandlers.add(handler)); + } + + @Override + public CompletableFuture whenConnected() { + return connected; + } + + @Override + public CompletableFuture whenClosed() { + return closedFuture; + } + + @Override + public Negotiator negotiator() { + if (options.negotiation != Negotiation.MANUAL) { + throw new IllegalStateException("The peer " + name + " negotiates automatically. " + + "Build it with Negotiation.MANUAL to drive the exchange by hand"); + } + + return negotiator; + } + + @Override + public CompletableFuture stats() { + CompletableFuture result = new CompletableFuture<>(); + + control.execute(() -> { + if (closed) { + result.completeExceptionally(closedError()); + return; + } + + try { + connection.getStats(report -> control.execute(() -> { + try { + result.complete(statsReader.read(report)); + } + catch (Throwable e) { + result.completeExceptionally(e); + } + })); + } + catch (Throwable e) { + result.completeExceptionally(asWebRtcException(e, "Reading the statistics failed")); + } + }); + + return toPublic(result); + } + + @Override + public Subscription onStats(Duration interval, Consumer handler) { + requireNonNull(interval, "interval"); + requireNonNull(handler, "handler"); + + long millis = interval.toMillis(); + + if (millis <= 0) { + throw new IllegalArgumentException("The interval must be positive, got " + interval); + } + + AtomicBoolean active = new AtomicBoolean(true); + AtomicReference> task = new AtomicReference<>(); + + Runnable sample = () -> { + if (!active.get()) { + return; + } + if (closing.get()) { + active.set(false); + + ScheduledFuture scheduled = task.get(); + + if (scheduled != null) { + scheduled.cancel(false); + } + return; + } + + stats().whenComplete((stats, error) -> { + if (!active.get()) { + return; + } + if (error == null) { + handler.accept(stats); + } + else if (!(unwrap(error) instanceof PeerClosedException)) { + deliverError(asWebRtcException(error, "Reading the statistics failed")); + } + }); + }; + + task.set(rtc.timer.scheduleAtFixedRate(sample, 0, millis, TimeUnit.MILLISECONDS)); + + return () -> { + active.set(false); + task.get().cancel(false); + }; + } + + @Override + public void restartIce() { + control.execute(() -> { + if (!closed) { + connection.restartIce(); + } + }); + } + + @Override + public CompletableFuture withNative(Function action) { + requireNonNull(action, "action"); + + CompletableFuture result = new CompletableFuture<>(); + + control.execute(() -> enqueue(() -> CompletableFuture.completedFuture(action.apply(connection))) + .whenComplete((value, error) -> complete(result, value, error))); + + return toPublic(result); + } + + @Override + public CompletableFuture closeAsync() { + closeInternal(); + + return closedFuture; + } + + @Override + public void close() { + closeInternal(); + } + + @Override + public String toString() { + return "Peer[" + name + ", " + state + "]"; + } + + /** + * Starts closing, once. + * + * @return a future that completes on the control executor once the + * connection is closed, before the handlers hear of it. + */ + CompletableFuture closeInternal() { + if (closing.compareAndSet(false, true)) { + control.execute(this::teardown); + } + + return closedInternal; + } + + boolean isClosing() { + return closing.get(); + } + + /** + * Hands a future out to the application: it completes on the event + * executor, so its callbacks run there like the handlers. + */ + CompletableFuture toPublic(CompletableFuture internal) { + ClientFuture result = new ClientFuture<>(); + + internal.whenComplete((value, error) -> events.execute(() -> complete(result, value, error))); + + return result; + } + + void deliverError(WebRtcException error) { + events.execute(() -> { + if (errorHandlers.isEmpty()) { + rtc.reportError(error); + } + else { + errorHandlers.accept(error); + } + }); + } + + /** + * Registers a handler on the event executor, where it sees exactly the + * events the other handlers saw. + */ + Subscription subscribe(Supplier register) { + DeferredSubscription subscription = new DeferredSubscription(); + + events.execute(() -> subscription.set(register.get())); + + return subscription; + } + + void call(Consumer handler, T value) { + try { + handler.accept(value); + } + catch (Throwable e) { + rtc.reportError(e); + } + } + + PeerClosedException closedError() { + return new PeerClosedException("The peer " + name + " is closed"); + } + + void forgetSender(SenderImpl sender) { + senders.remove(sender); + } + + void forgetChannel(DataChannelImpl channel) { + channels.remove(channel); + } + + boolean isClosed() { + return closed; + } + + static Throwable unwrap(Throwable error) { + while ((error instanceof CompletionException || error instanceof ExecutionException) + && error.getCause() != null) { + error = error.getCause(); + } + + return error; + } + + static WebRtcException asWebRtcException(Throwable error, String message) { + error = unwrap(error); + + if (error instanceof WebRtcException) { + return (WebRtcException) error; + } + + return new WebRtcException(message + ": " + error, error); + } + + static void complete(CompletableFuture future, T value, Throwable error) { + if (error != null) { + future.completeExceptionally(unwrap(error)); + } + else { + future.complete(value); + } + } + + private LocalTrackImpl checkTrack(LocalTrack track) { + requireNonNull(track, "track"); + + if (!(track instanceof LocalTrackImpl) || ((LocalTrackImpl) track).rtc != rtc) { + throw new IllegalArgumentException("The track was not created by the runtime of " + name); + } + + return (LocalTrackImpl) track; + } + + // Operations, on the control executor. + + /** + * Runs an operation once the ones queued before it finished. It fails + * with a {@link PeerClosedException} when the peer closed first, and with + * a {@link TimeoutException} when WebRTC never finishes it. + * + * @return a future that completes on the control executor. + */ + private CompletableFuture enqueue(Supplier> operation) { + CompletableFuture result = new CompletableFuture<>(); + + operations.add(() -> { + CompletableFuture running; + + if (closed) { + running = new CompletableFuture<>(); + running.completeExceptionally(closedError()); + } + else { + try { + running = operation.get(); + } + catch (Throwable e) { + running = new CompletableFuture<>(); + running.completeExceptionally(e); + } + } + + CompletableFuture current = running; + runningOperation = current; + + ScheduledFuture timeout = rtc.timer.schedule(() -> control.execute(() -> + current.completeExceptionally(new TimeoutException("WebRTC did not finish an operation of " + + name + " within " + rtc.operationTimeout))), + rtc.operationTimeout.toMillis(), TimeUnit.MILLISECONDS); + + current.whenComplete((value, error) -> control.execute(() -> { + timeout.cancel(false); + + runningOperation = null; + operationRunning = false; + + complete(result, value, error); + runNextOperation(); + })); + }); + + runNextOperation(); + + return result; + } + + private void runNextOperation() { + if (operationRunning) { + return; + } + + Runnable next = operations.poll(); + + if (next != null) { + operationRunning = true; + next.run(); + } + } + + private CompletableFuture setLocal() { + CompletableFuture future = new CompletableFuture<>(); + connection.setLocalDescription(descriptionObserver(future, "Applying the local description failed")); + return future; + } + + private CompletableFuture setLocal(RTCSessionDescription description) { + CompletableFuture future = new CompletableFuture<>(); + connection.setLocalDescription(description, descriptionObserver(future, "Applying the local description failed")); + return future; + } + + private CompletableFuture setRemote(RTCSessionDescription description) { + CompletableFuture future = new CompletableFuture<>(); + connection.setRemoteDescription(description, descriptionObserver(future, + "Applying the remote " + description.sdpType.name().toLowerCase() + " failed")); + return future; + } + + private CompletableFuture createOffer() { + CompletableFuture future = new CompletableFuture<>(); + connection.createOffer(new RTCOfferOptions(), createObserver(future, "Creating the offer failed")); + return future; + } + + private CompletableFuture createAnswer() { + CompletableFuture future = new CompletableFuture<>(); + connection.createAnswer(new RTCAnswerOptions(), createObserver(future, "Creating the answer failed")); + return future; + } + + private SetSessionDescriptionObserver descriptionObserver(CompletableFuture future, String message) { + return new SetSessionDescriptionObserver() { + + @Override + public void onSuccess() { + control.execute(() -> future.complete(null)); + } + + @Override + public void onFailure(String error) { + control.execute(() -> future.completeExceptionally(new NegotiationException(message + ": " + error))); + } + }; + } + + private CreateSessionDescriptionObserver createObserver( + CompletableFuture future, String message) { + return new CreateSessionDescriptionObserver() { + + @Override + public void onSuccess(RTCSessionDescription description) { + control.execute(() -> future.complete(description)); + } + + @Override + public void onFailure(String error) { + control.execute(() -> future.completeExceptionally(new NegotiationException(message + ": " + error))); + } + }; + } + + /** + * Adds a remote candidate, or holds it while there is no remote + * description to add it to. + */ + private CompletableFuture addRemoteCandidate(RTCIceCandidate candidate) { + CompletableFuture future = new CompletableFuture<>(); + + if (connection.getRemoteDescription() == null) { + heldCandidates.add(new HeldCandidate(candidate, future)); + } + else { + addCandidate(candidate, future); + } + + return future; + } + + private void addCandidate(RTCIceCandidate candidate, CompletableFuture future) { + connection.addIceCandidate(candidate, new AddIceCandidateObserver() { + + @Override + public void onSuccess() { + control.execute(() -> future.complete(null)); + } + + @Override + public void onFailure(String error) { + control.execute(() -> { + if (ignoreOffer) { + // A candidate of an offer this peer ignored. + future.complete(null); + } + else { + future.completeExceptionally(new NegotiationException("Adding a remote candidate failed: " + error)); + } + }); + } + }); + } + + private void releaseHeldCandidates() { + List held = new ArrayList<>(heldCandidates); + heldCandidates.clear(); + + for (HeldCandidate candidate : held) { + addCandidate(candidate.candidate, candidate.future); + } + } + + private CompletableFuture waitForGathering() { + if (connection.getIceGatheringState() == RTCIceGatheringState.COMPLETE) { + return CompletableFuture.completedFuture(null); + } + + CompletableFuture future = new CompletableFuture<>(); + gatheringWaiters.add(future); + + return future; + } + + private void sendLocalDescription() { + send(SignalMessage.of(connection.getLocalDescription())); + } + + private void send(SignalMessage message) { + if (signaling == null) { + return; + } + + events.execute(() -> { + try { + signaling.send(message); + } + catch (Throwable e) { + deliverError(new WebRtcException("The signaling failed to send " + message.type(), e)); + } + }); + } + + private void reportFailure(Object value, Throwable error) { + if (error != null && !(unwrap(error) instanceof PeerClosedException)) { + deliverError(asWebRtcException(error, "Negotiation of " + name + " failed")); + } + } + + // Automatic negotiation: perfect negotiation, on the control executor. + + private void onNegotiationNeeded() { + if (closed || options.negotiation != Negotiation.AUTOMATIC || negotiationQueued) { + return; + } + + negotiationQueued = true; + + enqueue(() -> { + negotiationQueued = false; + + if (connection.getSignalingState() != RTCSignalingState.STABLE) { + // WebRTC asks again once the exchange under way is done. + return CompletableFuture.completedFuture(null); + } + + return setLocal().thenRun(this::sendLocalDescription); + }).whenComplete(this::reportFailure); + } + + private void handleSignal(SignalMessage message) { + if (closed) { + return; + } + + switch (message.type()) { + case OFFER: + case ANSWER: + if (options.negotiation == Negotiation.MANUAL) { + deliverError(new NegotiationException("The peer " + name + " negotiates by hand. " + + "Apply descriptions through Peer.negotiator(), not Peer.signal()")); + return; + } + + enqueue(() -> applyRemoteDescription(message.description())) + .whenComplete(this::reportFailure); + break; + + case CANDIDATE: + enqueue(() -> ignoreOffer + ? CompletableFuture.completedFuture(null) + : addRemoteCandidate(message.candidate())) + .whenComplete(this::reportFailure); + break; + + default: + // WebRTC needs no end of candidates. + break; + } + } + + private CompletableFuture applyRemoteDescription(RTCSessionDescription description) { + boolean offer = description.sdpType == RTCSdpType.OFFER; + boolean collision = offer && connection.getSignalingState() != RTCSignalingState.STABLE; + + ignoreOffer = !options.polite && collision; + + if (ignoreOffer) { + // The polite peer gives way and answers this peer's offer instead. + watchCollision(); + return CompletableFuture.completedFuture(null); + } + + // A polite peer's own pending offer is rolled back implicitly. + return setRemote(description).thenCompose(ignored -> { + releaseHeldCandidates(); + + if (!offer) { + remoteAnswers++; + return CompletableFuture.completedFuture(null); + } + + return setLocal().thenRun(this::sendLocalDescription); + }); + } + + /** + * Tells the application when the remote peer never answered the offer this + * peer kept, which happens when both peers are impolite. + */ + private void watchCollision() { + if (collisionCheck != null) { + return; + } + + int answers = remoteAnswers; + + collisionCheck = rtc.timer.schedule(() -> control.execute(() -> { + collisionCheck = null; + + if (!closed && remoteAnswers == answers + && connection.getSignalingState() == RTCSignalingState.HAVE_LOCAL_OFFER) { + deliverError(new NegotiationException("Both peers sent an offer at once, and both are impolite, " + + "so neither gives way. Build exactly one of them with PeerOptions.Builder.polite(true)")); + } + }), COLLISION_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } + + // What WebRTC reports, on the control executor. + + private void connectionChanged(RTCPeerConnectionState nativeState) { + if (closed) { + return; + } + + PeerState newState = PeerState.of(nativeState); + state = newState; + + events.execute(() -> deliverState(newState)); + } + + private void gatheringChanged(RTCIceGatheringState gatheringState) { + if (gatheringState == RTCIceGatheringState.COMPLETE) { + List> waiters = new ArrayList<>(gatheringWaiters); + gatheringWaiters.clear(); + + waiters.forEach(waiter -> waiter.complete(null)); + } + } + + private void trackAdded(RTCRtpReceiver receiver, MediaStream[] streams) { + List streamIds = new ArrayList<>(); + + for (MediaStream stream : streams) { + streamIds.add(stream.id()); + stream.dispose(); + } + + if (closed) { + receiver.dispose(); + return; + } + + MediaStreamTrack track = receiver.getTrack(); + String id = track.getId(); + + if (remoteTrackById.containsKey(id)) { + // Renegotiation reported a track that is already known. + track.dispose(); + receiver.dispose(); + return; + } + + RemoteTrackImpl remote = track instanceof VideoTrack + ? new RemoteVideoImpl(this, receiver, (VideoTrack) track, id, streamIds) + : new RemoteAudioImpl(this, receiver, (AudioTrack) track, id, streamIds); + + remoteTrackById.put(id, remote); + remoteTracks.add(remote); + + events.execute(() -> { + deliveredTracks.add(remote); + trackHandlers.accept(remote); + }); + } + + private void trackRemoved(RTCRtpReceiver receiver) { + MediaStreamTrack track = receiver.getTrack(); + String id = track.getId(); + + track.dispose(); + receiver.dispose(); + + RemoteTrackImpl remote = remoteTrackById.remove(id); + + if (remote == null || closed) { + return; + } + + remoteTracks.remove(remote); + remote.end(); + + events.execute(() -> { + deliveredTracks.remove(remote); + remote.fireEnded(); + }); + } + + private void channelOpened(DataChannelImpl channel) { + if (closed) { + channel.closeFromPeer(closedError()); + return; + } + + channels.add(channel); + + events.execute(() -> { + deliveredChannels.add(channel); + channelHandlers.accept(channel); + }); + } + + void channelClosed(DataChannelImpl channel) { + channels.remove(channel); + + events.execute(() -> deliveredChannels.remove(channel)); + } + + private void teardown() { + closed = true; + + PeerClosedException error = closedError(); + + if (runningOperation != null) { + runningOperation.completeExceptionally(error); + } + if (collisionCheck != null) { + collisionCheck.cancel(false); + collisionCheck = null; + } + + for (CompletableFuture waiter : gatheringWaiters) { + waiter.completeExceptionally(error); + } + gatheringWaiters.clear(); + + for (HeldCandidate candidate : heldCandidates) { + candidate.future.completeExceptionally(error); + } + heldCandidates.clear(); + + for (DataChannelImpl channel : new ArrayList<>(channels)) { + channel.closeFromPeer(error); + } + channels.clear(); + + for (SenderImpl sender : senders) { + sender.closeFromPeer(); + } + senders.clear(); + + List ended = new ArrayList<>(remoteTracks); + + for (RemoteTrackImpl remote : ended) { + remote.end(); + } + remoteTracks.clear(); + remoteTrackById.clear(); + + try { + if (connection != null) { + connection.close(); + } + } + catch (Throwable e) { + rtc.reportError(e); + } + + state = PeerState.CLOSED; + rtc.removePeer(this); + closedInternal.complete(null); + + events.execute(() -> { + for (RemoteTrackImpl remote : ended) { + remote.fireEnded(); + } + deliveredTracks.clear(); + deliveredChannels.clear(); + + deliverState(PeerState.CLOSED); + + connected.completeExceptionally(error); + closedFuture.complete(null); + + trackHandlers.clear(); + channelHandlers.clear(); + stateHandlers.clear(); + errorHandlers.clear(); + }); + } + + // On the event executor. + + private void deliverState(PeerState newState) { + if (newState == deliveredState || deliveredState == PeerState.CLOSED) { + return; + } + + deliveredState = newState; + stateHandlers.accept(newState); + + if (newState == PeerState.CONNECTED) { + connected.complete(this); + } + else if (newState == PeerState.FAILED) { + connected.completeExceptionally(new WebRtcException("The connection of " + name + " failed")); + } + } + + + + /** + * Hears what WebRTC reports, on its signaling thread, and only posts it to + * the control executor. + */ + private final class Observer implements PeerConnectionObserver { + + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + control.execute(() -> { + if (!closed) { + send(SignalMessage.of(candidate)); + } + }); + } + + @Override + public void onConnectionChange(RTCPeerConnectionState state) { + control.execute(() -> connectionChanged(state)); + } + + @Override + public void onIceGatheringChange(RTCIceGatheringState state) { + control.execute(() -> gatheringChanged(state)); + } + + @Override + public void onRenegotiationNeeded() { + control.execute(PeerImpl.this::onNegotiationNeeded); + } + + @Override + public void onTrack(RTCRtpTransceiver transceiver) { + // The tracks are taken from onAddTrack, which names their streams. + control.execute(transceiver::dispose); + } + + @Override + public void onAddTrack(RTCRtpReceiver receiver, MediaStream[] mediaStreams) { + control.execute(() -> trackAdded(receiver, mediaStreams)); + } + + @Override + public void onRemoveTrack(RTCRtpReceiver receiver) { + control.execute(() -> trackRemoved(receiver)); + } + + @Override + public void onDataChannel(RTCDataChannel dataChannel) { + // Observed at once, so that no message is missed; the observer + // itself only posts as well. + DataChannelImpl channel = new DataChannelImpl(PeerImpl.this, dataChannel); + + control.execute(() -> channelOpened(channel)); + } + } + + + + private final class NegotiatorImpl implements Negotiator { + + @Override + public CompletableFuture createOffer() { + return createOffer(Gathering.TRICKLE); + } + + @Override + public CompletableFuture createOffer(Gathering gathering) { + requireNonNull(gathering, "gathering"); + + return run(() -> PeerImpl.this.createOffer() + .thenCompose(PeerImpl.this::setLocal) + .thenCompose(ignored -> gathered(gathering))); + } + + @Override + public CompletableFuture answer(RTCSessionDescription offer) { + return answer(offer, Gathering.TRICKLE); + } + + @Override + public CompletableFuture answer(RTCSessionDescription offer, Gathering gathering) { + requireNonNull(offer, "offer"); + requireNonNull(gathering, "gathering"); + + if (offer.sdpType != RTCSdpType.OFFER) { + throw new IllegalArgumentException("Expected an offer, got " + offer.sdpType); + } + + return run(() -> setRemote(offer) + .thenCompose(ignored -> { + releaseHeldCandidates(); + return PeerImpl.this.createAnswer(); + }) + .thenCompose(PeerImpl.this::setLocal) + .thenCompose(ignored -> gathered(gathering))); + } + + @Override + public CompletableFuture applyAnswer(RTCSessionDescription answer) { + requireNonNull(answer, "answer"); + + if (answer.sdpType != RTCSdpType.ANSWER && answer.sdpType != RTCSdpType.PR_ANSWER) { + throw new IllegalArgumentException("Expected an answer, got " + answer.sdpType); + } + + return run(() -> setRemote(answer).thenRun(PeerImpl.this::releaseHeldCandidates)); + } + + @Override + public CompletableFuture addCandidate(RTCIceCandidate candidate) { + requireNonNull(candidate, "candidate"); + + return run(() -> addRemoteCandidate(candidate)); + } + + private CompletableFuture gathered(Gathering gathering) { + CompletableFuture ready = gathering == Gathering.COMPLETE + ? waitForGathering() + : CompletableFuture.completedFuture(null); + + return ready.thenApply(ignored -> connection.getLocalDescription()); + } + + private CompletableFuture run(Supplier> operation) { + CompletableFuture result = new CompletableFuture<>(); + + control.execute(() -> enqueue(operation) + .whenComplete((value, error) -> complete(result, value, error))); + + return toPublic(result); + } + } + + + + private static final class HeldCandidate { + + final RTCIceCandidate candidate; + + final CompletableFuture future; + + + HeldCandidate(RTCIceCandidate candidate, CompletableFuture future) { + this.candidate = candidate; + this.future = future; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java new file mode 100644 index 00000000..5295479a --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java @@ -0,0 +1,283 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.RTCConfiguration; +import dev.onvoid.webrtc.RTCIceServer; +import dev.onvoid.webrtc.RTCIceTransportPolicy; + +/** + * How {@link WebRtc#createPeer} sets up a peer. + * 

{@code
+ * PeerOptions.builder()
+ *         .iceServer("stun:stun.l.google.com:19302")
+ *         .polite(true)
+ *         .signaling(message -> socket.send(message.toJson()))
+ *         .build();
+ * }
+ */ +public final class PeerOptions { + + final String name; + + final List iceServers; + + final boolean relayOnly; + + final Signaling signaling; + + final boolean polite; + + final Negotiation negotiation; + + final Set receive; + + final Consumer customizer; + + + private PeerOptions(Builder builder) { + this.name = builder.name; + this.iceServers = Collections.unmodifiableList(new ArrayList<>(builder.iceServers)); + this.relayOnly = builder.relayOnly; + this.signaling = builder.signaling; + this.polite = builder.polite; + this.negotiation = builder.negotiation; + this.receive = Collections.unmodifiableSet(builder.receive.isEmpty() + ? EnumSet.noneOf(MediaKind.class) : EnumSet.copyOf(builder.receive)); + this.customizer = builder.customizer; + } + + /** + * @return a builder for a peer without ICE servers that negotiates + * automatically. + */ + public static Builder builder() { + return new Builder(); + } + + RTCConfiguration toConfiguration() { + RTCConfiguration config = new RTCConfiguration(); + + for (RTCIceServer server : iceServers) { + RTCIceServer copy = new RTCIceServer(); + copy.urls.addAll(server.urls); + copy.username = server.username; + copy.password = server.password; + config.iceServers.add(copy); + } + + if (relayOnly) { + config.iceTransportPolicy = RTCIceTransportPolicy.RELAY; + } + + // Perfect negotiation lets the polite peer apply a remote offer while + // its own is pending, which rolls its own offer back. + config.enableImplicitRollback = negotiation == Negotiation.AUTOMATIC; + + if (customizer != null) { + customizer.accept(config); + } + + return config; + } + + + + /** Builds {@link PeerOptions}. */ + public static final class Builder { + + private final List iceServers = new ArrayList<>(); + + private final Set receive = EnumSet.noneOf(MediaKind.class); + + private String name; + + private boolean relayOnly; + + private Signaling signaling; + + private boolean polite; + + private Negotiation negotiation = Negotiation.AUTOMATIC; + + private Consumer customizer; + + + private Builder() { + } + + /** + * @param name the name in thread names and messages. + * + * @return this builder. + */ + public Builder name(String name) { + this.name = requireNonNull(name, "name"); + return this; + } + + /** + * Adds a STUN or TURN server that needs no credentials. + * + * @param urls the server's URLs, such as {@code + * stun:stun.l.google.com:19302}. + * + * @return this builder. + */ + public Builder iceServer(String... urls) { + return iceServer(Arrays.asList(urls), null, null); + } + + /** + * Adds a TURN server. + * + * @param url the server's URL, such as {@code + * turn:turn.example.com:3478}. + * @param username the user name. + * @param password the password. + * + * @return this builder. + */ + public Builder iceServer(String url, String username, String password) { + return iceServer(Collections.singletonList(url), requireNonNull(username, "username"), + requireNonNull(password, "password")); + } + + /** + * @param relayOnly true to connect through TURN servers only, which + * hides this side's addresses. + * + * @return this builder. + */ + public Builder relayOnly(boolean relayOnly) { + this.relayOnly = relayOnly; + return this; + } + + /** + * @param signaling what carries messages to the remote peer. Required + * for {@link Negotiation#AUTOMATIC}. + * + * @return this builder. + */ + public Builder signaling(Signaling signaling) { + this.signaling = requireNonNull(signaling, "signaling"); + return this; + } + + /** + * Sets the role in perfect negotiation. When both peers send an offer + * at once, the polite one drops its own and answers the other. Exactly + * one of two peers must be polite; the default is impolite. + * + * @param polite true for the polite role. + * + * @return this builder. + */ + public Builder polite(boolean polite) { + this.polite = polite; + return this; + } + + /** + * @param negotiation who drives the offer and answer exchange. + * + * @return this builder. + */ + public Builder negotiation(Negotiation negotiation) { + this.negotiation = requireNonNull(negotiation, "negotiation"); + return this; + } + + /** + * Receives audio or video without sending any, as a viewer does. + * + * @param kinds the kinds to receive. + * + * @return this builder. + */ + public Builder receive(MediaKind... kinds) { + for (MediaKind kind : kinds) { + receive.add(requireNonNull(kind, "kind")); + } + return this; + } + + /** + * Changes anything else of the {@link RTCConfiguration} the peer is + * created with, after this builder applied its own values. + * + * @param customizer changes the configuration. + * + * @return this builder. + */ + public Builder configure(Consumer customizer) { + this.customizer = requireNonNull(customizer, "customizer"); + return this; + } + + /** + * @return the options. + * + * @throws IllegalStateException if the peer negotiates automatically + * and has no signaling. + */ + public PeerOptions build() { + if (negotiation == Negotiation.AUTOMATIC && signaling == null) { + throw new IllegalStateException("A peer that negotiates automatically needs a Signaling. " + + "Set one, or use Negotiation.MANUAL to drive the exchange through Peer.negotiator()"); + } + + return new PeerOptions(this); + } + + private Builder iceServer(List urls, String username, String password) { + if (urls.isEmpty()) { + throw new IllegalArgumentException("An ICE server needs at least one URL"); + } + + RTCIceServer server = new RTCIceServer(); + + for (String url : urls) { + requireNonNull(url, "url"); + + if (!url.startsWith("stun:") && !url.startsWith("stuns:") + && !url.startsWith("turn:") && !url.startsWith("turns:")) { + throw new IllegalArgumentException("An ICE server URL starts with stun:, stuns:, turn: or turns:, got " + url); + } + + server.urls.add(url); + } + + server.username = username; + server.password = password; + iceServers.add(server); + + return this; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerState.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerState.java new file mode 100644 index 00000000..d0b6c1c9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerState.java @@ -0,0 +1,61 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import dev.onvoid.webrtc.RTCPeerConnectionState; + +/** + * The state of a {@link Peer}. + */ +public enum PeerState { + + /** Created, nothing negotiated yet. */ + NEW, + + /** Negotiating, or ICE and DTLS are establishing the connection. */ + CONNECTING, + + /** Media and data can flow. */ + CONNECTED, + + /** Connectivity was lost. It may come back by itself. */ + DISCONNECTED, + + /** The connection failed. It needs {@link Peer#restartIce()} or a new peer. */ + FAILED, + + /** The peer was closed. This state is final. */ + CLOSED; + + static PeerState of(RTCPeerConnectionState state) { + switch (state) { + case NEW: + return NEW; + case CONNECTING: + return CONNECTING; + case CONNECTED: + return CONNECTED; + case DISCONNECTED: + return DISCONNECTED; + case FAILED: + return FAILED; + default: + return CLOSED; + } + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerStats.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerStats.java new file mode 100644 index 00000000..d0580d7f --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerStats.java @@ -0,0 +1,161 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; + +import dev.onvoid.webrtc.RTCStatsReport; + +/** + * The statistics people look for first, taken from an {@link RTCStatsReport}. + * Bitrates are computed between two samples, so the first sample a peer takes + * has none. + */ +public final class PeerStats { + + private final Instant timestamp; + + private final String candidateType; + + private final Duration roundTripTime; + + private final long bytesSent; + + private final long bytesReceived; + + private final long outgoingBitrate; + + private final long incomingBitrate; + + private final double packetLoss; + + private final List outboundVideo; + + private final List inboundVideo; + + private final RTCStatsReport report; + + + PeerStats(Instant timestamp, String candidateType, Duration roundTripTime, + long bytesSent, long bytesReceived, long outgoingBitrate, + long incomingBitrate, double packetLoss, + List outboundVideo, List inboundVideo, + RTCStatsReport report) { + this.timestamp = timestamp; + this.candidateType = candidateType; + this.roundTripTime = roundTripTime; + this.bytesSent = bytesSent; + this.bytesReceived = bytesReceived; + this.outgoingBitrate = outgoingBitrate; + this.incomingBitrate = incomingBitrate; + this.packetLoss = packetLoss; + this.outboundVideo = outboundVideo; + this.inboundVideo = inboundVideo; + this.report = report; + } + + /** + * @return when the statistics were taken. + */ + public Instant timestamp() { + return timestamp; + } + + /** + * @return the type of the local candidate in use: {@code host}, {@code + * srflx}, {@code prflx} or {@code relay}. Null while not connected. + */ + public String candidateType() { + return candidateType; + } + + /** + * @return the current round trip time, or null while not measured. + */ + public Duration roundTripTime() { + return roundTripTime; + } + + /** + * @return the bytes sent on the connection so far. + */ + public long bytesSent() { + return bytesSent; + } + + /** + * @return the bytes received on the connection so far. + */ + public long bytesReceived() { + return bytesReceived; + } + + /** + * @return the bits sent per second since the previous sample, or -1 for + * the first sample. + */ + public long outgoingBitrate() { + return outgoingBitrate; + } + + /** + * @return the bits received per second since the previous sample, or -1 + * for the first sample. + */ + public long incomingBitrate() { + return incomingBitrate; + } + + /** + * @return the share of received packets lost so far, 0 to 1. + */ + public double packetLoss() { + return packetLoss; + } + + /** + * @return the video streams sent. + */ + public List outboundVideo() { + return outboundVideo; + } + + /** + * @return the video streams received. + */ + public List inboundVideo() { + return inboundVideo; + } + + /** + * @return the whole report the values were taken from. + */ + public RTCStatsReport report() { + return report; + } + + @Override + public String toString() { + return "PeerStats[candidate=" + candidateType + ", rtt=" + roundTripTime + + ", sent=" + bytesSent + " B, received=" + bytesReceived + + " B, out=" + outgoingBitrate + " bps, in=" + incomingBitrate + + " bps, loss=" + packetLoss + ", outboundVideo=" + outboundVideo + + ", inboundVideo=" + inboundVideo + "]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudio.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudio.java new file mode 100644 index 00000000..d64aca72 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudio.java @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.function.Consumer; + +/** + * An audio track the remote peer sends. + */ +public interface RemoteAudio extends RemoteTrack { + + /** + * Calls the handler with the decoded audio, in order. Audio reaches + * handlers only while the runtime plays out, which a {@link + * AudioMode#HEADLESS} runtime always does. + * + * @param handler called on a thread of this subscription. + * + * @return the subscription that stops the audio. + */ + Subscription onAudio(Consumer handler); + + /** + * Sets the playout gain of this track. + * + * @param volume 0 for silence to 10, where 1 plays the track as received. + */ + void setVolume(double volume); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudioImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudioImpl.java new file mode 100644 index 00000000..aca25b8d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteAudioImpl.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.media.audio.AudioTrack; + +/** + * A remote audio track. Each audio subscription is a sink of the native track + * with a thread of its own. + */ +final class RemoteAudioImpl extends RemoteTrackImpl implements RemoteAudio { + + private final AudioTrack track; + + // Confined to the peer's control executor. + + private final List dispatchers = new ArrayList<>(); + + + RemoteAudioImpl(PeerImpl peer, RTCRtpReceiver receiver, AudioTrack track, + String id, List streamIds) { + super(peer, receiver, track, MediaKind.AUDIO, id, streamIds); + + this.track = track; + } + + @Override + public RemoteAudio asAudio() { + return this; + } + + @Override + public Subscription onAudio(Consumer handler) { + requireNonNull(handler, "handler"); + + SerialExecutor executor = new SerialExecutor(peer.rtc.framePool, peer.rtc::reportError, false, null); + AudioDispatcher dispatcher = new AudioDispatcher(handler, executor, peer.rtc::reportError); + + onControl(() -> { + dispatchers.add(dispatcher); + track.addSink(dispatcher); + }); + + return () -> { + dispatcher.close(); + + onControl(() -> { + if (dispatchers.remove(dispatcher)) { + track.removeSink(dispatcher); + } + }); + }; + } + + @Override + public void setVolume(double volume) { + if (volume < 0 || volume > 10) { + throw new IllegalArgumentException("The volume must be between 0 and 10, got " + volume); + } + + onControl(() -> track.setVolume(volume)); + } + + @Override + void detachSinks() { + for (AudioDispatcher dispatcher : dispatchers) { + track.removeSink(dispatcher); + dispatcher.close(); + } + + dispatchers.clear(); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrack.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrack.java new file mode 100644 index 00000000..14ba60e3 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrack.java @@ -0,0 +1,100 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.List; +import java.util.function.Consumer; + +/** + * A track the remote peer sends, handed to {@link Peer#onTrack}. It ends when + * the remote peer stops sending it or the peer closes. + */ +public interface RemoteTrack { + + /** + * @return the track id. + */ + String id(); + + /** + * @return audio or video. + */ + MediaKind kind(); + + /** + * @return the streams the remote peer added the track to. + */ + List streamIds(); + + /** + * @return true for a video track. + */ + default boolean isVideo() { + return kind() == MediaKind.VIDEO; + } + + /** + * @return true for an audio track. + */ + default boolean isAudio() { + return kind() == MediaKind.AUDIO; + } + + /** + * @return this track as a video track. + * + * @throws IllegalStateException for an audio track. + */ + RemoteVideo asVideo(); + + /** + * @return this track as an audio track. + * + * @throws IllegalStateException for a video track. + */ + RemoteAudio asAudio(); + + /** + * @return true when the remote peer sends no media on the track for now. + */ + boolean isMuted(); + + /** + * Calls the handler when the track is muted or unmuted. + * + * @param handler called on the peer's event thread with the new value. + * + * @return the subscription that removes the handler. + */ + Subscription onMute(Consumer handler); + + /** + * Calls the handler once the track ended. A handler added after that is + * called at once. + * + * @param handler called on the peer's event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onEnded(Runnable handler); + + /** + * @return true once the track ended. + */ + boolean isEnded(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java new file mode 100644 index 00000000..c072da80 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java @@ -0,0 +1,193 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.media.MediaStreamTrack; +import dev.onvoid.webrtc.media.MediaStreamTrackMuteListener; + +/** + * A {@link RemoteTrack} of a {@link PeerImpl}. The native receiver and track + * view are used on the peer's control executor; the track's handlers run on + * the peer's event executor. + */ +abstract class RemoteTrackImpl implements RemoteTrack { + + final PeerImpl peer; + + private final RTCRtpReceiver receiver; + + private final MediaStreamTrack track; + + private final String id; + + private final MediaKind kind; + + private final List streamIds; + + private final MediaStreamTrackMuteListener muteListener; + + private volatile boolean muted; + + private volatile boolean ended; + + // Confined to the peer's event executor. + + private boolean endDelivered; + + private final Handlers muteHandlers; + + private final Handlers endedHandlers; + + + RemoteTrackImpl(PeerImpl peer, RTCRtpReceiver receiver, MediaStreamTrack track, + MediaKind kind, String id, List streamIds) { + this.peer = peer; + this.receiver = receiver; + this.track = track; + this.kind = kind; + this.id = id; + this.streamIds = Collections.unmodifiableList(streamIds); + this.muteHandlers = new Handlers<>(peer.rtc::reportError); + this.endedHandlers = new Handlers<>(peer.rtc::reportError); + this.muteListener = (mutedTrack, isMuted) -> peer.events.execute(() -> muteChanged(isMuted)); + + track.addTrackMuteListener(muteListener); + } + + @Override + public String id() { + return id; + } + + @Override + public MediaKind kind() { + return kind; + } + + @Override + public List streamIds() { + return streamIds; + } + + @Override + public RemoteVideo asVideo() { + throw new IllegalStateException("The track " + id + " is an audio track"); + } + + @Override + public RemoteAudio asAudio() { + throw new IllegalStateException("The track " + id + " is a video track"); + } + + @Override + public boolean isMuted() { + return muted; + } + + @Override + public boolean isEnded() { + return ended; + } + + @Override + public Subscription onMute(Consumer handler) { + requireNonNull(handler, "handler"); + + return peer.subscribe(() -> muteHandlers.add(handler)); + } + + @Override + public Subscription onEnded(Runnable handler) { + requireNonNull(handler, "handler"); + + return peer.subscribe(() -> { + if (endDelivered) { + peer.call(ignored -> handler.run(), null); + return () -> { }; + } + + return endedHandlers.add(ignored -> handler.run()); + }); + } + + @Override + public String toString() { + return "RemoteTrack[" + kind + ", " + id + "]"; + } + + /** + * Runs a change of the native track on the peer's control executor, or + * does nothing once the track ended. + */ + void onControl(Runnable change) { + peer.control.execute(() -> { + if (!ended) { + change.run(); + } + }); + } + + /** Detaches everything from the native track, on the control executor. */ + void end() { + if (ended) { + return; + } + + ended = true; + detachSinks(); + + try { + track.removeTrackMuteListener(muteListener); + track.dispose(); + receiver.dispose(); + } + catch (Throwable e) { + peer.rtc.reportError(e); + } + } + + /** Called once, on the control executor, while the native track exists. */ + abstract void detachSinks(); + + /** Tells the handlers the track ended, on the event executor. */ + void fireEnded() { + if (endDelivered) { + return; + } + + endDelivered = true; + endedHandlers.accept(null); + endedHandlers.clear(); + muteHandlers.clear(); + } + + private void muteChanged(boolean isMuted) { + if (endDelivered || isMuted == muted) { + return; + } + + muted = isMuted; + muteHandlers.accept(isMuted); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideo.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideo.java new file mode 100644 index 00000000..5c526b8e --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideo.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.function.Consumer; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A video track the remote peer sends. + */ +public interface RemoteVideo extends RemoteTrack { + + /** + * Calls the handler with the decoded frames, newest first: when the + * handler falls behind, the frames in between are dropped. + * + * @param handler called on a thread of this subscription. A frame is valid + * while the handler runs; call {@code retain()} to keep it + * longer and {@code release()} when done. + * + * @return the subscription that stops the frames. + */ + Subscription onFrame(Consumer handler); + + /** + * Calls the handler with the decoded frames. + * + * @param delivery what happens to frames when the handler falls behind. + * @param handler called on a thread of this subscription. A frame is + * valid while the handler runs; call {@code retain()} to + * keep it longer and {@code release()} when done. + * + * @return the subscription that stops the frames. + */ + Subscription onFrame(FrameDelivery delivery, Consumer handler); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideoImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideoImpl.java new file mode 100644 index 00000000..60999bc9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteVideoImpl.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.RTCRtpReceiver; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoTrack; + +/** + * A remote video track. Each frame subscription is a sink of the native track + * with a thread of its own. + */ +final class RemoteVideoImpl extends RemoteTrackImpl implements RemoteVideo { + + private final VideoTrack track; + + // Confined to the peer's control executor. + + private final List dispatchers = new ArrayList<>(); + + + RemoteVideoImpl(PeerImpl peer, RTCRtpReceiver receiver, VideoTrack track, + String id, List streamIds) { + super(peer, receiver, track, MediaKind.VIDEO, id, streamIds); + + this.track = track; + } + + @Override + public RemoteVideo asVideo() { + return this; + } + + @Override + public Subscription onFrame(Consumer handler) { + return onFrame(FrameDelivery.LATEST, handler); + } + + @Override + public Subscription onFrame(FrameDelivery delivery, Consumer handler) { + requireNonNull(delivery, "delivery"); + requireNonNull(handler, "handler"); + + SerialExecutor executor = new SerialExecutor(peer.rtc.framePool, peer.rtc::reportError, false, null); + FrameDispatcher dispatcher = new FrameDispatcher(handler, delivery, executor, peer.rtc::reportError); + + onControl(() -> { + dispatchers.add(dispatcher); + track.addSink(dispatcher); + }); + + return () -> { + dispatcher.close(); + + onControl(() -> { + if (dispatchers.remove(dispatcher)) { + track.removeSink(dispatcher); + } + }); + }; + } + + @Override + void detachSinks() { + for (FrameDispatcher dispatcher : dispatchers) { + track.removeSink(dispatcher); + dispatcher.close(); + } + + dispatchers.clear(); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/ScreenOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/ScreenOptions.java new file mode 100644 index 00000000..3dd76567 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/ScreenOptions.java @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; + +/** + * Which screen or window to capture and how. + * 
{@code
+ * ScreenOptions.screen(devices.screens().get(0)).frameRate(15).build();
+ * }
+ */ +public final class ScreenOptions { + + final DesktopSource source; + + final boolean window; + + final int frameRate; + + final int maxWidth; + + final int maxHeight; + + + private ScreenOptions(Builder builder) { + this.source = builder.source; + this.window = builder.window; + this.frameRate = builder.frameRate; + this.maxWidth = builder.maxWidth; + this.maxHeight = builder.maxHeight; + } + + /** + * @param screen one of {@link Devices#screens()}. + * + * @return a builder that captures the screen at 30 frames per second. + */ + public static Builder screen(DesktopSource screen) { + return new Builder(requireNonNull(screen, "screen"), false); + } + + /** + * @param window one of {@link Devices#windows()}. + * + * @return a builder that captures the window at 30 frames per second. + */ + public static Builder window(DesktopSource window) { + return new Builder(requireNonNull(window, "window"), true); + } + + + + /** Builds {@link ScreenOptions}. */ + public static final class Builder { + + private final DesktopSource source; + + private final boolean window; + + private int frameRate = 30; + + private int maxWidth; + + private int maxHeight; + + + private Builder(DesktopSource source, boolean window) { + this.source = source; + this.window = window; + } + + /** + * @param frameRate the frames per second to capture. + * + * @return this builder. + */ + public Builder frameRate(int frameRate) { + if (frameRate <= 0) { + throw new IllegalArgumentException("The frame rate must be positive, got " + frameRate); + } + + this.frameRate = frameRate; + return this; + } + + /** + * Scales captured frames down to fit, keeping their aspect ratio. + * + * @param width the largest width, in pixels. + * @param height the largest height, in pixels. + * + * @return this builder. + */ + public Builder maxSize(int width, int height) { + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("Invalid size " + width + "x" + height); + } + + this.maxWidth = width; + this.maxHeight = height; + return this; + } + + /** + * @return the options. + */ + public ScreenOptions build() { + return new ScreenOptions(this); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java new file mode 100644 index 00000000..d89d996e --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.concurrent.CompletableFuture; + +/** + * Sends one track on one peer, made by {@link Peer#addTrack}. + */ +public interface Sender { + + /** + * @return the track sent now, or null if nothing is sent. + */ + LocalTrack track(); + + /** + * Sends another track of the same kind, without renegotiating. + * + * @param track the track to send, or null to send nothing. + * + * @return a future that completes once the track is replaced. + */ + CompletableFuture replaceTrack(LocalTrack track); + + /** + * Changes how the track is encoded. Values left unset stay as they are. + * + * @param encoding the values to change. + * + * @return a future that completes once the values are applied. + */ + CompletableFuture configure(Encoding encoding); + + /** + * Stops sending. With {@link Negotiation#AUTOMATIC} the peer renegotiates. + * The track stays open. + * + * @return a future that completes once the sender is removed. + */ + CompletableFuture remove(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java new file mode 100644 index 00000000..045913e0 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java @@ -0,0 +1,253 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.RTCRtpEncodingParameters; +import dev.onvoid.webrtc.RTCRtpSendParameters; +import dev.onvoid.webrtc.RTCRtpSender; + +/** + * The {@link Sender} of a {@link PeerImpl}. The native sender is used on the + * peer's control executor only. + */ +final class SenderImpl implements Sender { + + private final PeerImpl peer; + + private final MediaKind kind; + + private volatile LocalTrackImpl track; + + // Confined to the peer's control executor. + + private RTCPeerConnection connection; + + private RTCRtpSender sender; + + private boolean removed; + + private Throwable attachError; + + + SenderImpl(PeerImpl peer, LocalTrackImpl track) { + this.peer = peer; + this.track = track; + this.kind = track.kind(); + } + + void attach(RTCPeerConnection connection, List streamIds) { + LocalTrackImpl local = track; + + if (!local.register(this)) { + throw new IllegalStateException("The track " + local.id() + " is closed"); + } + + try { + RTCRtpSender added = connection.addTrack(local.nativeTrack(), streamIds); + + if (added == null) { + throw new WebRtcException("WebRTC refused the track " + local.id()); + } + + this.connection = connection; + this.sender = added; + } + catch (RuntimeException e) { + local.unregister(this); + throw e; + } + } + + void failAttach(Throwable error) { + attachError = error; + removed = true; + track = null; + } + + @Override + public LocalTrack track() { + return track; + } + + @Override + public CompletableFuture replaceTrack(LocalTrack replacement) { + LocalTrackImpl next = replacement == null ? null : checkTrack(replacement); + + return run(() -> { + if (next != null && !next.register(this)) { + throw new IllegalStateException("The track " + next.id() + " is closed"); + } + + try { + sender.replaceTrack(next == null ? null : next.nativeTrack()); + } + catch (RuntimeException e) { + if (next != null) { + next.unregister(this); + } + throw e; + } + + LocalTrackImpl previous = track; + track = next; + + if (previous != null && previous != next) { + previous.unregister(this); + } + }); + } + + @Override + public CompletableFuture configure(Encoding encoding) { + if (encoding == null) { + throw new NullPointerException("encoding"); + } + + return run(() -> { + RTCRtpSendParameters parameters = sender.getParameters(); + + for (RTCRtpEncodingParameters values : parameters.encodings) { + if (encoding.active != null) { + values.active = encoding.active; + } + if (encoding.maxBitrate != null) { + values.maxBitrate = encoding.maxBitrate; + } + if (kind == MediaKind.VIDEO) { + if (encoding.maxFrameRate != null) { + values.maxFramerate = encoding.maxFrameRate; + } + if (encoding.scaleDownBy != null) { + values.scaleResolutionDownBy = encoding.scaleDownBy; + } + } + } + + sender.setParameters(parameters); + }); + } + + @Override + public CompletableFuture remove() { + return run(() -> { + connection.removeTrack(sender); + release(); + peer.forgetSender(this); + }); + } + + @Override + public String toString() { + LocalTrackImpl current = track; + + return "Sender[" + kind + ", " + (current == null ? "no track" : current.id()) + "]"; + } + + /** + * Stops sending a track that closes. + * + * @return a future that completes once the sender let go of the track. + */ + CompletableFuture trackClosed(LocalTrackImpl closedTrack) { + CompletableFuture done = new CompletableFuture<>(); + + peer.control.execute(() -> { + try { + if (!removed && sender != null && track == closedTrack && !peer.isClosed()) { + sender.replaceTrack(null); + } + } + catch (Throwable e) { + peer.rtc.reportError(e); + } + finally { + if (track == closedTrack) { + track = null; + } + done.complete(null); + } + }); + + return done; + } + + /** + * Lets go of the native sender while the peer closes, on its control + * executor. + */ + void closeFromPeer() { + release(); + } + + private void release() { + removed = true; + + if (sender != null) { + sender.dispose(); + sender = null; + } + + LocalTrackImpl previous = track; + track = null; + + if (previous != null) { + previous.unregister(this); + } + } + + private LocalTrackImpl checkTrack(LocalTrack replacement) { + if (!(replacement instanceof LocalTrackImpl) || ((LocalTrackImpl) replacement).rtc != peer.rtc) { + throw new IllegalArgumentException("The track was not created by the runtime of " + peer.name); + } + if (replacement.kind() != kind) { + throw new IllegalArgumentException("A " + kind + " sender cannot send a " + replacement.kind() + " track"); + } + + return (LocalTrackImpl) replacement; + } + + private CompletableFuture run(Runnable operation) { + CompletableFuture result = new CompletableFuture<>(); + + peer.control.execute(() -> { + try { + if (peer.isClosed()) { + throw peer.closedError(); + } + if (attachError != null) { + throw PeerImpl.asWebRtcException(attachError, "The track was never added"); + } + if (removed) { + throw new IllegalStateException("The sender was removed"); + } + + operation.run(); + result.complete(null); + } + catch (Throwable e) { + result.completeExceptionally(e instanceof IllegalStateException || e instanceof IllegalArgumentException + ? e : PeerImpl.asWebRtcException(e, "Changing the sender failed")); + } + }); + + return peer.toPublic(result); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SerialExecutor.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SerialExecutor.java new file mode 100644 index 00000000..6b36b4ce --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SerialExecutor.java @@ -0,0 +1,190 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayDeque; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.function.Consumer; + +/** + * Runs tasks one at a time, in the order they were submitted, on threads of + * another executor. Many of them share one pool, while the tasks of each run + * as if on a thread of their own. + *

+ * A task that throws does not stop the ones after it; its exception goes to + * the error handler. + */ +final class SerialExecutor implements Executor { + + /** Tasks run per turn, so one busy executor cannot starve the others of a shared pool. */ + private static final int BATCH_SIZE = 64; + + /** Marks the threads that run event tasks, to catch a future waited on there. */ + private static final ThreadLocal EVENT_DEPTH = new ThreadLocal() { + + @Override + protected int[] initialValue() { + return new int[1]; + } + }; + + private final Executor executor; + + private final Consumer errorHandler; + + private final boolean eventThread; + + private final String threadName; + + private final ArrayDeque tasks = new ArrayDeque<>(); + + private boolean scheduled; + + private boolean shutdown; + + private volatile Thread runner; + + + /** + * @param executor the executor whose threads run the tasks. + * @param errorHandler receives what a task throws. + * @param eventThread true when the tasks run application code, so that + * waiting on a future of this API there is refused. + * @param threadName the name the running thread takes while it runs the + * tasks, or null to leave it alone. + */ + SerialExecutor(Executor executor, Consumer errorHandler, + boolean eventThread, String threadName) { + this.executor = executor; + this.errorHandler = errorHandler; + this.eventThread = eventThread; + this.threadName = threadName; + } + + /** + * @return true when the calling thread runs an event task right now, of + * any executor. + */ + static boolean onEventThread() { + return EVENT_DEPTH.get()[0] > 0; + } + + @Override + public void execute(Runnable task) { + synchronized (tasks) { + if (shutdown) { + return; + } + + tasks.add(task); + + if (scheduled) { + return; + } + + scheduled = true; + } + + schedule(); + } + + /** + * @return true when the calling thread is running a task of this executor. + */ + boolean inExecutor() { + return runner == Thread.currentThread(); + } + + /** + * Drops the tasks not yet run and refuses new ones. A task running now + * finishes. + */ + void shutdown() { + synchronized (tasks) { + shutdown = true; + tasks.clear(); + } + } + + private void schedule() { + try { + executor.execute(this::drain); + } + catch (RejectedExecutionException e) { + synchronized (tasks) { + scheduled = false; + tasks.clear(); + } + } + } + + private void drain() { + Thread thread = Thread.currentThread(); + String previousName = null; + + if (threadName != null) { + previousName = thread.getName(); + thread.setName(threadName); + } + if (eventThread) { + EVENT_DEPTH.get()[0]++; + } + + runner = thread; + + boolean more = true; + + try { + for (int i = 0; i < BATCH_SIZE; i++) { + Runnable task; + + synchronized (tasks) { + task = tasks.poll(); + + if (task == null) { + scheduled = false; + more = false; + break; + } + } + + try { + task.run(); + } + catch (Throwable e) { + errorHandler.accept(e); + } + } + } + finally { + runner = null; + + if (eventThread) { + EVENT_DEPTH.get()[0]--; + } + if (previousName != null) { + thread.setName(previousName); + } + } + + if (more) { + // Give the other executors of the pool a turn. + schedule(); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalMessage.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalMessage.java new file mode 100644 index 00000000..f6da6fdd --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalMessage.java @@ -0,0 +1,297 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.util.Map; +import java.util.Objects; + +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCSdpType; +import dev.onvoid.webrtc.RTCSessionDescription; + +/** + * A message one peer sends the other through its {@link Signaling}: a session + * description or an ICE candidate. + *

+ * {@link #toJson()} writes the shapes the browser API uses for {@code + * RTCSessionDescriptionInit} and {@code RTCIceCandidateInit}, so a JavaScript + * peer can pass them straight to {@code setRemoteDescription} and {@code + * addIceCandidate}: + * 

{@code
+ * {"type":"offer","sdp":"v=0\r\n..."}
+ * {"candidate":"candidate:842163049 1 udp ...","sdpMid":"0","sdpMLineIndex":0}
+ * {"candidate":""}
+ * }
+ * The last one is the end of candidates. {@link #fromJson(String)} reads these, + * as well as the {@code {"description": ...}} and {@code {"candidate": {...}}} + * envelopes of the perfect negotiation examples for browsers. + */ +public final class SignalMessage { + + /** What a message carries. */ + public enum Type { + + /** A session description offer. */ + OFFER, + + /** A session description answer. */ + ANSWER, + + /** An ICE candidate. */ + CANDIDATE, + + /** The remote peer has no more candidates. */ + END_OF_CANDIDATES + + } + + private static final SignalMessage END_OF_CANDIDATES = + new SignalMessage(Type.END_OF_CANDIDATES, null, null); + + private final Type type; + + private final RTCSessionDescription description; + + private final RTCIceCandidate candidate; + + + private SignalMessage(Type type, RTCSessionDescription description, + RTCIceCandidate candidate) { + this.type = type; + this.description = description; + this.candidate = candidate; + } + + /** + * @param description an offer or an answer. + * + * @return a message that carries the description. + * + * @throws IllegalArgumentException if the description is neither an offer + * nor an answer. + */ + public static SignalMessage of(RTCSessionDescription description) { + requireNonNull(description, "description"); + requireNonNull(description.sdp, "description.sdp"); + + if (description.sdpType == RTCSdpType.OFFER) { + return new SignalMessage(Type.OFFER, description, null); + } + if (description.sdpType == RTCSdpType.ANSWER) { + return new SignalMessage(Type.ANSWER, description, null); + } + + throw new IllegalArgumentException("Only offers and answers are signaled, not " + description.sdpType); + } + + /** + * @param candidate a local ICE candidate. + * + * @return a message that carries the candidate. + */ + public static SignalMessage of(RTCIceCandidate candidate) { + requireNonNull(candidate, "candidate"); + requireNonNull(candidate.sdp, "candidate.sdp"); + + if (candidate.sdp.isEmpty()) { + return END_OF_CANDIDATES; + } + + return new SignalMessage(Type.CANDIDATE, null, candidate); + } + + /** + * @return the message that says no more candidates follow. + */ + public static SignalMessage endOfCandidates() { + return END_OF_CANDIDATES; + } + + /** + * Reads a message written by {@link #toJson()} or by a browser. + * + * @param json the message as JSON. + * + * @return the message. + * + * @throws IllegalArgumentException if the text is no JSON or no signaling + * message. + */ + public static SignalMessage fromJson(String json) { + requireNonNull(json, "json"); + + Object value = Json.parse(json); + + if (!(value instanceof Map)) { + throw new IllegalArgumentException("A signaling message is a JSON object"); + } + + return fromObject((Map) value); + } + + /** + * @return what the message carries. + */ + public Type type() { + return type; + } + + /** + * @return the session description of an {@link Type#OFFER} or an {@link + * Type#ANSWER}, otherwise null. + */ + public RTCSessionDescription description() { + return description; + } + + /** + * @return the ICE candidate of a {@link Type#CANDIDATE}, otherwise null. + */ + public RTCIceCandidate candidate() { + return candidate; + } + + /** + * @return the message as JSON, in the shapes the browser API uses. + */ + public String toJson() { + StringBuilder out = new StringBuilder(); + + switch (type) { + case OFFER: + case ANSWER: + out.append("{\"type\":"); + Json.writeString(out, type == Type.OFFER ? "offer" : "answer"); + out.append(",\"sdp\":"); + Json.writeString(out, description.sdp); + out.append('}'); + break; + + case CANDIDATE: + out.append("{\"candidate\":"); + Json.writeString(out, candidate.sdp); + out.append(",\"sdpMid\":"); + if (candidate.sdpMid == null) { + out.append("null"); + } + else { + Json.writeString(out, candidate.sdpMid); + } + out.append(",\"sdpMLineIndex\":").append(candidate.sdpMLineIndex); + out.append('}'); + break; + + default: + out.append("{\"candidate\":\"\"}"); + } + + return out.toString(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof SignalMessage)) { + return false; + } + + SignalMessage other = (SignalMessage) obj; + + return type == other.type && Objects.equals(toJson(), other.toJson()); + } + + @Override + public int hashCode() { + return toJson().hashCode(); + } + + @Override + public String toString() { + return "SignalMessage" + toJson(); + } + + private static SignalMessage fromObject(Map object) { + if (object.containsKey("description")) { + Object description = object.get("description"); + + if (!(description instanceof Map)) { + throw new IllegalArgumentException("\"description\" is not an object"); + } + + return fromObject((Map) description); + } + + if (object.containsKey("candidate")) { + Object candidate = object.get("candidate"); + + if (candidate == null) { + return END_OF_CANDIDATES; + } + if (candidate instanceof Map) { + return fromObject((Map) candidate); + } + if (!(candidate instanceof String)) { + throw new IllegalArgumentException("\"candidate\" is not a string"); + } + + String sdp = (String) candidate; + + if (sdp.isEmpty()) { + return END_OF_CANDIDATES; + } + + Object mid = object.get("sdpMid"); + Object index = object.get("sdpMLineIndex"); + + if (mid != null && !(mid instanceof String)) { + throw new IllegalArgumentException("\"sdpMid\" is not a string"); + } + if (index != null && !(index instanceof Number)) { + throw new IllegalArgumentException("\"sdpMLineIndex\" is not a number"); + } + if (mid == null && index == null) { + throw new IllegalArgumentException("A candidate needs \"sdpMid\" or \"sdpMLineIndex\""); + } + + return new SignalMessage(Type.CANDIDATE, null, new RTCIceCandidate( + (String) mid, index == null ? 0 : ((Number) index).intValue(), sdp)); + } + + Object type = object.get("type"); + Object sdp = object.get("sdp"); + + if (!(type instanceof String)) { + throw new IllegalArgumentException("The message has neither \"type\" nor \"candidate\""); + } + if (!(sdp instanceof String)) { + throw new IllegalArgumentException("The description has no \"sdp\" string"); + } + + switch ((String) type) { + case "offer": + return of(new RTCSessionDescription(RTCSdpType.OFFER, (String) sdp)); + case "answer": + return of(new RTCSessionDescription(RTCSdpType.ANSWER, (String) sdp)); + default: + throw new IllegalArgumentException("Unsupported description type \"" + type + "\""); + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Signaling.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Signaling.java new file mode 100644 index 00000000..92a6e54d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Signaling.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.function.Consumer; + +/** + * Carries a peer's {@link SignalMessage}s to the remote peer. The library has + * no transport of its own: a WebSocket, an HTTP endpoint, or anything else that + * delivers messages in order will do. + *

+ * In its simplest form a signaling is a lambda that sends, with incoming + * messages handed to {@link Peer#signal(SignalMessage)}: + * 

{@code
+ * PeerOptions.builder().signaling(message -> socket.send(message.toJson()));
+ * socket.onText(text -> peer.signal(SignalMessage.fromJson(text)));
+ * }
+ */ +@FunctionalInterface +public interface Signaling { + + /** + * Sends a message to the remote peer. Called on the peer's event thread, in + * the order the remote peer must receive the messages. Blocking here only + * delays the events of this peer. + * + * @param message the message to send. + */ + void send(SignalMessage message); + + /** + * Called once, when the peer is created. A transport that receives + * messages may push them into {@code inbound} instead of calling {@link + * Peer#signal(SignalMessage)}. Does nothing by default. + * + * @param inbound takes the messages the remote peer sent, from any thread. + */ + default void attach(Consumer inbound) { + } + + /** + * Creates two linked ends: what one end sends, the peer attached to the + * other end receives. Suits tests and peers within one process. + * + * @return the two ends. + */ + static SignalingPair loopback() { + return new SignalingPair(); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalingPair.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalingPair.java new file mode 100644 index 00000000..51116f68 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SignalingPair.java @@ -0,0 +1,93 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +/** + * Two linked {@link Signaling} ends, made by {@link Signaling#loopback()}. + * Messages sent through one end reach the peer attached to the other end, in + * order. Messages sent before that peer exists are held until it attaches. + */ +public final class SignalingPair { + + private final End first = new End(); + + private final End second = new End(); + + + SignalingPair() { + first.remote = second; + second.remote = first; + } + + /** + * @return the end for the first peer. + */ + public Signaling first() { + return first; + } + + /** + * @return the end for the second peer. + */ + public Signaling second() { + return second; + } + + + + private static final class End implements Signaling { + + private final List held = new ArrayList<>(); + + private End remote; + + private Consumer inbound; + + + @Override + public void send(SignalMessage message) { + remote.deliver(message); + } + + @Override + public synchronized void attach(Consumer inbound) { + if (this.inbound != null) { + throw new IllegalStateException("This end of the loopback is already attached to a peer"); + } + + this.inbound = inbound; + + // Under the lock, so that nothing delivered meanwhile overtakes them. + // A peer's inbound only queues the message. + held.forEach(inbound); + held.clear(); + } + + private synchronized void deliver(SignalMessage message) { + if (inbound == null) { + held.add(message); + } + else { + inbound.accept(message); + } + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/StatsReader.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/StatsReader.java new file mode 100644 index 00000000..0e4ba125 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/StatsReader.java @@ -0,0 +1,146 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import dev.onvoid.webrtc.RTCStats; +import dev.onvoid.webrtc.RTCStatsReport; +import dev.onvoid.webrtc.RTCStatsType; + +/** + * Takes {@link PeerStats} from the reports of one peer. It keeps the previous + * sample to compute bitrates. Used on the peer's control executor. + */ +final class StatsReader { + + private long lastNanos = -1; + + private long lastBytesSent; + + private long lastBytesReceived; + + + PeerStats read(RTCStatsReport report) { + Map stats = report.getStats(); + + long bytesSent = 0; + long bytesReceived = 0; + String candidateType = null; + Duration roundTripTime = null; + long packetsReceived = 0; + long packetsLost = 0; + List outbound = new ArrayList<>(); + List inbound = new ArrayList<>(); + + for (RTCStats entry : stats.values()) { + Map values = entry.getAttributes(); + + if (entry.getType() == RTCStatsType.TRANSPORT) { + bytesSent += longValue(values.get("bytesSent")); + bytesReceived += longValue(values.get("bytesReceived")); + + RTCStats pair = lookup(stats, values.get("selectedCandidatePairId")); + + if (pair != null) { + Object rtt = pair.getAttributes().get("currentRoundTripTime"); + + if (rtt instanceof Number) { + roundTripTime = Duration.ofNanos((long) (((Number) rtt).doubleValue() * 1e9)); + } + + RTCStats local = lookup(stats, pair.getAttributes().get("localCandidateId")); + + if (local != null) { + candidateType = string(local.getAttributes().get("candidateType")); + } + } + } + else if (entry.getType() == RTCStatsType.INBOUND_RTP) { + packetsReceived += longValue(values.get("packetsReceived")); + packetsLost += Math.max(0, longValue(values.get("packetsLost"))); + + if ("video".equals(values.get("kind"))) { + inbound.add(video(stats, values, string(values.get("trackIdentifier")), null)); + } + } + else if (entry.getType() == RTCStatsType.OUTBOUND_RTP) { + if ("video".equals(values.get("kind"))) { + RTCStats source = lookup(stats, values.get("mediaSourceId")); + String trackId = source == null ? null : string(source.getAttributes().get("trackIdentifier")); + + outbound.add(video(stats, values, trackId, string(values.get("qualityLimitationReason")))); + } + } + } + + long now = System.nanoTime(); + long outgoingBitrate = -1; + long incomingBitrate = -1; + + if (lastNanos >= 0 && now > lastNanos) { + double seconds = (now - lastNanos) / 1e9; + + outgoingBitrate = (long) (Math.max(0, bytesSent - lastBytesSent) * 8 / seconds); + incomingBitrate = (long) (Math.max(0, bytesReceived - lastBytesReceived) * 8 / seconds); + } + + lastNanos = now; + lastBytesSent = bytesSent; + lastBytesReceived = bytesReceived; + + long packetsExpected = packetsReceived + packetsLost; + double packetLoss = packetsExpected == 0 ? 0 : (double) packetsLost / packetsExpected; + + return new PeerStats(Instant.now(), candidateType, roundTripTime, bytesSent, bytesReceived, + outgoingBitrate, incomingBitrate, packetLoss, + Collections.unmodifiableList(outbound), Collections.unmodifiableList(inbound), report); + } + + private static VideoStats video(Map stats, Map values, + String trackId, String qualityLimitation) { + RTCStats codec = lookup(stats, values.get("codecId")); + + return new VideoStats(trackId, + codec == null ? null : string(codec.getAttributes().get("mimeType")), + (int) longValue(values.get("frameWidth")), + (int) longValue(values.get("frameHeight")), + doubleValue(values.get("framesPerSecond")), + qualityLimitation); + } + + private static RTCStats lookup(Map stats, Object id) { + return id == null ? null : stats.get(id.toString()); + } + + private static long longValue(Object value) { + return value instanceof Number ? ((Number) value).longValue() : 0; + } + + private static double doubleValue(Object value) { + return value instanceof Number ? ((Number) value).doubleValue() : 0; + } + + private static String string(Object value) { + return value == null ? null : value.toString(); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Subscription.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Subscription.java new file mode 100644 index 00000000..0e159d43 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Subscription.java @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * A registered handler. Closing it removes the handler; closing it again does + * nothing. A handler is also removed when the object it listens to closes. + */ +@FunctionalInterface +public interface Subscription extends AutoCloseable { + + /** + * Removes the handler. It may still be called once if a call was already + * on its way. + */ + @Override + void close(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/VideoStats.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/VideoStats.java new file mode 100644 index 00000000..35dcc8fc --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/VideoStats.java @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * The statistics of one video stream, sent or received. + */ +public final class VideoStats { + + private final String trackId; + + private final String codec; + + private final int width; + + private final int height; + + private final double framesPerSecond; + + private final String qualityLimitation; + + + VideoStats(String trackId, String codec, int width, int height, + double framesPerSecond, String qualityLimitation) { + this.trackId = trackId; + this.codec = codec; + this.width = width; + this.height = height; + this.framesPerSecond = framesPerSecond; + this.qualityLimitation = qualityLimitation; + } + + /** + * @return the id of the track the stream belongs to, or null if unknown. + */ + public String trackId() { + return trackId; + } + + /** + * @return the codec's MIME type, such as {@code video/VP8}, or null if + * none was negotiated yet. + */ + public String codec() { + return codec; + } + + /** + * @return the frame width in pixels, 0 before the first frame. + */ + public int width() { + return width; + } + + /** + * @return the frame height in pixels, 0 before the first frame. + */ + public int height() { + return height; + } + + /** + * @return the frame rate. + */ + public double framesPerSecond() { + return framesPerSecond; + } + + /** + * @return what limits the quality of a sent stream: {@code none}, {@code + * cpu}, {@code bandwidth} or {@code other}. Null for a received stream. + */ + public String qualityLimitation() { + return qualityLimitation; + } + + @Override + public String toString() { + return "VideoStats[" + codec + ", " + width + "x" + height + ", " + + framesPerSecond + " fps" + (qualityLimitation == null ? "" : ", limited by " + qualityLimitation) + "]"; + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java new file mode 100644 index 00000000..28fce478 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java @@ -0,0 +1,560 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static java.util.Objects.requireNonNull; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioDeviceModuleBase; +import dev.onvoid.webrtc.media.audio.AudioProcessing; +import dev.onvoid.webrtc.media.audio.AudioProcessingConfig; +import dev.onvoid.webrtc.media.audio.HeadlessAudioDeviceModule; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.HardwareVideoEncoderFactory; + +/** + * The entry point of the client API. A runtime owns a {@link + * PeerConnectionFactory}, its audio device module, the threads of this API, + * and every peer and track created through it. Closing it closes them all, in + * the order the native layer needs. + * 
{@code
+ * try (WebRtc rtc = WebRtc.create()) {
+ *     Peer peer = rtc.createPeer(PeerOptions.builder()
+ *             .iceServer("stun:stun.l.google.com:19302")
+ *             .signaling(message -> socket.send(message.toJson()))
+ *             .build());
+ *     socket.onText(text -> peer.signal(SignalMessage.fromJson(text)));
+ *
+ *     peer.addTrack(rtc.media().microphone());
+ *     peer.onTrack(track -> System.out.println("Receiving " + track.kind()));
+ *     ...
+ * }
+ * }
+ *

Threads

+ * Every method of this API may be called from any thread, and none waits for a + * WebRTC thread. The library never runs application code on a thread WebRTC + * needs: what WebRTC reports is copied and queued, then handed to the + * application on the event thread of the peer it belongs to. The handlers of + * one peer run one at a time and in order; those of different peers may run + * at the same time, unless the {@link Builder#eventExecutor(Executor) event + * executor} is serial, such as the Swing event dispatch thread. + */ +public final class WebRtc implements AutoCloseable { + + private static final AtomicInteger RUNTIME_COUNT = new AtomicInteger(); + + final PeerConnectionFactory factory; + + final AudioMode audioMode; + + final Executor eventExecutor; + + final boolean ownsEventExecutor; + + final ExecutorService controlPool; + + final ExecutorService framePool; + + final ScheduledExecutorService timer; + + final Duration operationTimeout; + + private final AudioDeviceModuleBase audioModule; + + private final AudioProcessing audioProcessing; + + private final Duration closeTimeout; + + private final Consumer errorHandler; + + private final LocalMediaImpl media; + + private final Set peers = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + private final AtomicInteger peerCount = new AtomicInteger(); + + private final AtomicBoolean closing = new AtomicBoolean(); + + private final CompletableFuture closed = new CompletableFuture<>(); + + + private WebRtc(Builder builder) { + int runtime = RUNTIME_COUNT.incrementAndGet(); + + this.audioMode = builder.audioMode; + this.operationTimeout = builder.operationTimeout; + this.closeTimeout = builder.closeTimeout; + this.errorHandler = builder.errorHandler != null ? builder.errorHandler : WebRtc::uncaught; + + this.controlPool = Executors.newCachedThreadPool(threads("webrtc-client-control-" + runtime + "-")); + this.framePool = Executors.newCachedThreadPool(threads("webrtc-client-frames-" + runtime + "-")); + this.timer = Executors.newSingleThreadScheduledExecutor(threads("webrtc-client-timer-" + runtime + "-")); + + if (builder.eventExecutor != null) { + this.eventExecutor = builder.eventExecutor; + this.ownsEventExecutor = false; + } + else { + this.eventExecutor = Executors.newCachedThreadPool(threads("webrtc-client-events-" + runtime + "-")); + this.ownsEventExecutor = true; + } + + AudioDeviceModuleBase module = null; + AudioProcessing processing = null; + PeerConnectionFactory created = null; + + try { + module = audioMode == AudioMode.HEADLESS + ? new HeadlessAudioDeviceModule() + : new AudioDeviceModule(); + + if (builder.audioProcessingConfig != null) { + processing = new AudioProcessing(); + processing.applyConfig(builder.audioProcessingConfig); + } + + PeerConnectionFactory.Builder factoryBuilder = PeerConnectionFactory.builder() + .setAudioDeviceModule(module); + + if (processing != null) { + factoryBuilder.setAudioProcessing(processing); + } + if (builder.fieldTrials != null) { + factoryBuilder.setFieldTrials(builder.fieldTrials); + } + if (builder.hardwareVideoCodecs) { + factoryBuilder.setVideoEncoderFactory(new HardwareVideoEncoderFactory()); + factoryBuilder.setVideoDecoderFactory(new HardwareVideoDecoderFactory()); + } + if (builder.factoryCustomizer != null) { + builder.factoryCustomizer.accept(factoryBuilder); + } + + created = factoryBuilder.build(); + + if (audioMode == AudioMode.HEADLESS) { + // Playout drives the receive side of every connection of the + // factory, so it runs even when nothing is received yet. + module.initPlayout(); + module.startPlayout(); + } + } + catch (RuntimeException | Error e) { + if (created != null) { + created.dispose(); + } + if (module != null) { + module.dispose(); + } + if (processing != null) { + processing.dispose(); + } + shutdownPools(); + + throw new ConfigurationException("The WebRTC runtime could not be created: " + e.getMessage(), e); + } + + this.audioModule = module; + this.audioProcessing = processing; + this.factory = created; + this.media = new LocalMediaImpl(this, module); + } + + /** + * Creates a runtime that captures the microphone, plays to the speakers + * and uses the built-in codecs. + * + * @return the runtime. + */ + public static WebRtc create() { + return builder().build(); + } + + /** + * @return a builder for a runtime. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Creates a peer. Unlike the other methods, this one waits for the + * connection to be created, so that an invalid configuration fails here. + * + * @param options how to set up the peer. + * + * @return the peer. + * + * @throws ConfigurationException if WebRTC refuses the configuration. + * @throws PeerClosedException if the runtime is closed. + */ + public Peer createPeer(PeerOptions options) { + requireNonNull(options, "options"); + checkOpen(); + + int number = peerCount.incrementAndGet(); + String name = options.name != null ? options.name : "peer-" + number; + + PeerImpl peer = new PeerImpl(this, options, name); + peers.add(peer); + + try { + peer.open(); + } + catch (RuntimeException e) { + peers.remove(peer); + peer.closeInternal(); + throw e; + } + + if (closing.get()) { + // The runtime began to close meanwhile and may have missed it. + peer.closeInternal(); + } + + return peer; + } + + /** + * @return the source of the tracks this runtime sends. + */ + public LocalMedia media() { + return media; + } + + /** + * @return the peers that are not closed. + */ + public List peers() { + return Collections.unmodifiableList(new ArrayList(peers)); + } + + /** + * @return the audio mode the runtime was built with. + */ + public AudioMode audioMode() { + return audioMode; + } + + /** + * Closes every peer, then every track, then the factory, and stops the + * threads of this API. + * + * @return a future that completes once all is closed. + */ + public CompletableFuture closeAsync() { + if (closing.compareAndSet(false, true)) { + Thread thread = new Thread(this::shutdown, "webrtc-client-close"); + thread.setDaemon(true); + thread.start(); + } + + return closed; + } + + /** + * Closes every peer, then every track, then the factory, and stops the + * threads of this API. Waits for that up to the close timeout, which is + * the one wait in this API. May be called more than once. + */ + @Override + public void close() { + CompletableFuture future = closeAsync(); + + try { + future.get(closeTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Exception e) { + reportError(new WebRtcException("The runtime did not close within " + closeTimeout, e)); + } + } + + void reportError(Throwable error) { + try { + errorHandler.accept(error); + } + catch (Throwable e) { + uncaught(e); + } + } + + void removePeer(PeerImpl peer) { + peers.remove(peer); + } + + void checkOpen() { + if (closing.get()) { + throw new PeerClosedException("The runtime is closed"); + } + } + + boolean isClosing() { + return closing.get(); + } + + private void shutdown() { + try { + List> peerCloses = new ArrayList<>(); + + for (PeerImpl peer : new ArrayList<>(peers)) { + peerCloses.add(peer.closeInternal()); + } + + await(peerCloses, "peers"); + await(media.closeAll(), "local tracks"); + + media.dispose(); + + try { + if (audioMode == AudioMode.HEADLESS) { + audioModule.stopPlayout(); + } + } + catch (Throwable e) { + // The module is disposed next anyway. + } + + audioModule.dispose(); + factory.dispose(); + + if (audioProcessing != null) { + audioProcessing.dispose(); + } + } + catch (Throwable e) { + reportError(e); + } + finally { + shutdownPools(); + closed.complete(null); + } + } + + private void await(List> futures, String what) { + try { + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .get(closeTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Exception e) { + reportError(new WebRtcException("Closing the " + what + " did not finish within " + closeTimeout, e)); + } + } + + private void shutdownPools() { + controlPool.shutdown(); + framePool.shutdown(); + timer.shutdownNow(); + + if (ownsEventExecutor) { + ((ExecutorService) eventExecutor).shutdown(); + } + } + + private static ThreadFactory threads(String prefix) { + AtomicInteger count = new AtomicInteger(); + + return task -> { + Thread thread = new Thread(task, prefix + count.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + private static void uncaught(Throwable error) { + Thread thread = Thread.currentThread(); + thread.getUncaughtExceptionHandler().uncaughtException(thread, error); + } + + + + /** Builds a {@link WebRtc} runtime. */ + public static final class Builder { + + private AudioMode audioMode = AudioMode.DEVICES; + + private AudioProcessingConfig audioProcessingConfig; + + private boolean hardwareVideoCodecs; + + private Map fieldTrials; + + private Executor eventExecutor; + + private Consumer errorHandler; + + private Duration operationTimeout = Duration.ofSeconds(30); + + private Duration closeTimeout = Duration.ofSeconds(10); + + private Consumer factoryCustomizer; + + + private Builder() { + } + + /** + * @param mode where sent audio comes from; {@link AudioMode#DEVICES} + * by default. + * + * @return this builder. + */ + public Builder audio(AudioMode mode) { + this.audioMode = requireNonNull(mode, "mode"); + return this; + } + + /** + * @param config the audio processing applied to captured audio. + * + * @return this builder. + */ + public Builder audioProcessing(AudioProcessingConfig config) { + this.audioProcessingConfig = requireNonNull(config, "config"); + return this; + } + + /** + * @param enabled true to encode and decode video on the GPU where it + * can, and in software otherwise. + * + * @return this builder. + */ + public Builder hardwareVideoCodecs(boolean enabled) { + this.hardwareVideoCodecs = enabled; + return this; + } + + /** + * @param fieldTrials WebRTC field trials, by name. + * + * @return this builder. + */ + public Builder fieldTrials(Map fieldTrials) { + this.fieldTrials = requireNonNull(fieldTrials, "fieldTrials"); + return this; + } + + /** + * Sets where handlers and future callbacks run, such as {@code + * SwingUtilities::invokeLater} or {@code Platform::runLater}. The + * order of each peer's events is kept whatever the executor. By + * default the runtime runs them on threads of its own. + * + * @param executor runs handlers. + * + * @return this builder. + */ + public Builder eventExecutor(Executor executor) { + this.eventExecutor = requireNonNull(executor, "executor"); + return this; + } + + /** + * Sets what receives the exceptions handlers throw and the failures + * nobody else hears of. By default they go to the uncaught exception + * handler of the thread they happen on. + * + * @param handler receives the errors. It must not throw. + * + * @return this builder. + */ + public Builder errorHandler(Consumer handler) { + this.errorHandler = requireNonNull(handler, "handler"); + return this; + } + + /** + * @param timeout how long an operation, such as applying a + * description, may take before it fails with a {@code + * TimeoutException}; 30 seconds by default. + * + * @return this builder. + */ + public Builder operationTimeout(Duration timeout) { + this.operationTimeout = positive(timeout); + return this; + } + + /** + * @param timeout how long {@link WebRtc#close()} waits; 10 seconds by + * default. + * + * @return this builder. + */ + public Builder closeTimeout(Duration timeout) { + this.closeTimeout = positive(timeout); + return this; + } + + /** + * Changes anything else of the factory, after this builder applied its + * own values. The audio device module must be left as it is. + * + * @param customizer changes the factory builder. + * + * @return this builder. + */ + public Builder configureFactory(Consumer customizer) { + this.factoryCustomizer = requireNonNull(customizer, "customizer"); + return this; + } + + /** + * @return the runtime. + * + * @throws ConfigurationException if the factory cannot be created. + */ + public WebRtc build() { + return new WebRtc(this); + } + + private static Duration positive(Duration duration) { + requireNonNull(duration, "duration"); + + if (duration.isNegative() || duration.isZero()) { + throw new IllegalArgumentException("The duration must be positive, got " + duration); + } + + return duration; + } + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtcException.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtcException.java new file mode 100644 index 00000000..6195e32d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtcException.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +/** + * The base of all failures the client API reports, either thrown or as the cause of a failed future. + */ +public class WebRtcException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /** + * Creates the exception with a message that says what went wrong. + * + * @param message the detail message. + */ + public WebRtcException(String message) { + super(message); + } + + /** + * Creates the exception with a message and the failure that caused it. + * + * @param message the detail message. + * @param cause the underlying failure. + */ + public WebRtcException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java index 2be00842..77a60a8e 100644 --- a/webrtc/src/main/java/module-info.java +++ b/webrtc/src/main/java/module-info.java @@ -6,6 +6,7 @@ requires java.desktop; exports dev.onvoid.webrtc; + exports dev.onvoid.webrtc.client; exports dev.onvoid.webrtc.logging; exports dev.onvoid.webrtc.media; exports dev.onvoid.webrtc.media.audio; diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java new file mode 100644 index 00000000..0580e3a0 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java @@ -0,0 +1,86 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Gives each test a headless runtime of its own, which needs no audio + * hardware, and collects what reaches its error handler. + */ +@Execution(ExecutionMode.SAME_THREAD) +abstract class ClientTestBase { + + static final long TIMEOUT_SECONDS = 15; + + final List errors = new CopyOnWriteArrayList<>(); + + WebRtc rtc; + + + @BeforeEach + void createRuntime() { + rtc = WebRtc.builder() + .audio(AudioMode.HEADLESS) + .errorHandler(errors::add) + .build(); + } + + @AfterEach + void closeRuntime() { + rtc.close(); + + // Includes what closing reported, such as native objects that did + // not go away. + assertTrue(errors.isEmpty(), () -> "Errors: " + errors); + } + + /** + * Creates two peers linked by a loopback signaling. The second one is + * polite. + */ + Peer[] pair() { + SignalingPair link = Signaling.loopback(); + + Peer caller = rtc.createPeer(PeerOptions.builder() + .name("caller") + .signaling(link.first()) + .build()); + Peer callee = rtc.createPeer(PeerOptions.builder() + .name("callee") + .signaling(link.second()) + .polite(true) + .build()); + + return new Peer[] { caller, callee }; + } + + static T await(CompletableFuture future) throws Exception { + return future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java new file mode 100644 index 00000000..aa5d3441 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java @@ -0,0 +1,227 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MediaTests extends ClientTestBase { + + private ScheduledExecutorService feeder; + + + @BeforeEach + void createFeeder() { + feeder = Executors.newSingleThreadScheduledExecutor(); + } + + @AfterEach + void stopFeeder() throws InterruptedException { + feeder.shutdownNow(); + feeder.awaitTermination(5, TimeUnit.SECONDS); + } + + @Test + void pushedAudioArrives() throws Exception { + Peer[] peers = pair(); + CustomAudio voice = rtc.media().customAudio("voice"); + CompletableFuture heard = new CompletableFuture<>(); + + peers[1].onTrack(track -> track.asAudio().onAudio(heard::complete)); + peers[0].addTrack(voice); + + // One second in one go; the track sends it in 10 ms chunks. + voice.push(tone(48000, 1, 1000), PcmFormat.of(48000, 1)); + assertTrue(voice.queuedMillis() > 500); + + AudioChunk chunk = await(heard); + + assertTrue(chunk.frames() > 0); + assertEquals(chunk.frames() * chunk.channels() * 2, chunk.samples().length); + } + + @Test + void pushedVideoArrives() throws Exception { + Peer[] peers = pair(); + CustomVideo video = rtc.media().customVideo("video"); + CompletableFuture seen = new CompletableFuture<>(); + CompletableFuture preview = new CompletableFuture<>(); + + peers[1].onTrack(track -> track.asVideo().onFrame(frame -> + seen.complete(new int[] { frame.buffer.getWidth(), frame.buffer.getHeight() }))); + video.onFrame(frame -> preview.complete(new int[] { frame.buffer.getWidth(), frame.buffer.getHeight() })); + peers[0].addTrack(video); + + feedVideo(video, 320, 240); + + assertEquals(320, await(seen)[0]); + assertEquals(240, await(preview)[1]); + } + + @Test + void slowFrameHandlerGetsNewest() throws Exception { + Peer[] peers = pair(); + CustomVideo video = rtc.media().customVideo("video"); + AtomicInteger handled = new AtomicInteger(); + CompletableFuture enough = new CompletableFuture<>(); + + peers[1].onTrack(track -> track.asVideo().onFrame(frame -> { + try { + // Much slower than the 30 frames per second sent. + Thread.sleep(200); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (handled.incrementAndGet() == 3) { + enough.complete(null); + } + })); + peers[0].addTrack(video); + + feedVideo(video, 160, 120); + + await(enough); + + // The connection kept up while the handler fell behind. + PeerStats stats = await(peers[1].stats()); + assertTrue(stats.inboundVideo().get(0).framesPerSecond() > 10, + () -> "Inbound video: " + stats.inboundVideo()); + } + + @Test + void removedSenderEndsRemoteTrack() throws Exception { + Peer[] peers = pair(); + CustomVideo video = rtc.media().customVideo("video"); + CompletableFuture remote = new CompletableFuture<>(); + CompletableFuture ended = new CompletableFuture<>(); + + peers[1].onTrack(track -> { + remote.complete(track); + track.onEnded(() -> ended.complete(null)); + }); + + Sender sender = peers[0].addTrack(video); + RemoteTrack track = await(remote); + + assertEquals(MediaKind.VIDEO, track.kind()); + assertEquals(1, track.streamIds().size()); + + await(sender.remove()); + await(ended); + + assertTrue(track.isEnded()); + assertTrue(peers[0].senders().isEmpty()); + assertTrue(peers[1].remoteTracks().isEmpty()); + // The track was only removed, not closed. + assertTrue(!video.isEnded()); + } + + @Test + void closedTrackLeavesEveryPeer() throws Exception { + Peer[] first = pair(); + Peer[] second = pair(); + CustomAudio voice = rtc.media().customAudio("voice"); + CompletableFuture ended = new CompletableFuture<>(); + + Sender one = first[0].addTrack(voice); + Sender two = second[0].addTrack(voice); + voice.onEnded(() -> ended.complete(null)); + await(first[0].whenConnected()); + await(second[0].whenConnected()); + + voice.close(); + await(ended); + + assertNull(one.track()); + assertNull(two.track()); + assertTrue(voice.isEnded()); + + // The senders stay and may send another track. + CustomAudio other = rtc.media().customAudio("other"); + await(one.replaceTrack(other)); + assertEquals(other, one.track()); + } + + @Test + void senderConfigured() throws Exception { + Peer[] peers = pair(); + Sender sender = peers[0].addTrack(rtc.media().customVideo("video")); + await(peers[0].whenConnected()); + + await(sender.configure(Encoding.builder().maxBitrate(300_000).maxFrameRate(15).build())); + + assertThrows(IllegalArgumentException.class, () -> sender.replaceTrack(rtc.media().customAudio("audio"))); + } + + @Test + void audioModeChecked() { + assertThrows(ConfigurationException.class, () -> rtc.media().microphone()); + } + + private void feedVideo(CustomVideo video, int width, int height) { + AtomicInteger shade = new AtomicInteger(); + + feeder.scheduleAtFixedRate(() -> { + NativeI420Buffer buffer = NativeI420Buffer.allocate(width, height); + ByteBuffer y = buffer.getDataY(); + int offset = shade.addAndGet(3); + + for (int i = 0; i < y.capacity(); i++) { + y.put(i, (byte) (i + offset)); + } + + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + video.push(frame); + frame.release(); + }, 0, 33, TimeUnit.MILLISECONDS); + } + + private static byte[] tone(int sampleRate, int channels, int millis) { + int frames = sampleRate * millis / 1000; + byte[] pcm = new byte[frames * channels * 2]; + + for (int i = 0; i < frames; i++) { + short sample = (short) (Math.sin(2 * Math.PI * 440 * i / sampleRate) * 8000); + + for (int c = 0; c < channels; c++) { + int index = (i * channels + c) * 2; + pcm[index] = (byte) sample; + pcm[index + 1] = (byte) (sample >> 8); + } + } + + return pcm; + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java new file mode 100644 index 00000000..08bd3863 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java @@ -0,0 +1,364 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import dev.onvoid.webrtc.RTCSessionDescription; + +import org.junit.jupiter.api.Test; + +class PeerTests extends ClientTestBase { + + @Test + void channelRoundTrip() throws Exception { + Peer[] peers = pair(); + CompletableFuture reply = new CompletableFuture<>(); + + peers[1].onChannel(channel -> channel.onMessage(message -> channel.send("pong:" + message.text()))); + + DataChannel channel = peers[0].createChannel("chat"); + channel.onMessage(message -> reply.complete(message.text())); + channel.send("ping"); + + assertEquals("pong:ping", await(reply)); + assertEquals(PeerState.CONNECTED, await(peers[0].whenConnected()).state()); + assertEquals(PeerState.CONNECTED, await(peers[1].whenConnected()).state()); + assertTrue(errors.isEmpty(), () -> "Errors: " + errors); + } + + @Test + void stateHandlerGetsCurrentState() throws Exception { + Peer[] peers = pair(); + peers[0].createChannel("chat"); + await(peers[0].whenConnected()); + + // Registered after the fact, the handler still hears the state. + CompletableFuture state = new CompletableFuture<>(); + peers[0].onStateChange(state::complete); + + assertEquals(PeerState.CONNECTED, await(state)); + } + + @Test + void earlyMessagesKept() throws Exception { + Peer[] peers = pair(); + CompletableFuture remote = new CompletableFuture<>(); + peers[1].onChannel(remote::complete); + + DataChannel channel = peers[0].createChannel("chat"); + await(channel.send("one")); + await(channel.send("two")); + + // No message handler while the messages arrive. + DataChannel received = await(remote); + Thread.sleep(300); + + List messages = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch both = new CountDownLatch(2); + received.onMessage(message -> { + messages.add(message.text()); + both.countDown(); + }); + + assertTrue(both.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(Arrays.asList("one", "two"), messages); + } + + @Test + void eventsInOrderOneAtATime() throws Exception { + Peer[] peers = pair(); + int count = 2000; + List received = new ArrayList<>(); + AtomicInteger running = new AtomicInteger(); + AtomicInteger overlaps = new AtomicInteger(); + CountDownLatch all = new CountDownLatch(count); + + peers[1].onChannel(channel -> channel.onMessage(message -> { + if (running.incrementAndGet() > 1) { + overlaps.incrementAndGet(); + } + received.add(Integer.parseInt(message.text())); + running.decrementAndGet(); + all.countDown(); + })); + + DataChannel channel = peers[0].createChannel("numbers"); + + for (int i = 0; i < count; i++) { + channel.send(Integer.toString(i)); + } + + assertTrue(all.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(0, overlaps.get()); + + for (int i = 0; i < count; i++) { + assertEquals(i, received.get(i)); + } + } + + @Test + void handlerMayClosePeer() throws Exception { + Peer[] peers = pair(); + + peers[0].onStateChange(state -> { + if (state == PeerState.CONNECTED) { + peers[0].close(); + } + }); + peers[0].createChannel("chat"); + + await(peers[0].whenClosed()); + + assertEquals(PeerState.CLOSED, peers[0].state()); + assertFalse(rtc.peers().contains(peers[0])); + } + + @Test + void joinOnEventThreadRefused() throws Exception { + Peer[] peers = pair(); + CompletableFuture refused = new CompletableFuture<>(); + + peers[0].onStateChange(state -> { + try { + peers[0].whenClosed().join(); + refused.complete(null); + } + catch (Throwable e) { + refused.complete(e); + } + }); + + assertInstanceOf(BlockingCallException.class, await(refused)); + + // A future that is done may be read anywhere. + CompletableFuture doneRead = new CompletableFuture<>(); + peers[0].createChannel("chat"); + peers[0].whenConnected().thenAccept(peer -> doneRead.complete(peer.whenConnected().join())); + + assertEquals(peers[0], await(doneRead)); + } + + @Test + void blockedHandlerDelaysOnlyItsPeer() throws Exception { + Peer[] slow = pair(); + Peer[] fast = pair(); + CountDownLatch release = new CountDownLatch(1); + + slow[0].onStateChange(state -> { + try { + // Holds the event thread of this peer only. + release.await(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + slow[0].createChannel("chat"); + + CompletableFuture reply = new CompletableFuture<>(); + fast[1].onChannel(channel -> channel.onMessage(message -> reply.complete(message.text()))); + fast[0].createChannel("chat").send("through"); + + try { + assertEquals("through", await(reply)); + + // The blocked peer still connects underneath; only its events wait. + assertFalse(slow[0].whenConnected().isDone()); + } + finally { + release.countDown(); + } + + await(slow[0].whenConnected()); + } + + @Test + void politePeerResolvesCollision() throws Exception { + Peer[] peers = pair(); + CompletableFuture atCaller = new CompletableFuture<>(); + CompletableFuture atCallee = new CompletableFuture<>(); + + // Both create a channel at once, so both send an offer at once. + DataChannel fromCaller = peers[0].createChannel("from-caller"); + DataChannel fromCallee = peers[1].createChannel("from-callee"); + + peers[0].onChannel(channel -> channel.onMessage(message -> atCaller.complete(message.text()))); + peers[1].onChannel(channel -> channel.onMessage(message -> atCallee.complete(message.text()))); + + fromCaller.send("hello callee"); + fromCallee.send("hello caller"); + + assertEquals("hello callee", await(atCallee)); + assertEquals("hello caller", await(atCaller)); + assertTrue(errors.isEmpty(), () -> "Errors: " + errors); + } + + @Test + void impoliteCollisionReported() throws Exception { + SignalingPair link = Signaling.loopback(); + Peer first = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); + Peer second = rtc.createPeer(PeerOptions.builder().signaling(link.second()).build()); + + CompletableFuture error = new CompletableFuture<>(); + first.onError(error::complete); + second.onError(error::complete); + + first.createChannel("a"); + second.createChannel("b"); + + WebRtcException reported = error.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertInstanceOf(NegotiationException.class, reported); + assertTrue(reported.getMessage().contains("polite(true)"), reported.getMessage()); + } + + @Test + void manualNegotiation() throws Exception { + Peer offerer = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build()); + Peer answerer = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build()); + + CompletableFuture received = new CompletableFuture<>(); + answerer.onChannel(channel -> channel.onMessage(message -> received.complete(message.text()))); + offerer.createChannel("manual").send("no trickle"); + + // Without a signaling, the descriptions carry all candidates. + RTCSessionDescription offer = await(offerer.negotiator().createOffer(Gathering.COMPLETE)); + RTCSessionDescription answer = await(answerer.negotiator().answer(offer, Gathering.COMPLETE)); + await(offerer.negotiator().applyAnswer(answer)); + + assertTrue(offer.sdp.contains("a=candidate"), "The offer has no candidates"); + assertEquals("no trickle", await(received)); + assertThrows(IllegalStateException.class, () -> pair()[0].negotiator()); + } + + @Test + void closeFailsPendingWork() throws Exception { + Peer peer = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build()); + DataChannel channel = peer.createChannel("never-opened"); + + // Held, since the channel never opens. + CompletableFuture sent = channel.send("lost"); + CompletableFuture connected = peer.whenConnected(); + + peer.close(); + await(peer.whenClosed()); + + assertInstanceOf(PeerClosedException.class, cause(sent)); + assertInstanceOf(PeerClosedException.class, cause(connected)); + assertInstanceOf(PeerClosedException.class, cause(peer.negotiator().createOffer())); + assertInstanceOf(PeerClosedException.class, cause(channel.send("after"))); + assertEquals(ChannelState.CLOSED, channel.state()); + } + + @Test + void runtimeCloseClosesPeers() throws Exception { + Peer[] peers = pair(); + peers[0].createChannel("chat"); + await(peers[0].whenConnected()); + + rtc.close(); + + assertTrue(peers[0].whenClosed().isDone()); + assertTrue(peers[1].whenClosed().isDone()); + assertEquals(PeerState.CLOSED, peers[1].state()); + assertThrows(PeerClosedException.class, this::pair); + } + + @Test + void statsAfterConnecting() throws Exception { + Peer[] peers = pair(); + DataChannel channel = peers[0].createChannel("chat"); + await(channel.whenOpen()); + await(channel.send(new byte[10_000])); + + PeerStats first = await(peers[0].stats()); + PeerStats second = await(peers[0].stats()); + + assertEquals("host", first.candidateType()); + assertTrue(first.bytesSent() > 10_000, () -> "Bytes sent: " + first.bytesSent()); + assertEquals(-1, first.outgoingBitrate()); + assertTrue(second.outgoingBitrate() >= 0); + } + + @Test + void subscriptionStopsHandler() throws Exception { + Peer[] peers = pair(); + List messages = Collections.synchronizedList(new ArrayList<>()); + CompletableFuture remote = new CompletableFuture<>(); + peers[1].onChannel(remote::complete); + + DataChannel channel = peers[0].createChannel("chat"); + DataChannel received = await(remote); + + CompletableFuture first = new CompletableFuture<>(); + Subscription subscription = received.onMessage(message -> { + messages.add(message.text()); + first.complete(null); + }); + CompletableFuture last = new CompletableFuture<>(); + received.onMessage(message -> { + if (message.text().equals("two")) { + last.complete(null); + } + }); + + channel.send("one"); + await(first); + subscription.close(); + channel.send("two"); + await(last); + + assertEquals(Arrays.asList("one"), messages); + } + + @Test + void failedAttachClosesPeer() { + SignalingPair link = Signaling.loopback(); + rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); + + // One end of a loopback cannot serve two peers. + assertThrows(IllegalStateException.class, + () -> rtc.createPeer(PeerOptions.builder().signaling(link.first()).build())); + assertEquals(1, rtc.peers().size()); + } + + private static Throwable cause(CompletableFuture future) throws Exception { + try { + future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + return null; + } + catch (ExecutionException e) { + return e.getCause(); + } + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/SignalMessageTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/SignalMessageTests.java new file mode 100644 index 00000000..f5641529 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/SignalMessageTests.java @@ -0,0 +1,85 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCSdpType; +import dev.onvoid.webrtc.RTCSessionDescription; + +import org.junit.jupiter.api.Test; + +class SignalMessageTests { + + private static final String SDP = "v=0\r\no=- 4611731400430051336 2 IN IP4 127.0.0.1\r\ns=\"quoted\" \\ \u0001\r\n"; + + @Test + void descriptionRoundTrip() { + SignalMessage offer = SignalMessage.of(new RTCSessionDescription(RTCSdpType.OFFER, SDP)); + SignalMessage read = SignalMessage.fromJson(offer.toJson()); + + assertEquals(SignalMessage.Type.OFFER, read.type()); + assertEquals(SDP, read.description().sdp); + assertEquals(RTCSdpType.OFFER, read.description().sdpType); + assertEquals(offer, read); + } + + @Test + void candidateRoundTrip() { + RTCIceCandidate candidate = new RTCIceCandidate("0", 1, + "candidate:842163049 1 udp 1677729535 203.0.113.7 49203 typ srflx raddr 0.0.0.0 rport 0"); + SignalMessage read = SignalMessage.fromJson(SignalMessage.of(candidate).toJson()); + + assertEquals(SignalMessage.Type.CANDIDATE, read.type()); + assertEquals("0", read.candidate().sdpMid); + assertEquals(1, read.candidate().sdpMLineIndex); + assertEquals(candidate.sdp, read.candidate().sdp); + } + + @Test + void browserShapes() { + // What a browser's JSON.stringify writes for the perfect negotiation examples. + SignalMessage description = SignalMessage.fromJson( + "{ \"description\": { \"type\": \"answer\", \"sdp\": \"v=0\\r\\n\" } }"); + SignalMessage candidate = SignalMessage.fromJson("{\"candidate\":{\"candidate\":\"candidate:1 1 udp 1 10.0.0.1 9 typ host\"," + + "\"sdpMid\":\"1\",\"sdpMLineIndex\":null,\"usernameFragment\":\"abcd\"}}"); + + assertEquals(SignalMessage.Type.ANSWER, description.type()); + assertEquals("v=0\r\n", description.description().sdp); + assertEquals("1", candidate.candidate().sdpMid); + assertEquals(SignalMessage.Type.END_OF_CANDIDATES, SignalMessage.fromJson("{\"candidate\":null}").type()); + assertEquals(SignalMessage.Type.END_OF_CANDIDATES, + SignalMessage.fromJson("{\"candidate\":\"\",\"sdpMid\":\"0\",\"sdpMLineIndex\":0}").type()); + assertEquals("{\"candidate\":\"\"}", SignalMessage.endOfCandidates().toJson()); + assertNull(SignalMessage.endOfCandidates().candidate()); + } + + @Test + void invalidMessagesRejected() { + assertThrows(IllegalArgumentException.class, () -> SignalMessage.fromJson("[]")); + assertThrows(IllegalArgumentException.class, () -> SignalMessage.fromJson("{\"type\":\"offer\"")); + assertThrows(IllegalArgumentException.class, () -> SignalMessage.fromJson("{\"type\":\"rollback\",\"sdp\":\"\"}")); + assertThrows(IllegalArgumentException.class, () -> SignalMessage.fromJson("{\"candidate\":\"candidate:1\"}")); + assertThrows(IllegalArgumentException.class, () -> SignalMessage.fromJson("{\"hello\":1}")); + assertThrows(IllegalArgumentException.class, + () -> SignalMessage.of(new RTCSessionDescription(RTCSdpType.ROLLBACK, ""))); + } + +} From a3e4a006adca01065e303ae3072dcbacc7b14fdd Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 00:49:55 +0200 Subject: [PATCH 02/25] Add client API guide and example The guide covers the runtime, signaling and perfect negotiation, threads, media, data channels, manual negotiation, stats and closing. The example connects two peers in one process without hardware or a server. --- docs/.vitepress/sidebar.ts | 1 + docs/guide/client-api.md | 522 ++++++++++++++++++ docs/guide/examples.md | 20 + docs/guide/get-started.md | 4 + .../webrtc/examples/ClientApiExample.java | 241 ++++++++ .../dev/onvoid/webrtc/client/PeerTests.java | 14 + 6 files changed, 802 insertions(+) create mode 100644 docs/guide/client-api.md create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index a4d3ab08..ed67587c 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -13,6 +13,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { items: [ { text: 'What is webrtc-java?', link: '/introduction' }, { text: 'Getting Started', link: '/get-started' }, + { text: 'Client API', link: '/client-api' }, ], }, { diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md new file mode 100644 index 00000000..6b13bad4 --- /dev/null +++ b/docs/guide/client-api.md @@ -0,0 +1,522 @@ +# Client API + +The client API in `dev.onvoid.webrtc.client` is a layer over the rest of the library that takes care of the parts most applications get wrong: threads, the offer and answer exchange, and the lifetime of native objects. A whole video call fits on one screen: + +```java +import dev.onvoid.webrtc.client.*; + +try (WebRtc rtc = WebRtc.create()) { + Peer peer = rtc.createPeer(PeerOptions.builder() + .iceServer("stun:stun.l.google.com:19302") + .polite(joinedSecond) + .signaling(message -> socket.send(message.toJson())) + .build()); + socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); + + peer.addTrack(rtc.media().microphone()); + peer.addTrack(rtc.media().camera()); + + peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(frame -> view.render(frame)); + } + }); + peer.onStateChange(state -> System.out.println("Call is " + state)); + + peer.whenConnected().join(); + hangUp.await(); +} +``` + +Closing the `WebRtc` runtime at the end closes the peer, the camera, the microphone and the factory, in that order. There is no observer to implement and no `dispose()` to call. + +What the client API takes care of: +- **Threads**: your handlers never run on a thread WebRTC needs, so they may block or close a peer without stalling or deadlocking anything. The handlers of one peer run one at a time, in order. +- **Negotiation**: peers negotiate by themselves whenever tracks or channels change, with [perfect negotiation](https://w3c.github.io/webrtc-pc/#perfect-negotiation-example). Offer collisions are resolved, and candidates that arrive before their description are held. +- **Lifetime**: everything belongs to the runtime that created it and is closed with it. Senders, receivers and transceivers stay inside the library. +- **Futures**: operations return `CompletableFuture`s that fail with typed exceptions, instead of taking observers. + +The client API is plain Java on top of the classes the other guides describe, and it does not replace them. Its peers, tracks and channels are the same native objects, and [Using the Low-Level API](#using-the-low-level-api) shows how to reach them for anything it does not cover. + +## The Runtime + +A `WebRtc` runtime owns a `PeerConnectionFactory`, its audio device module and the threads of the client API. `WebRtc.create()` captures the microphone and plays to the speakers. The builder sets everything else: + +```java +WebRtc rtc = WebRtc.builder() + .audio(AudioMode.HEADLESS) + .hardwareVideoCodecs(true) + .eventExecutor(SwingUtilities::invokeLater) + .errorHandler(error -> log.error("WebRTC", error)) + .operationTimeout(Duration.ofSeconds(30)) + .closeTimeout(Duration.ofSeconds(10)) + .build(); +``` + +| Method | Default | What it sets | +|---|---|---| +| `audio(AudioMode)` | `DEVICES` | Where sent audio comes from, see below. | +| `audioProcessing(AudioProcessingConfig)` | none | The [audio processing](/guide/audio/audio-processing) of captured audio. | +| `hardwareVideoCodecs(boolean)` | `false` | Encodes and decodes video on the GPU where it can, as the [Video Codecs](/guide/advanced/video-codecs) guide describes. | +| `fieldTrials(Map)` | none | [Field trials](/guide/advanced/field-trials). | +| `eventExecutor(Executor)` | threads of the runtime | Where handlers run, see [Threads](#threads). | +| `errorHandler(Consumer)` | the thread's uncaught exception handler | Receives what handlers throw, and failures nobody else hears of. | +| `operationTimeout(Duration)` | 30 seconds | How long an operation such as applying a description may take before it fails with a `TimeoutException`. | +| `closeTimeout(Duration)` | 10 seconds | How long `close()` waits. | +| `configureFactory(Consumer)` | none | Anything else of the factory, such as codec factories of your own. | + +### Audio Modes + +WebRTC feeds the audio a factory captures into every audio sender of that factory, so captured audio and audio you push cannot be mixed in one runtime. The runtime's `AudioMode` picks one of them up front: + +| Mode | Sends | Plays remote audio | Use it for | +|---|---|---|---| +| `DEVICES` | the microphone | to the speakers | calls on a desktop | +| `CUSTOM` | audio pushed through `CustomAudio` | to the speakers | applications that generate or relay audio | +| `HEADLESS` | audio pushed through `CustomAudio` | to `RemoteAudio.onAudio` handlers only | servers, bots and tests, without any audio hardware | + +Asking a runtime for the other kind fails at once with a `ConfigurationException` that names the mode, rather than when the first track is added. Two runtimes can run side by side if an application needs both kinds. + +## Peers and Signaling + +`rtc.createPeer(options)` creates a connection to one remote peer. It is the one call of the client API that waits, briefly, for the connection to be created, so that a configuration WebRTC refuses fails right there with a `ConfigurationException`. + +```java +Peer peer = rtc.createPeer(PeerOptions.builder() + .name("alice") + .iceServer("stun:stun.l.google.com:19302") + .iceServer("turn:turn.example.com:3478", "user", "secret") + .polite(true) + .signaling(message -> socket.send(message.toJson())) + .build()); +``` + +| Method | What it sets | +|---|---| +| `name(String)` | The name in thread names and error messages. | +| `iceServer(String...)` | A STUN or TURN server without credentials. | +| `iceServer(String, String, String)` | A TURN server with a user name and password. | +| `relayOnly(boolean)` | Connects through TURN servers only, which hides this side's addresses. | +| `signaling(Signaling)` | What carries messages to the remote peer. Required unless negotiation is manual. | +| `polite(boolean)` | The role in perfect negotiation, see below. | +| `negotiation(Negotiation)` | `AUTOMATIC` by default, or `MANUAL`, see [Manual Negotiation](#manual-negotiation). | +| `receive(MediaKind...)` | Receives audio or video without sending any, as a viewer does. | +| `configure(Consumer)` | Anything else the [peer connection configuration](/guide/networking/peer-connection-config) has. | + +### Signaling + +The library has no transport of its own. A `Signaling` sends a peer's messages to the remote peer, and messages from the remote peer are handed to `peer.signal(...)`, in the order they were sent: + +```java +// Outgoing: called on the peer's event thread, in order. +Signaling signaling = message -> socket.send(message.toJson()); + +// Incoming: from any thread. +socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); +``` + +A transport class can implement both directions by overriding `Signaling.attach(Consumer)`, which the peer calls once when it is created, with the consumer to push received messages into. + +A `SignalMessage` is a session description (`OFFER`, `ANSWER`) or an ICE candidate (`CANDIDATE`). `toJson()` writes the shapes the browser API uses, so a JavaScript peer can pass them to `setRemoteDescription` and `addIceCandidate` as they are: + +```json +{"type":"offer","sdp":"v=0\r\no=- 4611731400430051336 2 IN IP4 127.0.0.1\r\n..."} +{"candidate":"candidate:842163049 1 udp 1677729535 203.0.113.7 49203 typ srflx raddr 0.0.0.0 rport 0","sdpMid":"0","sdpMLineIndex":0} +``` + +`SignalMessage.fromJson(String)` reads these, what a browser's `JSON.stringify(pc.localDescription)` and `JSON.stringify(candidate)` write, and the `{"description": ...}` and `{"candidate": {...}}` envelopes of the perfect negotiation examples. Routing, such as room names or peer ids, belongs in an envelope of your own around this JSON. + +A browser peer that only answers needs no more than this: + +```js +socket.onmessage = async ({ data }) => { + const message = JSON.parse(data); + + if (message.type) { + await pc.setRemoteDescription(message); + + if (message.type === "offer") { + await pc.setLocalDescription(); + socket.send(JSON.stringify(pc.localDescription)); + } + } + else if (message.candidate) { + await pc.addIceCandidate(message); + } +}; + +pc.onicecandidate = ({ candidate }) => candidate && socket.send(JSON.stringify(candidate)); +``` + +A browser peer that adds tracks of its own sends offers too, and then has to implement perfect negotiation as well, as the W3C example linked above does. + +For tests and demos, `Signaling.loopback()` links two peers in the same process: + +```java +SignalingPair link = Signaling.loopback(); + +Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); +Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); +``` + +### Polite and Impolite Peers + +With automatic negotiation, a peer sends an offer whenever its tracks or channels change. When both peers send one at the same moment, the **polite** peer drops its own offer and answers the other one, and the impolite peer ignores the offer it got. Exactly one of two peers must be polite. Peers are impolite by default, so set `polite(true)` on one side, for example on the peer that joined second. + +When two impolite peers collide, neither gives way. An impolite peer that waited 5 seconds for an answer reports a `NegotiationException` that says to make one of them polite. + +### Peer State + +`peer.state()` returns the last state the connection reported: + +| State | Meaning | +|---|---| +| `NEW` | Created, nothing negotiated yet. | +| `CONNECTING` | ICE and DTLS are establishing the connection. | +| `CONNECTED` | Media and data can flow. | +| `DISCONNECTED` | Connectivity was lost. It may come back by itself. | +| `FAILED` | The connection failed. `peer.restartIce()` tries to recover it. | +| `CLOSED` | The peer was closed. This state is final. | + +```java +peer.onStateChange(state -> statusLabel.setText(state.toString())); + +peer.whenConnected().thenAccept(p -> System.out.println("Connected")); +peer.whenClosed().thenRun(() -> System.out.println("Closed")); +``` + +`onStateChange` calls a new handler with the current state first, so a handler added late still learns where the peer stands. `whenConnected()` fails if the connection fails or closes first. + +## Threads + +Every method of the client API may be called from any thread, including from inside a handler. None of them waits for a WebRTC thread: methods that change a peer return at once and take effect in the order they were called. + +| Code | Runs on | May it block? | +|---|---|---| +| Your calls into the API | your thread | They don't block. The exceptions are `WebRtc.createPeer`, which waits briefly for the connection, and `WebRtc.close()`, which waits up to the close timeout. | +| Peer, track and channel handlers, future callbacks, `Signaling.send` | the peer's event thread | Yes, but that delays only the events of this peer. They must not wait for a future of this API, see below. | +| `onFrame` and `onAudio` handlers | a thread of each subscription | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped. | +| `LocalTrack.onEnded` and `Devices.onChange` handlers | a background thread of the runtime | Briefly. | +| WebRTC's own callbacks | WebRTC's threads | Not your code. The library only copies what they report and queues it. | + +The handlers of one peer never run at the same time, and they run in the order of the events. The handlers of different peers may run at the same time. With the default executor, the event thread is named after its peer, such as `webrtc-client-events-alice`, which shows up in thread dumps. + +### Handlers on the UI Thread + +Pass the UI toolkit's executor, and every handler may touch UI components directly: + +```java +WebRtc rtc = WebRtc.builder() + .eventExecutor(SwingUtilities::invokeLater) // or Platform::runLater for JavaFX + .build(); + +peer.onStateChange(state -> statusLabel.setText(state.toString())); +hangUpButton.addActionListener(event -> peer.close()); // never blocks the EDT +``` + +The order of each peer's events is kept with any executor, including a thread pool. Frame handlers stay on their own threads even then, since frames come far too often for a UI thread. + +### Don't Wait in a Handler + +A future of the client API completes on the event thread of its peer. A handler that waits for one would wait for its own thread, so `join()` and `get()` there throw a `BlockingCallException` instead of hanging. Chain the next step instead: + +```java +peer.onChannel(channel -> { + // channel.whenOpen().join() would throw here. + channel.whenOpen().thenCompose(open -> open.send("hello")); +}); +``` + +A future that is already done may be read anywhere, and any thread that is not an event thread, such as `main`, may wait. + +::: warning +On Java 8, only the futures the API returns refuse to be waited on. Futures derived from them with `thenApply` and the like do not, so waiting for one of those in a handler hangs that peer. From Java 9 on, the derived futures refuse too. + +A UI thread used as event executor is the same: waiting on it for a future of this API hangs the UI, since the future completes on it. +::: + +## Sending Media + +`rtc.media()` creates the tracks a runtime sends, and `peer.addTrack(track)` sends one. With automatic negotiation, the peer renegotiates by itself. + +```java +LocalMedia media = rtc.media(); + +LocalVideo camera = media.camera(CameraOptions.builder() + .device(media.devices().cameras().get(0)) + .resolution(1280, 720) + .frameRate(30) + .build()); +LocalAudio microphone = media.microphone(); + +Sender videoSender = peer.addTrack(camera); +Sender audioSender = peer.addTrack(microphone); +``` + +| Method | Track | +|---|---| +| `camera()`, `camera(CameraOptions)` | A camera. The supported capability nearest to the requested resolution and frame rate is used. | +| `microphone()`, `microphone(AudioOptions)` | The microphone, with echo cancellation, noise suppression and gain control unless `AudioOptions` say otherwise. `DEVICES` mode only. | +| `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. | +| `customVideo(String)` | Frames you push. | +| `customAudio(String)` | Audio you push. `CUSTOM` and `HEADLESS` modes only. | + +```java +Devices devices = media.devices(); +LocalVideo screen = media.screen(ScreenOptions.screen(devices.screens().get(0)).frameRate(15).build()); +``` + +Tracks added without stream ids share one stream per peer, so that a browser plays their audio and video in sync. Pass ids to `addTrack(track, streamIds...)` to group them yourself. + +One track may be sent on any number of peers. Opening a camera that is already open shares its capture when the options are the same, and fails with a `MediaException` when they differ. + +### Pushing Video and Audio + +```java +CustomVideo video = media.customVideo("render"); + +VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); +video.push(frame); +frame.release(); +``` + +`push(frame)` treats the frame as captured now. A source with timing of its own, such as a file, passes a timestamp with `push(frame, timestampMicros)`; see [Custom Video Source](/guide/video/custom-video-source) for why that matters. + +```java +CustomAudio voice = media.customAudio("voice"); + +// 16-bit little-endian PCM, any length, from any thread. +voice.push(pcm, PcmFormat.of(48000, 1)); +``` + +`CustomAudio` takes PCM in pieces of any length and from any thread. It cuts the audio into the 10 ms chunks WebRTC works with and sends one every 10 ms from a thread of its own, so pushing a whole second at once is as fine as pushing every 10 ms. `queuedMillis()` tells how much is still waiting, and `clear()` drops it. + +### Controlling Senders and Tracks + +```java +camera.setEnabled(false); // sends black frames, the camera stays open + +videoSender.configure(Encoding.builder() + .maxBitrate(1_500_000) + .maxFrameRate(30) + .build()); + +videoSender.replaceTrack(screen); // no renegotiation +audioSender.remove(); // renegotiates; the track stays open + +camera.close(); // leaves every peer and stops the camera +``` + +Closing a track stops sending it on every peer, without renegotiating, and then releases the capture. The senders stay, and can be given another track with `replaceTrack`. + +### Devices + +`media.devices()` returns the cameras, microphones and speakers. The first call reads them from the system; after that, the lists are snapshots kept current in the background, so reading them is cheap on any thread. `screens()` and `windows()` ask the system on each call. + +```java +Devices devices = media.devices(); + +devices.onChange(changed -> System.out.println("Cameras: " + changed.cameras())); +devices.useMicrophone(devices.microphones().get(1)); +devices.useSpeaker(devices.speakers().get(0)); +``` + +`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. + +## Receiving Media + +`peer.onTrack` is called for every track the remote peer sends. Tracks that already arrived are handed to a handler added later. + +```java +peer.onTrack(track -> { + System.out.println("Receiving " + track.kind() + " in " + track.streamIds()); + + if (track.isVideo()) { + track.asVideo().onFrame(frame -> view.render(frame)); + } + else { + track.asAudio().onAudio(chunk -> recognizer.feed(chunk.samples())); + } + + track.onMute(muted -> System.out.println("Muted: " + muted)); + track.onEnded(() -> System.out.println("Track ended")); +}); +``` + +A track ends when the remote peer stops sending it or the peer closes. + +### Video Frames + +Each `onFrame` subscription has a thread of its own, so a slow renderer holds up neither WebRTC's decoder nor the peer's other events. By default the handler always gets the newest frame, and the frames it had no time for are dropped. `FrameDelivery.queue(n)` keeps up to `n` frames instead: + +```java +video.onFrame(FrameDelivery.queue(10), frame -> encoder.encode(frame)); +``` + +A frame is valid while the handler runs. Call `frame.retain()` to keep it longer, and `frame.release()` when done. + +### Audio + +`onAudio` hands decoded audio to a handler in order, usually 10 ms of 16-bit PCM per `AudioChunk`. Audio reaches these handlers only while the runtime plays out, which a `HEADLESS` runtime always does and a `DEVICES` or `CUSTOM` runtime does to its speakers. `setVolume(double)` changes the playout gain of a track, from 0 to 10, where 1 plays it as received. + +### Receiving Without Sending + +A viewer that sends nothing asks for what it wants to receive, so that the offer includes it: + +```java +Peer viewer = rtc.createPeer(PeerOptions.builder() + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .signaling(signaling) + .build()); +``` + +## Data Channels + +```java +DataChannel chat = peer.createChannel("chat"); + +chat.onMessage(message -> System.out.println("Peer says: " + message.text())); +chat.send("hello"); // sent as soon as the channel opens + +peer.onChannel(channel -> { // channels the remote peer opens + if (channel.label().equals("files")) { + channel.onMessage(message -> output.write(message.bytes())); + } +}); +``` + +- A channel may be used right away: messages sent before it opens are held until it does. +- Messages that arrive before the first `onMessage` handler are kept for it, so none are lost while a handler is being set up. +- `send` takes a `String`, a `byte[]` or a `ByteBuffer`, and copies the data before it returns. +- A `Message` holds a copy of the data. `isText()` tells text from binary, `text()` decodes it as UTF-8, and `bytes()` returns a copy. + +The future `send` returns completes when the message is handed to the transport. While more than 1 MiB waits to be sent, further messages are held in Java, and their futures complete only as the backlog shrinks. A loop that waits for each send therefore slows down to what the network carries, instead of filling native memory: + +```java +for (byte[] chunk : chunks) { + channel.send(chunk).get(); // from a thread that is not an event thread +} +``` + +`ChannelOptions` set how a channel delivers: + +```java +DataChannel positions = peer.createChannel("positions", ChannelOptions.builder() + .ordered(false) + .maxRetransmits(0) + .build()); +``` + +`negotiatedId(int)` creates a channel both peers set up themselves with the same id, without announcing it; such a channel never shows up in `onChannel`. The [Data Channels](/guide/data/data-channels) guide explains the options in detail. + +## Manual Negotiation + +WHIP, WHEP and most SFUs dictate their own offer and answer exchange. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. This WHEP client receives a stream: + +```java +Peer viewer = rtc.createPeer(PeerOptions.builder() + .negotiation(Negotiation.MANUAL) + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .build()); + +viewer.onTrack(track -> System.out.println("Receiving " + track.kind())); + +viewer.negotiator().createOffer(Gathering.COMPLETE) + .thenApplyAsync(offer -> http.postSdp(whepUrl, offer.sdp), ioPool) // blocking HTTP off the event thread + .thenCompose(sdp -> viewer.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, sdp))) + .thenCompose(applied -> viewer.whenConnected()) + .get(10, TimeUnit.SECONDS); +``` + +| Method | What it does | +|---|---| +| `createOffer()`, `createOffer(Gathering)` | Creates the local offer and applies it. | +| `answer(offer)`, `answer(offer, Gathering)` | Applies the remote offer, then creates the answer and applies it. | +| `applyAnswer(answer)` | Applies the remote answer to the local offer. | +| `addCandidate(candidate)` | Adds a remote candidate, or holds it until there is a remote description to add it to. | + +A manual peer with a `Signaling` still sends its local candidates through it. `peer.negotiator()` throws an `IllegalStateException` on a peer that negotiates automatically. + +## Statistics + +`peer.stats()` returns the numbers people look for first, taken from the full [RTC stats](/guide/monitoring/rtc-stats) report: + +```java +peer.onStats(Duration.ofSeconds(1), stats -> System.out.printf( + "%s candidate, rtt %s, %d kbit/s out, %d kbit/s in, %.1f%% lost%n", + stats.candidateType(), stats.roundTripTime(), + stats.outgoingBitrate() / 1000, stats.incomingBitrate() / 1000, + stats.packetLoss() * 100)); +``` + +| Method | Value | +|---|---| +| `candidateType()` | The type of the local candidate in use: `host`, `srflx`, `prflx` or `relay`. Null while not connected. | +| `roundTripTime()` | The current round trip time, or null while not measured. | +| `bytesSent()`, `bytesReceived()` | The bytes sent and received on the connection so far. | +| `outgoingBitrate()`, `incomingBitrate()` | Bits per second since the previous sample, or -1 for a peer's first sample. | +| `packetLoss()` | The share of received packets lost so far, 0 to 1. | +| `outboundVideo()`, `inboundVideo()` | Codec, resolution and frame rate of each video stream, and what limits a sent one. | +| `report()` | The whole `RTCStatsReport`. | + +## Closing + +You close what you created, and closing an owner closes what it owns: + +| Object | Closing it also | +|---|---| +| `WebRtc` | closes all peers, then all tracks, then the factory, and stops the threads of the client API | +| `Peer` | closes its channels and senders and ends its remote tracks. Local tracks stay open for other peers. | +| `LocalTrack` | stops sending it on every peer and stops its capture | +| `DataChannel` | fails the messages not yet handed to the transport | +| `Subscription` | removes the handler | + +`Peer.close()` and every other `close()` except the runtime's return at once, may be called more than once, and are fine inside a handler. After a peer closed, its only event is the `CLOSED` state. Operations that were pending fail with a `PeerClosedException`, and so do those called later. `closeAsync()` returns a future for callers that want to wait. + +`WebRtc.close()` is the one close that waits, for up to the close timeout, which makes it fit for try-with-resources. + +## Errors + +Failures arrive as exceptions, thrown or as the cause of a failed future, all of them a `WebRtcException`: + +| Exception | When | +|---|---| +| `NegotiationException` | A description or candidate was rejected, or two impolite peers collided. | +| `PeerClosedException` | An operation was pending when its peer, channel or runtime closed, or came after that. | +| `MediaException` | A device is missing, busy, refused access, or already open with other options. | +| `ConfigurationException` | A call does not fit how the runtime was built, or WebRTC refused a configuration. | +| `BlockingCallException` | A future was waited on from an event thread. | + +Failures that no caller waits for, such as a remote description rejected during automatic negotiation, go to the peer's `onError` handlers: + +```java +peer.onError(error -> log.warn("Peer {} reported", peer.name(), error)); +``` + +Without an `onError` handler, they go to the runtime's error handler, as do exceptions that handlers throw. A handler that throws never stops the delivery of later events. + +## Using the Low-Level API + +For what the client API does not cover, the underlying objects stay reachable: + +```java +// Anything RTCConfiguration offers. +PeerOptions.builder().configure(config -> config.bundlePolicy = RTCBundlePolicy.MAX_BUNDLE); + +// Anything the factory builder offers. +WebRtc.builder().configureFactory(factory -> factory.setVideoEncoderFactory(myEncoders)); + +// The connection itself, in order with the peer's other operations. +peer.withNative(connection -> connection.getSignalingState()) + .thenAccept(state -> System.out.println("Signaling state: " + state)); +``` + +The function passed to `withNative` runs after the operations called before it and before those called after it, so it never interferes with negotiation. It must not block. Objects it obtains from the connection belong to it, as in the low-level API, and must not be used after it returns. + +## Example + +The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process through a signaling that carries JSON text. It sends pushed video, audio and chat messages from one to the other, and prints statistics once a second. It needs no audio or video hardware: + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ClientApiExample" -D"exec.args=10" +``` diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 9d4ebc38..6f127274 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -2,6 +2,26 @@ This section provides an overview of the example applications included in the `webrtc-examples` Maven module. These examples demonstrate various features and capabilities of the webrtc-java library. +## Client API + +The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process with the [Client API](/guide/client-api), and sends video, audio and chat messages from one to the other. It needs no audio or video hardware and no server. + +**Key features demonstrated:** +- Creating a headless `WebRtc` runtime, which needs no microphone or speakers +- Connecting two peers through a signaling that carries JSON text, as a WebSocket would +- Letting perfect negotiation run the offer and answer exchange, with one polite peer +- Pushing video frames, and audio in pieces of any length +- Receiving remote tracks, their frames and their audio +- Exchanging messages on a data channel +- Reading the connection's statistics once a second +- Closing the peers, tracks and factory by closing the runtime + +This example is the shortest way to see a complete call working, and a starting point for applications built on the client API. + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ClientApiExample" -D"exec.args=10" +``` + ## PeerConnection The [`PeerConnectionExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/PeerConnectionExample.java) demonstrates how to set up a peer connection with audio and video tracks to be able to send and receive media. diff --git a/docs/guide/get-started.md b/docs/guide/get-started.md index b7ef0cc8..31388e32 100644 --- a/docs/guide/get-started.md +++ b/docs/guide/get-started.md @@ -116,6 +116,10 @@ implementation group: "dev.onvoid.webrtc", name: "webrtc-java", version: "{{ VER ## Basic Setup +::: tip +This guide uses the core API, which mirrors WebRTC's own. The [Client API](/guide/client-api) builds on it: it negotiates by itself, keeps your handlers off WebRTC's threads, and closes everything it created with one call. Most applications are simpler with it. +::: + ### Initialize the WebRTC Library First, create a `PeerConnectionFactory` which is the entry point for most WebRTC operations: diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java new file mode 100644 index 00000000..46662f3d9 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java @@ -0,0 +1,241 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomAudio; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.DataChannel; +import dev.onvoid.webrtc.client.PcmFormat; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.SignalMessage; +import dev.onvoid.webrtc.client.VideoStats; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * Example of the high-level client API ({@code dev.onvoid.webrtc.client}): two peers in one + * process send each other video, audio and chat messages. + *

+ * This example shows how to: + *

    + *
  • Create a {@link WebRtc} runtime that needs no audio hardware
    • + *
  • Connect two peers through a signaling that carries JSON text, as a WebSocket would
    • + *
  • Let perfect negotiation run the offer and answer exchange, with one polite peer
    • + *
  • Send pushed video frames and pushed audio of any length
    • + *
  • Receive remote tracks, their frames and their audio
    • + *
  • Exchange messages on a data channel
    • + *
  • Read the connection's statistics once a second
    • + *
+ *

+ * No observer, no {@code dispose()} and no thread of its own is needed: handlers run on the event + * thread of their peer, and closing the runtime closes everything it created. + *

+ * Run it for a number of seconds, five by default: + * 

+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ClientApiExample" -D"exec.args=10"
+ *
+ * + * @author Alex Andres + */ +public class ClientApiExample { + + private static final int WIDTH = 640; + + private static final int HEIGHT = 360; + + private static final PcmFormat AUDIO_FORMAT = PcmFormat.of(48000, 1); + + + public static void main(String[] args) throws Exception { + int seconds = args.length > 0 ? Integer.parseInt(args[0]) : 5; + + // A headless runtime sends pushed audio and needs no microphone or speakers. + try (WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build()) { + AtomicReference callerRef = new AtomicReference<>(); + AtomicReference calleeRef = new AtomicReference<>(); + + // Each signaling turns messages into JSON text and hands that text to the other peer. + // A real application sends it through its WebSocket instead. + Peer caller = rtc.createPeer(PeerOptions.builder() + .name("caller") + .signaling(message -> transmit("caller", calleeRef.get(), message.toJson())) + .build()); + Peer callee = rtc.createPeer(PeerOptions.builder() + .name("callee") + .polite(true) + .signaling(message -> transmit("callee", callerRef.get(), message.toJson())) + .build()); + + callerRef.set(caller); + calleeRef.set(callee); + + receive(callee); + caller.onStateChange(state -> System.out.println("Caller is " + state)); + + // Pushed media. The same tracks could be sent on any number of peers. + CustomVideo video = rtc.media().customVideo("pattern"); + CustomAudio audio = rtc.media().customAudio("tone"); + caller.addTrack(video); + caller.addTrack(audio); + + // Sent once the channel opens. + DataChannel chat = caller.createChannel("chat"); + chat.onMessage(message -> System.out.println("Caller got \"" + message.text() + "\"")); + chat.send("Hello from the caller"); + + caller.onStats(Duration.ofSeconds(1), stats -> { + if (stats.candidateType() == null) { + // Not connected yet. + return; + } + + String size = stats.outboundVideo().isEmpty() ? "-" : describe(stats.outboundVideo().get(0)); + + System.out.printf("Caller sends %d kbit/s over a %s candidate, video %s%n", + Math.max(0, stats.outgoingBitrate()) / 1000, stats.candidateType(), size); + }); + + ScheduledExecutorService feeder = Executors.newSingleThreadScheduledExecutor(); + + try { + feed(feeder, video, audio); + + // The main thread may wait; a handler must not. + caller.whenConnected().get(10, TimeUnit.SECONDS); + + Thread.sleep(seconds * 1000L); + } + finally { + feeder.shutdownNow(); + } + } + // The runtime closed the peers, the tracks and the factory, in that order. + + System.out.println("Closed"); + } + + /** + * Prints what the callee receives. + */ + private static void receive(Peer callee) { + AtomicInteger frames = new AtomicInteger(); + AtomicLong audioMillis = new AtomicLong(); + + callee.onTrack(track -> { + String kind = track.kind().name().toLowerCase(); + + System.out.println("Callee receives the " + kind + " track in stream " + track.streamIds()); + + if (track.isVideo()) { + // A slow handler gets the newest frame next; frames in between are dropped. + track.asVideo().onFrame(frame -> { + if (frames.incrementAndGet() % 30 == 1) { + System.out.printf("Callee got video frame %d, %dx%d%n", frames.get(), + frame.buffer.getWidth(), frame.buffer.getHeight()); + } + }); + } + else { + track.asAudio().onAudio(chunk -> { + long millis = audioMillis.addAndGet(chunk.frames() * 1000L / chunk.sampleRate()); + + if (millis % 1000 == 0) { + System.out.printf("Callee got %d ms of audio%n", millis); + } + }); + } + + track.onEnded(() -> System.out.println("Callee's " + kind + " track ended")); + }); + + // Channels the caller opens. Messages that arrive before onMessage are kept for it. + callee.onChannel(channel -> channel.onMessage(message -> { + System.out.println("Callee got \"" + message.text() + "\""); + channel.send("Echo: " + message.text()); + })); + } + + /** + * Stands in for a WebSocket: carries a message as text and reads it on the other side. + */ + private static void transmit(String from, Peer to, String json) { + SignalMessage message = SignalMessage.fromJson(json); + + if (message.type() != SignalMessage.Type.CANDIDATE) { + System.out.printf("%s sends an %s of %d characters%n", from, + message.type().name().toLowerCase(), json.length()); + } + + to.signal(message); + } + + /** + * Pushes a moving gradient at 30 frames per second, and a 440 Hz tone a second at a time. + */ + private static void feed(ScheduledExecutorService feeder, CustomVideo video, CustomAudio audio) { + AtomicInteger shade = new AtomicInteger(); + + feeder.scheduleAtFixedRate(() -> { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + ByteBuffer y = buffer.getDataY(); + int offset = shade.addAndGet(2); + + for (int i = 0; i < y.capacity(); i++) { + y.put(i, (byte) (i % WIDTH + offset)); + } + + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + video.push(frame); + frame.release(); + }, 0, 33, TimeUnit.MILLISECONDS); + + // Audio may be pushed in pieces of any length; the track sends it in 10 ms chunks. + byte[] second = tone(AUDIO_FORMAT.sampleRate()); + + feeder.scheduleAtFixedRate(() -> audio.push(second, AUDIO_FORMAT), 0, 1, TimeUnit.SECONDS); + } + + private static byte[] tone(int sampleRate) { + byte[] pcm = new byte[sampleRate * 2]; + + for (int i = 0; i < sampleRate; i++) { + short sample = (short) (Math.sin(2 * Math.PI * 440 * i / sampleRate) * 6000); + + pcm[2 * i] = (byte) sample; + pcm[2 * i + 1] = (byte) (sample >> 8); + } + + return pcm; + } + + private static String describe(VideoStats video) { + return String.format("%s %dx%d at %.0f fps", video.codec(), video.width(), video.height(), + video.framesPerSecond()); + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java index 08bd3863..6014c05d 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java @@ -159,6 +159,20 @@ void joinOnEventThreadRefused() throws Exception { assertInstanceOf(BlockingCallException.class, await(refused)); + // On Java 9 and later, so do the futures derived from them. + CompletableFuture derivedRefused = new CompletableFuture<>(); + peers[1].onStateChange(state -> { + try { + peers[1].whenClosed().thenApply(ignored -> state).join(); + derivedRefused.complete(null); + } + catch (Throwable e) { + derivedRefused.complete(e); + } + }); + + assertInstanceOf(BlockingCallException.class, await(derivedRefused)); + // A future that is done may be read anywhere. CompletableFuture doneRead = new CompletableFuture<>(); peers[0].createChannel("chat"); From 4cc51d991cf90cfd706585b049bceec67b15e249 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 01:00:02 +0200 Subject: [PATCH 03/25] Release frames delivered to client frame handlers A video sink owns the reference to each frame it gets. The dispatcher retained it once more, so every delivered frame leaked, and frames that arrived after a subscription closed were never released. --- .../onvoid/webrtc/client/FrameDispatcher.java | 26 +++++------ .../dev/onvoid/webrtc/client/MediaTests.java | 43 +++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java index f48e09a6..457f77c7 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/FrameDispatcher.java @@ -25,8 +25,10 @@ /** * Hands the frames of a video track to a handler on a thread of its own, so * that a slow handler holds up neither WebRTC nor any other subscription. - * Frames that wait are retained; those dropped when the queue is full, and - * those the handler got, are released. + *

+ * The native sink hands over one reference to each frame, which this object + * owns: it releases a frame once the handler returned, when the queue is full + * and the frame is dropped, or when the subscription closed. */ final class FrameDispatcher implements VideoTrackSink { @@ -59,18 +61,18 @@ public void onVideoFrame(VideoFrame frame) { synchronized (queue) { if (closed) { - return; + dropped = frame; } + else { + queue.add(frame); - frame.retain(); - queue.add(frame); - - if (queue.size() > capacity) { - dropped = queue.poll(); - } - if (!scheduled) { - scheduled = true; - executor.execute(this::drain); + if (queue.size() > capacity) { + dropped = queue.poll(); + } + if (!scheduled) { + scheduled = true; + executor.execute(this::drain); + } } } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java index aa5d3441..72eca15d 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java @@ -20,9 +20,14 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -184,6 +189,44 @@ void senderConfigured() throws Exception { assertThrows(IllegalArgumentException.class, () -> sender.replaceTrack(rtc.media().customAudio("audio"))); } + @Test + void deliveredFramesReleased() throws Exception { + Path status = Paths.get("/proc/self/status"); + assumeTrue(Files.isReadable(status), "Needs /proc to read the resident memory"); + + CustomVideo video = rtc.media().customVideo("video"); + int count = 300; + CountDownLatch delivered = new CountDownLatch(count); + + // Queues every frame, so that each of them reaches the handler. + Subscription subscription = video.onFrame(frame -> delivered.countDown()); + long before = residentBytes(status); + + NativeI420Buffer buffer = NativeI420Buffer.allocate(1280, 720); + for (int i = 0; i < count; i++) { + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + video.push(frame); + Thread.sleep(2); + } + buffer.release(); + + delivered.await(TIMEOUT_SECONDS, TimeUnit.SECONDS); + subscription.close(); + + // Each delivered frame is a copy of about 1.4 MB; keeping them all would hold over 400 MB. + long grown = residentBytes(status) - before; + assertTrue(grown < 150L * 1024 * 1024, () -> "Resident memory grew by " + grown / (1024 * 1024) + " MB"); + } + + private static long residentBytes(Path status) throws Exception { + for (String line : Files.readAllLines(status)) { + if (line.startsWith("VmRSS:")) { + return Long.parseLong(line.replaceAll("[^0-9]", "")) * 1024; + } + } + throw new IllegalStateException("No VmRSS in " + status); + } + @Test void audioModeChecked() { assertThrows(ConfigurationException.class, () -> rtc.media().microphone()); From 02c17137b1426da88d035332be94a84adc0bcdef Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 01:09:01 +0200 Subject: [PATCH 04/25] Fix web example server for Jetty 12.1 Jetty 12.1 needs a context around the handlers; without one every page and WebSocket request failed with a NullPointerException. The WebSocket upgrade handler now wraps the static content inside a root context. --- .../webrtc/examples/web/server/WebServer.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/server/WebServer.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/server/WebServer.java index 98eb42a5..0dc21168 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/server/WebServer.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/server/WebServer.java @@ -21,9 +21,8 @@ import dev.onvoid.webrtc.examples.web.WebClientExample; -import org.eclipse.jetty.http.pathmap.PathSpec; import org.eclipse.jetty.server.*; -import org.eclipse.jetty.server.handler.PathMappingsHandler; +import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.util.resource.ResourceFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -77,8 +76,11 @@ public static void main(String[] args) throws Exception { // Add only HTTPS connector. server.addConnector(httpsConnector); + // Since Jetty 12.1 the handlers need a context; without one every request fails. + ContextHandler context = new ContextHandler("/"); + // Create WebSocket upgrade handler. - WebSocketUpgradeHandler wsHandler = WebSocketUpgradeHandler.from(server, container -> { + WebSocketUpgradeHandler wsHandler = WebSocketUpgradeHandler.from(server, context, container -> { container.addMapping("/ws", (upgradeRequest, upgradeResponse, callback) -> { if (upgradeRequest.getSubProtocols().contains("ws-signaling")) { upgradeResponse.setAcceptedSubProtocol("ws-signaling"); @@ -93,12 +95,12 @@ public static void main(String[] args) throws Exception { ResourceFactory resourceFactory = ResourceFactory.of(resourceHandler); resourceHandler.setBaseResource(resourceFactory.newClassLoaderResource("/resources/web/")); - // Create a path mappings handler to combine WebSocket and static content. - PathMappingsHandler pathHandler = new PathMappingsHandler(); - pathHandler.addMapping(PathSpec.from("/ws/*"), wsHandler); - pathHandler.addMapping(PathSpec.from("/"), resourceHandler); + // The upgrade handler takes the WebSocket requests to its mappings and + // hands every other request to the static content. + wsHandler.setHandler(resourceHandler); - server.setHandler(pathHandler); + context.setHandler(wsHandler); + server.setHandler(context); server.start(); LOG.info("HTTPS: https://localhost:{}", PORT); From 5ec6cacb5efbdb9e63e138f11e12b9ff7f95b351 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 01:09:01 +0200 Subject: [PATCH 05/25] Port the web example to the client API The Java client is a peer of the client API, with SignalingManager as its Signaling over the example's own message format. A new peer calls the browser whenever it joins. WebSocket sends are queued, since a WebSocket takes one at a time, and the browser reuses its connection when the Java side renegotiates. --- docs/guide/examples.md | 12 +- .../examples/util/MediaFrameLogger.java | 88 ++-- .../webrtc/examples/web/WebClientExample.java | 194 ++++---- .../examples/web/client/SignalingManager.java | 345 +++++-------- .../examples/web/client/WebSocketClient.java | 23 +- .../web/connection/PeerConnectionManager.java | 461 ------------------ .../PeerConnectionSignalingHandler.java | 51 -- .../examples/web/media/AudioGenerator.java | 37 +- .../examples/web/media/MediaManager.java | 253 ++-------- .../examples/web/media/VideoGenerator.java | 21 +- .../src/main/resources/web/WebRTCClient.js | 27 +- 11 files changed, 368 insertions(+), 1144 deletions(-) delete mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionManager.java delete mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionSignalingHandler.java diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 6f127274..d35615c5 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -136,15 +136,15 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.EncryptedRecordingExa ## Web Client -The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) demonstrates how to combine WebSocket signaling with WebRTC peer connections for real-time communication between web and Java clients. +The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) calls a browser from Java over WebSocket signaling, with the [Client API](/guide/client-api). The browser client is served by the example's `WebServer`. **Key features demonstrated:** - Connecting to a signaling server using WebSockets -- Setting up audio and video media tracks -- Creating and managing WebRTC peer connections -- Handling signaling messages (offers, answers, ICE candidates) -- Joining and leaving rooms for multi-user communication -- Using custom audio sources for audio streaming +- Implementing a `Signaling` over the application's own message format, which also names the sender and receiver of each message +- Sending generated audio and video, pushed into tracks of the runtime +- Letting the peer negotiate by itself, and starting a new peer whenever the browser joins the room +- Joining and leaving rooms +- Logging the audio and video the browser sends This example is valuable for applications that need to establish WebRTC connections between Java clients and web browsers, enabling cross-platform real-time communication with audio and video. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/util/MediaFrameLogger.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/util/MediaFrameLogger.java index ae1f748a..f1c12a86 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/util/MediaFrameLogger.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/util/MediaFrameLogger.java @@ -16,15 +16,18 @@ package dev.onvoid.webrtc.examples.util; -import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.client.AudioChunk; import dev.onvoid.webrtc.media.video.VideoFrame; -import dev.onvoid.webrtc.media.video.VideoTrackSink; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Utility class for logging audio and video frame information. + * Utility class for logging audio and video frame information, as handlers of + * the client API's {@code RemoteAudio.onAudio} and {@code RemoteVideo.onFrame}. + * Each logger writes a summary once a second. * * @author Alex Andres */ @@ -32,82 +35,81 @@ public class MediaFrameLogger { private static final Logger LOG = LoggerFactory.getLogger(MediaFrameLogger.class); + private static final long LOG_INTERVAL_MS = 1000; + /** - * Creates a new audio track sink that logs information about received audio data. + * Creates a handler that logs information about received audio. * - * @return An AudioTrackSink that logs audio frame information. + * @return A handler for {@code RemoteAudio.onAudio}. */ - public static AudioTrackSink createAudioLogger() { + public static Consumer createAudioLogger() { return new AudioFrameLogger(); } /** - * Creates a new video track sink that logs information about received video frames. + * Creates a handler that logs information about received video frames. * - * @return A VideoTrackSink that logs video frame information. + * @return A handler for {@code RemoteVideo.onFrame}. */ - public static VideoTrackSink createVideoLogger() { + public static Consumer createVideoLogger() { return new VideoFrameLogger(); } /** - * A simple implementation of AudioTrackSink that logs information about received audio data. + * Logs the number of audio chunks received per second and the format of the last one. + * The client API calls it on one thread, in order. */ - private static class AudioFrameLogger implements AudioTrackSink { - - private static final long LOG_INTERVAL_MS = 1000; // Log every second - private int frameCount = 0; + private static class AudioFrameLogger implements Consumer { + + private int chunkCount = 0; + private long lastLogTime = System.currentTimeMillis(); - + + @Override - public void onData(byte[] data, int bitsPerSample, int sampleRate, int channels, int frames) { - frameCount++; - + public void accept(AudioChunk chunk) { + chunkCount++; + long now = System.currentTimeMillis(); + if (now - lastLogTime >= LOG_INTERVAL_MS) { - LOG.info(String.format("Received %d audio frames in the last %.1f seconds", - frameCount, (now - lastLogTime) / 1000.0)); - LOG.info(String.format("Last audio data: %d bytes, %d bits/sample, %d Hz, %d channels, %d frames", - data.length, bitsPerSample, sampleRate, channels, frames)); - - frameCount = 0; + LOG.info(String.format("Received %d audio chunks in the last %.1f seconds, the last one %s", + chunkCount, (now - lastLogTime) / 1000.0, chunk)); + + chunkCount = 0; lastLogTime = now; } } } - - /** - * A simple implementation of VideoTrackSink that logs information about received frames. + * Logs the number of video frames received per second and the size of the last one. + * The frame is released by the client API once the handler returned. */ - private static class VideoFrameLogger implements VideoTrackSink { - - private static final long LOG_INTERVAL_MS = 1000; // Log every second + private static class VideoFrameLogger implements Consumer { + private int frameCount = 0; + private long lastLogTime = System.currentTimeMillis(); - + + @Override - public void onVideoFrame(VideoFrame frame) { + public void accept(VideoFrame frame) { frameCount++; - + long now = System.currentTimeMillis(); + if (now - lastLogTime >= LOG_INTERVAL_MS) { - LOG.info(String.format("Received %d video frames in the last %.1f seconds", - frameCount, (now - lastLogTime) / 1000.0)); - LOG.info(String.format("Last frame: %dx%d, rotation: %d, timestamp: %dms", - frame.buffer.getWidth(), frame.buffer.getHeight(), frame.rotation, - frame.timestampNs / 1000000)); - + LOG.info(String.format("Received %d video frames in the last %.1f seconds, the last one %dx%d, rotation %d", + frameCount, (now - lastLogTime) / 1000.0, frame.buffer.getWidth(), + frame.buffer.getHeight(), frame.rotation)); + frameCount = 0; lastLogTime = now; } - - // Release the native resources associated with this frame to prevent memory leaks. - frame.release(); } } -} \ No newline at end of file +} diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java index 80efc981..7af8341c 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java @@ -21,31 +21,41 @@ import java.net.URI; import java.util.logging.LogManager; -import dev.onvoid.webrtc.examples.web.model.LeaveMessage; -import dev.onvoid.webrtc.media.MediaStreamTrack; - +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.examples.util.MediaFrameLogger; import dev.onvoid.webrtc.examples.web.client.SignalingManager; -import dev.onvoid.webrtc.examples.web.connection.PeerConnectionManager; import dev.onvoid.webrtc.examples.web.media.MediaManager; import dev.onvoid.webrtc.examples.web.model.JoinMessage; -import dev.onvoid.webrtc.examples.util.MediaFrameLogger; +import dev.onvoid.webrtc.examples.web.model.LeaveMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Example demonstrating how to combine WebSocket signaling with WebRTC peer connections. + * Example demonstrating how to combine WebSocket signaling with the client API + * ({@code dev.onvoid.webrtc.client}), to call the browser client of the + * {@link dev.onvoid.webrtc.examples.web.server.WebServer WebServer}. *

* This example shows how to: *

    *
  • Connect to a signaling server using WebSockets
    • - *
  • Set up media tracks
    • - *
  • Establish a WebRTC peer connection with audio and video
    • + *
  • Implement a {@code Signaling} over a message format of the application's own
    • + *
  • Send generated audio and video, pushed into tracks of the runtime
    • + *
  • Let the peer negotiate by itself, and start a new peer whenever the browser joins
    • + *
  • Log the audio and video the browser sends
    • *
+ *

+ * Negotiation, the threads of the callbacks and the release of native objects are + * left to the client API. The browser only ever answers, so the Java side keeps + * the default, impolite role of perfect negotiation. * * @author Alex Andres */ -public class WebClientExample { +public final class WebClientExample implements AutoCloseable { static { setupLogging(); @@ -53,15 +63,18 @@ public class WebClientExample { private static final Logger LOG = LoggerFactory.getLogger(WebClientExample.class); + /** Owns the factory, the tracks and the peers. */ + private final WebRtc rtc; + /** Manages communication with the signaling server. */ private final SignalingManager signaling; - /** Handles WebRTC peer connection setup and management. */ - private final PeerConnectionManager peerConnectionManager; - - /** Manages audio and video tracks and devices. */ + /** Creates and feeds the audio and video tracks. */ private final MediaManager mediaManager; + /** The connection to the browser, replaced whenever the browser joins anew. */ + private Peer peer; + /** * Creates a new WebClientExample that combines WebSocket signaling with WebRTC. @@ -70,82 +83,30 @@ public class WebClientExample { * @param subprotocol The WebSocket subprotocol to use. */ public WebClientExample(URI signalingUri, String subprotocol) { - signaling = new SignalingManager(signalingUri, subprotocol); - signaling.setUserId("java-client"); - - peerConnectionManager = new PeerConnectionManager(); - - mediaManager = new MediaManager(); - mediaManager.createTracks(peerConnectionManager, true, true); + // Sends generated audio, and plays what the browser sends to the speakers. + rtc = WebRtc.builder().audio(AudioMode.CUSTOM).build(); - setupCallbacks(); + mediaManager = new MediaManager(rtc); - LOG.info("WebClientExample created with audio and video tracks"); - } - - /** - * Sets up callbacks between components. - */ - private void setupCallbacks() { - // Set up signaling callbacks. - signaling.setOnOfferReceived(peerConnectionManager::handleOffer); - signaling.setOnAnswerReceived(peerConnectionManager::handleAnswer); - signaling.setOnCandidateReceived(peerConnectionManager::handleCandidate); + signaling = new SignalingManager(signalingUri, subprotocol); + signaling.setUserId("java-client"); signaling.setOnJoinReceived(this::handleJoin); signaling.setOnLeaveReceived(this::handleLeave); - // Set up peer connection callbacks. - peerConnectionManager.setOnLocalDescriptionCreated(signaling::sendSessionDescription); - peerConnectionManager.setOnIceCandidateGenerated(signaling::sendIceCandidate); - peerConnectionManager.setOnTrackReceived(this::handleTrackReceived); - } - - /** - * Handles a received media track. - * - * @param track The received media track. - */ - private void handleTrackReceived(MediaStreamTrack track) { - String kind = track.getKind(); - LOG.info("Received track: {}", kind); - - if (kind.equals(MediaStreamTrack.AUDIO_TRACK_KIND)) { - mediaManager.addAudioSink(MediaFrameLogger.createAudioLogger()); - } - else if (kind.equals(MediaStreamTrack.VIDEO_TRACK_KIND)) { - mediaManager.addVideoSink(MediaFrameLogger.createVideoLogger()); - } + LOG.info("WebClientExample created with audio and video tracks"); } /** * Connects to the signaling server. - * + * * @return true if the connection was successful, false otherwise. */ public boolean connect() { return signaling.connect(); } - /** - * Disconnects from the signaling server and closes the peer connection. - */ - public void disconnect() { - signaling.disconnect(); - - if (mediaManager != null) { - mediaManager.dispose(); - } - if (peerConnectionManager != null) { - peerConnectionManager.close(); - } - - LOG.info("Disconnected from signaling server and closed peer connection"); - } - /** * Joins a room on the signaling server. - *

- * This method sends a join request to the signaling server for the specified room. * * @param roomName the name of the room to join on the signaling server */ @@ -159,59 +120,103 @@ public void joinRoom(String roomName) { } /** - * Initiates a call by creating and sending an offer. + * Calls the browser with a new peer, which sends its offer by itself once its tracks + * are added. A peer that was there before is closed. */ - public void call() { + public synchronized void call() { if (!signaling.isConnected()) { LOG.warn("Cannot initiate call: not connected to signaling server"); return; } - peerConnectionManager.setInitiator(true); - peerConnectionManager.createOffer(); + if (peer != null) { + peer.close(); + } + + // The new peer attaches to the signaling, so the browser's messages go to it. + peer = rtc.createPeer(PeerOptions.builder() + .name("web-client") + .iceServer("stun:stun.l.google.com:19302") + .signaling(signaling) + .build()); + + // The tracks share one stream, so the browser plays them in sync. + peer.addTrack(mediaManager.audio()); + peer.addTrack(mediaManager.video()); + + peer.onStateChange(state -> LOG.info("Connection to the browser is {}", state)); + peer.onTrack(this::handleTrackReceived); + peer.onError(error -> LOG.warn("The connection to the browser reported an error", error)); + } + + /** + * Hangs up and disconnects from the signaling server. The runtime closes the peer and + * the tracks, in the order they need. + */ + @Override + public void close() { + signaling.disconnect(); + mediaManager.close(); + rtc.close(); + + LOG.info("Disconnected from signaling server and closed the peer connection"); + } + + /** + * Logs a track the browser sends. Runs on the peer's event thread, the frames and the + * audio on threads of their own. + * + * @param track The received media track. + */ + private void handleTrackReceived(RemoteTrack track) { + LOG.info("Received {} track", track.kind()); + + if (track.isAudio()) { + track.asAudio().onAudio(MediaFrameLogger.createAudioLogger()); + } + else { + track.asVideo().onFrame(MediaFrameLogger.createVideoLogger()); + } + + track.onEnded(() -> LOG.info("The browser stopped sending its {} track", track.kind())); } /** - * Handles an incoming join message from the remote peer. + * Handles a join message: a browser that joins the room gets a new peer and with it a + * fresh offer, since it has no connection to answer the earlier one with. * * @param message The join message. */ private void handleJoin(JoinMessage message) { LOG.info("Received join message from peer: {}", message.getFrom()); - // Remote peer wants to join a room, we need to create an offer if we're not the initiator. - if (!peerConnectionManager.isInitiator()) { - peerConnectionManager.createOffer(); - } + call(); } /** - * Handles an incoming leave message from the remote peer. + * Handles a leave message: the browser is gone, so its connection is closed. * * @param message The leave message. */ - private void handleLeave(LeaveMessage message) { + private synchronized void handleLeave(LeaveMessage message) { LOG.info("Peer {} has left the room", message.getFrom()); - // Handle peer leaving logic, such as closing tracks or updating UI. + if (peer != null) { + peer.close(); + peer = null; + } } /** * Entry point to demonstrate the WebClientExample. */ public static void main(String[] args) { - try { - WebClientExample client = new WebClientExample( - URI.create("wss://localhost:8443/ws"), - "ws-signaling"); - + try (WebClientExample client = new WebClientExample(URI.create("wss://localhost:8443/ws"), "ws-signaling")) { if (client.connect()) { LOG.info("Connected to signaling server"); - // Join a room. + // Join a room, then call whoever is in it already. client.joinRoom("default-room"); - - // Initiate the call. client.call(); // Keep the application running to observe state changes. @@ -221,9 +226,6 @@ public static void main(String[] args) { else { LOG.error("Failed to connect to signaling server"); } - - // Clean up. - client.disconnect(); } catch (Exception e) { LOG.error("Error in WebClientExample", e); @@ -243,4 +245,4 @@ public static void setupLogging() { System.err.println(e.getMessage()); } } -} \ No newline at end of file +} diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/SignalingManager.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/SignalingManager.java index 2d6fd4ea..892d98d0 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/SignalingManager.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/SignalingManager.java @@ -17,14 +17,16 @@ package dev.onvoid.webrtc.examples.web.client; import java.net.URI; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import com.fasterxml.jackson.databind.ObjectMapper; import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCSdpType; import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.client.SignalMessage; +import dev.onvoid.webrtc.client.Signaling; import dev.onvoid.webrtc.examples.web.model.*; @@ -32,22 +34,23 @@ import org.slf4j.LoggerFactory; /** - * Manages WebSocket signaling for WebRTC connections. + * Carries a peer's signaling over a WebSocket, to and from the browser client. *

- * This class handles the communication between peers during WebRTC connection establishment. - * It manages sending and receiving: - *

    - *
  • Session descriptions (offers and answers)
    • - *
  • ICE candidates
    • - *
  • Join messages for room-based signaling
    • - *
- *

- * The manager connects to a signaling server via WebSocket and translates WebRTC-related - * objects to JSON messages that can be transmitted over the signaling channel. + * The browser client and the server speak a message format of their own, which + * names the sender and the receiver of each message and wraps the session + * description or ICE candidate in a {@code data} object: + * 

{@code
+ * {"type":"offer","from":"java-client","to":"web-client","data":{"sdp":"v=0..."}}
+ * {"type":"ice-candidate","from":"web-client","to":"java-client","data":{"candidate":"...","sdpMid":"0","sdpMLineIndex":0}}
+ * }
+ * This class translates between that format and the {@link SignalMessage}s of the + * client API. As a {@link Signaling}, it sends a peer's messages and hands the + * peer the ones the browser sends. It also sends and receives the join and leave + * messages of the room. * * @author Alex Andres */ -public class SignalingManager { +public class SignalingManager implements Signaling { private static final Logger LOG = LoggerFactory.getLogger(SignalingManager.class); @@ -57,14 +60,8 @@ public class SignalingManager { /** JSON mapper for serializing and deserializing signaling messages. */ private final ObjectMapper jsonMapper; - /** Callback that is triggered when an offer message is received from a remote peer. */ - private Consumer onOfferReceived; - - /** Callback that is triggered when an answer message is received from a remote peer. */ - private Consumer onAnswerReceived; - - /** Callback that is triggered when an ICE candidate is received from a remote peer. */ - private Consumer onCandidateReceived; + /** The peer that receives the session descriptions and candidates of the browser. */ + private volatile Consumer inbound; /** Callback that is triggered when a join message is received. */ private Consumer onJoinReceived; @@ -73,6 +70,7 @@ public class SignalingManager { private Consumer onLeaveReceived; private String userId = "java-client"; + private String remotePeerId = "web-client"; @@ -80,7 +78,7 @@ public class SignalingManager { * Creates a new SignalingManager with the specified URI and subprotocol. * * @param signalingUri The URI of the signaling server. - * @param subprotocol The WebSocket subprotocol to use. + * @param subprotocol The WebSocket subprotocol to use. */ public SignalingManager(URI signalingUri, String subprotocol) { this.signaling = new WebSocketClient(signalingUri, subprotocol); @@ -98,28 +96,13 @@ public SignalingManager(URI signalingUri, String subprotocol) { */ public boolean connect() { try { - CountDownLatch connectLatch = new CountDownLatch(1); - - signaling.connect() - .thenAccept(ws -> { - LOG.info("Connected to signaling server"); - connectLatch.countDown(); - }) - .exceptionally(e -> { - LOG.error("Failed to connect to signaling server", e); - connectLatch.countDown(); - return null; - }); - - // Wait for connection. - connectLatch.await(5, TimeUnit.SECONDS); - - return signaling.isConnected(); + signaling.connect().get(5, TimeUnit.SECONDS); } catch (Exception e) { - LOG.error("Error connecting to signaling server", e); - return false; + LOG.error("Failed to connect to signaling server", e); } + + return signaling.isConnected(); } /** @@ -134,150 +117,66 @@ public void disconnect() { } /** - * Sends a join message to the signaling server. + * Sends a join message to the signaling server to join a specific room. * - * @param roomId The ID of the room to join. + * @param roomId The identifier of the room to join. */ public void sendJoin(String roomId) { - try { - JoinMessage joinMessage = new JoinMessage(userId, roomId, "Java Client"); - - // Convert to JSON and send. - String joinMessageJson = jsonMapper.writeValueAsString(joinMessage); - signaling.sendMessage(joinMessageJson); - - LOG.info("Sent join message for room: {}", roomId); - } - catch (Exception e) { - LOG.error("Error creating JSON join message", e); - } - } - - /** - * Sends a session description (offer or answer) to the remote peer. - * - * @param sdp The session description to send. - */ - public void sendSessionDescription(RTCSessionDescription sdp) { - String type = sdp.sdpType.toString().toLowerCase(); - - try { - SessionDescriptionMessage message = new SessionDescriptionMessage( - type, - userId, - remotePeerId, - sdp.sdp - ); - - String messageJson = jsonMapper.writeValueAsString(message); - signaling.sendMessage(messageJson); + send(new JoinMessage(userId, roomId, "Java Client")); - LOG.info("Sent {} to remote peer", type); - } - catch (Exception e) { - LOG.error("Error creating JSON message", e); - } + LOG.info("Sent join message for room: {}", roomId); } /** - * Sends an ICE candidate to the remote peer. + * Sends a message of a peer to the browser. Called on the peer's event thread, in order. * - * @param candidate The ICE candidate to send. + * @param message The session description or candidate to send. */ - public void sendIceCandidate(RTCIceCandidate candidate) { - try { - IceCandidateMessage message = new IceCandidateMessage( - userId, - remotePeerId, - candidate.sdp, - candidate.sdpMid, - candidate.sdpMLineIndex - ); - - String candidateJson = jsonMapper.writeValueAsString(message); - signaling.sendMessage(candidateJson); - - LOG.info("Sent ICE candidate to remote peer"); - } - catch (Exception e) { - LOG.error("Error creating JSON message", e); - } - } + @Override + public void send(SignalMessage message) { + switch (message.type()) { + case OFFER, ANSWER -> { + String type = message.type() == SignalMessage.Type.OFFER ? "offer" : "answer"; + + send(new SessionDescriptionMessage(type, userId, remotePeerId, message.description().sdp)); + LOG.info("Sent {} to remote peer", type); + } + case CANDIDATE -> { + RTCIceCandidate candidate = message.candidate(); - /** - * Sets up the WebSocket signaling to handle incoming messages. - */ - private void setupSignaling() { - signaling.addMessageListener(message -> { - try { - // Parse the message to extract the type. - String type = jsonMapper.readTree(message).path("type").asText(); - MessageType messageType = MessageType.fromString(type); - - switch (messageType) { - case OFFER -> handleOfferMessage(message); - case ANSWER -> handleAnswerMessage(message); - case ICE_CANDIDATE -> handleIceCandidateMessage(message); - case JOIN -> handleJoinMessage(message); - case LEAVE -> handleLeaveMessage(message); - case HEARTBEAT_ACK -> LOG.info("Received: {}", type); - default -> LOG.info("Received message with unknown type: {}", type); - } + send(new IceCandidateMessage(userId, remotePeerId, candidate.sdp, candidate.sdpMid, + candidate.sdpMLineIndex)); } - catch (Exception e) { - LOG.error("Error parsing message type", e); + default -> { + // The browser client expects no end of candidates. } - }); - - signaling.addCloseListener(() -> { - LOG.info("Signaling connection closed"); - }); - - signaling.addErrorListener(error -> { - LOG.error("Signaling error", error); - }); - } - - /** - * Sets a callback to be invoked when an offer is received. - * - * @param callback The callback to invoke. - */ - public void setOnOfferReceived(Consumer callback) { - this.onOfferReceived = callback; - } - - /** - * Sets a callback to be invoked when an answer is received. - * - * @param callback The callback to invoke. - */ - public void setOnAnswerReceived(Consumer callback) { - this.onAnswerReceived = callback; + } } /** - * Sets a callback to be invoked when an ICE candidate is received. + * Called when a peer is created with this signaling. The browser's messages go to + * the peer created last, which replaces the one before. * - * @param callback The callback to invoke. + * @param inbound Takes the messages the browser sends. */ - public void setOnCandidateReceived(Consumer callback) { - this.onCandidateReceived = callback; + @Override + public void attach(Consumer inbound) { + this.inbound = inbound; } /** - * Sets a callback to be invoked when a join message is received. + * Sets the callback for join messages. * - * @param callback The callback to invoke. + * @param callback The function to be called when a join message is received. */ public void setOnJoinReceived(Consumer callback) { this.onJoinReceived = callback; } /** - * Sets a callback to be invoked when a leave message is received. + * Sets the callback for leave messages. * - * @param callback The callback to invoke. + * @param callback The function to be called when a leave message is received. */ public void setOnLeaveReceived(Consumer callback) { this.onLeaveReceived = callback; @@ -293,7 +192,7 @@ public void setUserId(String userId) { } /** - * Sets the remote peer ID. + * Sets the remote peer ID that this client is communicating with. * * @param remotePeerId The remote peer ID to set. */ @@ -302,114 +201,92 @@ public void setRemotePeerId(String remotePeerId) { } /** - * Gets the user ID for this client. - * - * @return The user ID. + * @return The user ID of this client. */ public String getUserId() { return userId; } /** - * Gets the remote peer ID. - * - * @return The remote peer ID. + * @return The ID of the remote peer this client communicates with. */ public String getRemotePeerId() { return remotePeerId; } /** - * Checks if the signaling connection is established. - * - * @return true if connected, false otherwise. + * @return true if connected to the signaling server. */ public boolean isConnected() { return signaling != null && signaling.isConnected(); } /** - * Handles an incoming offer message. - * - * @param message The JSON message containing an offer. + * Sets up the listeners of the WebSocket client. */ - private void handleOfferMessage(String message) { - if (onOfferReceived != null) { + private void setupSignaling() { + signaling.addMessageListener(message -> { try { - SessionDescriptionMessage sdpMessage = jsonMapper.readValue(message, SessionDescriptionMessage.class); - onOfferReceived.accept(sdpMessage); + String type = jsonMapper.readTree(message).path("type").asText(); + + switch (MessageType.fromString(type)) { + case OFFER -> receiveDescription(RTCSdpType.OFFER, message); + case ANSWER -> receiveDescription(RTCSdpType.ANSWER, message); + case ICE_CANDIDATE -> receiveCandidate(message); + case JOIN -> notify(onJoinReceived, jsonMapper.readValue(message, JoinMessage.class)); + case LEAVE -> notify(onLeaveReceived, jsonMapper.readValue(message, LeaveMessage.class)); + case HEARTBEAT_ACK -> LOG.debug("Received: {}", type); + default -> LOG.info("Received message with unknown type: {}", type); + } } catch (Exception e) { - LOG.error("Error parsing offer message", e); + LOG.error("Error handling signaling message", e); } - } + }); + + signaling.addCloseListener(() -> LOG.info("Signaling connection closed")); + signaling.addErrorListener(error -> LOG.error("Signaling error", error)); } - /** - * Handles an incoming answer message. - * - * @param message The JSON message containing an answer. - */ - private void handleAnswerMessage(String message) { - if (onAnswerReceived != null) { - try { - SessionDescriptionMessage sdpMessage = jsonMapper.readValue(message, SessionDescriptionMessage.class); - onAnswerReceived.accept(sdpMessage); - } - catch (Exception e) { - LOG.error("Error parsing answer message", e); - } - } + private void receiveDescription(RTCSdpType type, String json) throws Exception { + SessionDescriptionMessage message = jsonMapper.readValue(json, SessionDescriptionMessage.class); + + receive(SignalMessage.of(new RTCSessionDescription(type, message.getSdp()))); } - /** - * Handles an incoming ICE candidate message. - * - * @param message The JSON message containing an ICE candidate. - */ - private void handleIceCandidateMessage(String message) { - if (onCandidateReceived != null) { - try { - IceCandidateMessage iceMessage = jsonMapper.readValue(message, IceCandidateMessage.class); - onCandidateReceived.accept(iceMessage); - } - catch (Exception e) { - LOG.error("Error parsing ICE candidate message", e); - } + private void receiveCandidate(String json) throws Exception { + IceCandidateMessage.IceCandidateData data = jsonMapper.readValue(json, IceCandidateMessage.class).getData(); + + receive(SignalMessage.of(new RTCIceCandidate(data.getSdpMid(), data.getSdpMLineIndex(), data.getCandidate()))); + } + + private void receive(SignalMessage message) { + Consumer peer = inbound; + + if (peer != null) { + peer.accept(message); + } + else { + LOG.warn("Dropped a {} that arrived before there was a peer", message.type()); } } - /** - * Handles an incoming join message. - * - * @param message The JSON message containing join information. - */ - private void handleJoinMessage(String message) { - if (onJoinReceived != null) { - try { - JoinMessage joinMessage = jsonMapper.readValue(message, JoinMessage.class); - onJoinReceived.accept(joinMessage); - } - catch (Exception e) { - LOG.error("Error parsing join message", e); - } + private void send(SignalingMessage message) { + try { + signaling.sendMessage(jsonMapper.writeValueAsString(message)) + .exceptionally(error -> { + LOG.error("Failed to send a {} message", message.getType(), error); + return null; + }); + } + catch (Exception e) { + LOG.error("Error creating a JSON {} message", message.getType(), e); } } - /** - * Handles an incoming leave message. - * - * @param message The JSON message indicating a peer has left. - */ - private void handleLeaveMessage(String message) { - if (onLeaveReceived != null) { - try { - LeaveMessage leaveMessage = jsonMapper.readValue(message, LeaveMessage.class); - onLeaveReceived.accept(leaveMessage); - } - catch (Exception e) { - LOG.error("Error parsing join message", e); - } + private static void notify(Consumer callback, T message) { + if (callback != null) { + callback.accept(message); } } -} \ No newline at end of file +} diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/WebSocketClient.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/WebSocketClient.java index e2a87628..029b2c9a 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/WebSocketClient.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/client/WebSocketClient.java @@ -61,9 +61,12 @@ public class WebSocketClient { private final List closeListeners; private final List> errorListeners; - private WebSocket webSocket; + private volatile WebSocket webSocket; private ScheduledExecutorService heartbeatExecutor; - private boolean connected = false; + private volatile boolean connected = false; + + /** The send the next one waits for. */ + private CompletableFuture lastSend = CompletableFuture.completedFuture(null); /** @@ -146,15 +149,25 @@ public CompletableFuture connect() { * * @return A CompletableFuture that completes when the message is sent. */ - public CompletableFuture sendMessage(String message) { - if (!connected || webSocket == null) { + public synchronized CompletableFuture sendMessage(String message) { + WebSocket socket = webSocket; + + if (!connected || socket == null) { return CompletableFuture.failedFuture( new IllegalStateException("Not connected to WebSocket server")); } LOG.debug("Sending message: {}", message); - return webSocket.sendText(message, true); + // A WebSocket takes one message at a time, so each send waits for the + // one before it, whether that succeeded or not. + CompletableFuture send = lastSend + .handle((ignored, error) -> null) + .thenCompose(ignored -> socket.sendText(message, true)); + + lastSend = send; + + return send; } /** diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionManager.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionManager.java deleted file mode 100644 index 5707f857..00000000 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionManager.java +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Copyright 2025 Alex Andres - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.onvoid.webrtc.examples.web.connection; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Consumer; - -import dev.onvoid.webrtc.CreateSessionDescriptionObserver; -import dev.onvoid.webrtc.PeerConnectionFactory; -import dev.onvoid.webrtc.PeerConnectionObserver; -import dev.onvoid.webrtc.RTCAnswerOptions; -import dev.onvoid.webrtc.RTCConfiguration; -import dev.onvoid.webrtc.RTCDataChannel; -import dev.onvoid.webrtc.RTCIceCandidate; -import dev.onvoid.webrtc.RTCIceConnectionState; -import dev.onvoid.webrtc.RTCIceGatheringState; -import dev.onvoid.webrtc.RTCIceServer; -import dev.onvoid.webrtc.RTCOfferOptions; -import dev.onvoid.webrtc.RTCPeerConnection; -import dev.onvoid.webrtc.RTCPeerConnectionState; -import dev.onvoid.webrtc.RTCRtpReceiver; -import dev.onvoid.webrtc.RTCRtpSender; -import dev.onvoid.webrtc.RTCRtpTransceiver; -import dev.onvoid.webrtc.RTCSdpType; -import dev.onvoid.webrtc.RTCSessionDescription; -import dev.onvoid.webrtc.RTCSignalingState; -import dev.onvoid.webrtc.SetSessionDescriptionObserver; -import dev.onvoid.webrtc.examples.web.model.IceCandidateMessage; -import dev.onvoid.webrtc.examples.web.model.SessionDescriptionMessage; -import dev.onvoid.webrtc.media.MediaStream; -import dev.onvoid.webrtc.media.MediaStreamTrack; - -import dev.onvoid.webrtc.media.audio.AudioOptions; -import dev.onvoid.webrtc.media.audio.AudioTrack; -import dev.onvoid.webrtc.media.audio.AudioTrackSource; -import dev.onvoid.webrtc.media.audio.CustomAudioSource; -import dev.onvoid.webrtc.media.video.VideoTrack; -import dev.onvoid.webrtc.media.video.VideoTrackSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Manages WebRTC peer connections, including creation, configuration, - * and session description handling. Also handles signaling messages related to - * peer connections such as offers, answers, and ICE candidates. - * - * @author Alex Andres - */ -public class PeerConnectionManager implements PeerConnectionSignalingHandler { - - private static final Logger LOG = LoggerFactory.getLogger(PeerConnectionManager.class); - - private final PeerConnectionFactory factory; - private final RTCPeerConnection peerConnection; - private final List senders = new ArrayList<>(); - private final List audioSources = new ArrayList<>(); - - private Consumer onLocalDescriptionCreated; - private Consumer onIceCandidateGenerated; - private Consumer onTrackReceived; - - private boolean isInitiator = false; - - - /** - * Creates a new PeerConnectionManager. - */ - public PeerConnectionManager() { - factory = new PeerConnectionFactory(); - - // Create peer connection with default configuration. - RTCIceServer iceServer = new RTCIceServer(); - iceServer.urls.add("stun:stun.l.google.com:19302"); - - RTCConfiguration config = new RTCConfiguration(); - config.iceServers.add(iceServer); - - peerConnection = factory.createPeerConnection(config, new PeerConnectionObserverImpl()); - - LOG.info("PeerConnectionManager created"); - } - - /** - * Adds a media track to the peer connection. - * - * @param track The media track to add. - * @param streamIds The stream IDs to associate with the track. - */ - public void addTrack(MediaStreamTrack track, List streamIds) { - // Keep the returned RTCRtpSender around so it can be disposed in - // close(); it is not owned by the peer connection. - senders.add(peerConnection.addTrack(track, streamIds)); - - LOG.info("Added track: {}", track.getKind()); - } - - /** - * Creates an audio track with the specified options and label. - * - * @param options Configuration options for the audio track source. - * @param label A unique identifier for the audio track. - * - * @return A new AudioTrack instance configured with the provided options. - */ - public AudioTrack createAudioTrack(AudioOptions options, String label) { - AudioTrackSource audioSource = factory.createAudioSource(options); - - // Keep the source around so it can be disposed in close(); it is - // ref-counted and not owned by the audio track. - audioSources.add(audioSource); - - return factory.createAudioTrack(label, audioSource); - } - - /** - * Creates an audio track with a custom audio source that can push audio frames. - * - * @param source The custom audio source providing the audio frames. - * @param label A unique identifier for the audio track. - * - * @return A new AudioTrack instance connected to the provided custom source. - */ - public AudioTrack createAudioTrack(CustomAudioSource source, String label) { - return factory.createAudioTrack(label, source); - } - - /** - * Creates a video track with the specified source and label. - * - * @param source The video track source providing the video frames. - * @param label A unique identifier for the video track. - * - * @return A new VideoTrack instance connected to the provided source. - */ - public VideoTrack createVideoTrack(VideoTrackSource source, String label) { - return factory.createVideoTrack(label, source); - } - - /** - * Closes the peer connection. - */ - public void close() { - for (RTCRtpSender sender : senders) { - sender.dispose(); - } - senders.clear(); - - for (AudioTrackSource audioSource : audioSources) { - audioSource.dispose(); - } - audioSources.clear(); - - if (peerConnection != null) { - peerConnection.close(); - } - - if (factory != null) { - factory.dispose(); - } - - LOG.info("Peer connection closed"); - } - - /** - * Creates an offer to initiate a connection. - */ - public void createOffer() { - RTCOfferOptions options = new RTCOfferOptions(); - - peerConnection.createOffer(options, new CreateSessionDescriptionObserver() { - @Override - public void onSuccess(RTCSessionDescription sdp) { - setLocalDescription(sdp); - } - - @Override - public void onFailure(String error) { - LOG.error("Failed to create offer: {}", error); - } - }); - } - - /** - * Creates an answer in response to an offer. - */ - public void createAnswer() { - RTCAnswerOptions options = new RTCAnswerOptions(); - peerConnection.createAnswer(options, new CreateSessionDescriptionObserver() { - @Override - public void onSuccess(RTCSessionDescription answer) { - setLocalDescription(answer); - } - - @Override - public void onFailure(String error) { - LOG.error("Failed to create answer: {}", error); - } - }); - } - - /** - * Sets the local session description. - * - * @param sdp The session description to set. - */ - private void setLocalDescription(RTCSessionDescription sdp) { - peerConnection.setLocalDescription(sdp, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - LOG.info("Local description set successfully"); - - if (onLocalDescriptionCreated != null) { - onLocalDescriptionCreated.accept(sdp); - } - } - - @Override - public void onFailure(String error) { - LOG.error("Failed to set local session description: {}", error); - } - }); - } - - /** - * Sets the remote session description. - * - * @param sdp The session description to set. - * @param isOffer True if the description is an offer, false otherwise. - */ - public void setRemoteDescription(RTCSessionDescription sdp, boolean isOffer) { - peerConnection.setRemoteDescription(sdp, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - LOG.info("Remote description set successfully"); - - if (isOffer) { - createAnswer(); - } - } - - @Override - public void onFailure(String error) { - LOG.error("Failed to set remote description: {}", error); - } - }); - } - - /** - * Adds an ICE candidate to the peer connection. - * - * @param candidate The ICE candidate to add. - */ - public void addIceCandidate(RTCIceCandidate candidate) { - peerConnection.addIceCandidate(candidate); - LOG.info("Added ICE candidate: {}", candidate.sdp); - } - - /** - * Sets a callback to be invoked when a local session description is created. - * - * @param callback The callback to invoke. - */ - public void setOnLocalDescriptionCreated(Consumer callback) { - this.onLocalDescriptionCreated = callback; - } - - /** - * Sets a callback to be invoked when an ICE candidate is generated. - * - * @param callback The callback to invoke. - */ - public void setOnIceCandidateGenerated(Consumer callback) { - this.onIceCandidateGenerated = callback; - } - - /** - * Sets a callback to be invoked when a media track is received. - * - * @param callback The callback to invoke. - */ - public void setOnTrackReceived(Consumer callback) { - this.onTrackReceived = callback; - } - - /** - * Sets whether this peer is the initiator of the connection. - * - * @param isInitiator true if this peer is the initiator, false otherwise. - */ - public void setInitiator(boolean isInitiator) { - this.isInitiator = isInitiator; - } - - /** - * Gets whether this peer is the initiator of the connection. - * - * @return true if this peer is the initiator, false otherwise. - */ - public boolean isInitiator() { - return isInitiator; - } - - @Override - public void handleOffer(SessionDescriptionMessage message) { - LOG.info("Received offer"); - - try { - String sdpString = message.getSdp(); - String fromPeer = message.getFrom(); - - LOG.info("Parsed offer from: {}", fromPeer); - - // Create remote session description. - RTCSessionDescription sdp = new RTCSessionDescription(RTCSdpType.OFFER, sdpString); - - // Set remote description and create answer - setRemoteDescription(sdp, true); - } - catch (Exception e) { - LOG.error("Error parsing offer JSON", e); - } - } - - @Override - public void handleAnswer(SessionDescriptionMessage message) { - LOG.info("Received answer"); - - try { - String sdpString = message.getSdp(); - - LOG.info("Successfully parsed answer JSON"); - - // Create remote session description. - RTCSessionDescription sdp = new RTCSessionDescription(RTCSdpType.ANSWER, sdpString); - - // Set remote description - setRemoteDescription(sdp, false); - } - catch (Exception e) { - LOG.error("Error parsing answer JSON", e); - } - } - - @Override - public void handleCandidate(IceCandidateMessage message) { - LOG.info("Received ICE candidate"); - - try { - IceCandidateMessage.IceCandidateData data = message.getData(); - String sdpMid = data.getSdpMid(); - int sdpMLineIndex = data.getSdpMLineIndex(); - String candidate = data.getCandidate(); - - LOG.info("Successfully parsed ICE candidate JSON"); - - RTCIceCandidate iceCandidate = new RTCIceCandidate(sdpMid, sdpMLineIndex, candidate); - addIceCandidate(iceCandidate); - } - catch (Exception e) { - LOG.error("Error parsing ICE candidate JSON", e); - } - } - - - - /** - * Implementation of PeerConnectionObserver to handle WebRTC events. - */ - private class PeerConnectionObserverImpl implements PeerConnectionObserver { - - @Override - public void onIceCandidate(RTCIceCandidate candidate) { - LOG.info("New ICE candidate: {}", candidate.sdp); - - if (onIceCandidateGenerated != null) { - onIceCandidateGenerated.accept(candidate); - } - } - - @Override - public void onConnectionChange(RTCPeerConnectionState state) { - LOG.info("Connection state changed to: {}", state); - } - - @Override - public void onIceConnectionChange(RTCIceConnectionState state) { - LOG.info("ICE connection state changed to: {}", state); - } - - @Override - public void onIceGatheringChange(RTCIceGatheringState state) { - LOG.info("ICE gathering state changed to: {}", state); - } - - @Override - public void onSignalingChange(RTCSignalingState state) { - LOG.info("Signaling state changed to: {}", state); - } - - @Override - public void onDataChannel(RTCDataChannel dataChannel) { - LOG.info("Data channel created: {}", dataChannel.getLabel()); - } - - @Override - public void onRenegotiationNeeded() { - LOG.info("Renegotiation needed"); - createOffer(); - } - - @Override - public void onAddTrack(RTCRtpReceiver receiver, MediaStream[] mediaStreams) { - LOG.info("Track added: {}", receiver.getTrack().getKind()); - - // The receiver is a query result the application owns; dispose it - // once its track has been retrieved. - receiver.dispose(); - } - - @Override - public void onRemoveTrack(RTCRtpReceiver receiver) { - LOG.info("Track removed: {}", receiver.getTrack().getKind()); - - receiver.dispose(); - } - - @Override - public void onTrack(RTCRtpTransceiver transceiver) { - RTCRtpReceiver receiver = transceiver.getReceiver(); - MediaStreamTrack track = receiver.getTrack(); - String kind = track.getKind(); - - LOG.info("{} track added to transceiver", kind); - - if (onTrackReceived != null) { - onTrackReceived.accept(track); - } - - // The receiver and transceiver are query results the application - // owns; dispose them once the track has been retrieved. - receiver.dispose(); - transceiver.dispose(); - } - } -} \ No newline at end of file diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionSignalingHandler.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionSignalingHandler.java deleted file mode 100644 index fda95cb3..00000000 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/connection/PeerConnectionSignalingHandler.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2025 Alex Andres - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.onvoid.webrtc.examples.web.connection; - -import dev.onvoid.webrtc.examples.web.model.IceCandidateMessage; -import dev.onvoid.webrtc.examples.web.model.SessionDescriptionMessage; - -/** - * This interface defines methods for processing signaling messages - * directly related to WebRTC peer connections. - * - * @author Alex Andres - */ -public interface PeerConnectionSignalingHandler { - - /** - * Handles an incoming offer from a remote peer. - * - * @param message The session description message containing the offer. - */ - void handleOffer(SessionDescriptionMessage message); - - /** - * Handles an incoming answer from a remote peer. - * - * @param message The session description message containing the answer. - */ - void handleAnswer(SessionDescriptionMessage message); - - /** - * Handles an incoming ICE candidate from a remote peer. - * - * @param message The message containing the ICE candidate. - */ - void handleCandidate(IceCandidateMessage message); - -} \ No newline at end of file diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/AudioGenerator.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/AudioGenerator.java index a0ea68c3..3d9804fa 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/AudioGenerator.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/AudioGenerator.java @@ -22,7 +22,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.client.CustomAudio; +import dev.onvoid.webrtc.client.PcmFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,9 +31,10 @@ /** * A generator that produces synthetic audio samples as a continuous sine wave. *

- * This class creates a 440 Hz (A4 note) sine wave and delivers audio frames - * at regular 10 ms intervals. The generated audio is stereo with 16-bit samples - * at a 48 kHz sample rate. The audio generator runs on a dedicated scheduled + * This class creates a 440 Hz (A4 note) sine wave and pushes 100 ms of it every + * 100 ms. The track cuts the audio into the 10 ms chunks WebRTC sends, so the + * generator need not keep 10 ms time itself. The generated audio is stereo with + * 16-bit samples at a 48 kHz sample rate. The audio generator runs on a dedicated scheduled * thread that can be started and stopped on demand. *

*

@@ -46,8 +48,8 @@ public class AudioGenerator { private static final Logger LOG = LoggerFactory.getLogger(AudioGenerator.class); - /** The custom audio source that receives generated audio frames. */ - private final CustomAudioSource customAudioSource; + /** The track that receives the generated audio. */ + private final CustomAudio track; /** Flag indicating whether the audio generator is running. */ private final AtomicBoolean generatorRunning = new AtomicBoolean(false); @@ -62,7 +64,8 @@ public class AudioGenerator { private static final int DEFAULT_BITS_PER_SAMPLE = 16; private static final int DEFAULT_SAMPLE_RATE = 48000; private static final int DEFAULT_CHANNELS = 2; - private static final int DEFAULT_FRAME_COUNT = 480; // 10ms at 48kHz + private static final int DEFAULT_FRAME_COUNT = 4800; // 100 ms at 48 kHz + private static final PcmFormat FORMAT = PcmFormat.of(DEFAULT_SAMPLE_RATE, DEFAULT_CHANNELS); /** * Sine wave phase tracking for audio generation. @@ -71,13 +74,13 @@ public class AudioGenerator { private double sinePhase = 0.0; - public AudioGenerator(CustomAudioSource audioSource) { - customAudioSource = audioSource; + public AudioGenerator(CustomAudio track) { + this.track = track; } /** - * Starts the audio sample generator thread that pushes audio frames - * to the custom audio source every 10 ms. + * Starts the audio sample generator thread that pushes 100 ms of audio + * to the track every 100 ms. */ public void start() { if (generatorRunning.get()) { @@ -122,19 +125,13 @@ public void start() { } } - // Push the audio data to the custom audio source. - customAudioSource.pushAudio( - audioData, - DEFAULT_BITS_PER_SAMPLE, - DEFAULT_SAMPLE_RATE, - DEFAULT_CHANNELS, - DEFAULT_FRAME_COUNT - ); + // Queue the audio; the track sends it in 10 ms chunks. + track.push(audioData, FORMAT); } catch (Exception e) { LOG.error("Error in audio generator thread", e); } - }, 0, 10, TimeUnit.MILLISECONDS); + }, 0, 100, TimeUnit.MILLISECONDS); LOG.info("Audio generator started"); } diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/MediaManager.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/MediaManager.java index ec626765..13d3b013 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/MediaManager.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/MediaManager.java @@ -16,260 +16,81 @@ package dev.onvoid.webrtc.examples.web.media; -import java.util.List; - -import dev.onvoid.webrtc.media.audio.AudioOptions; -import dev.onvoid.webrtc.media.audio.AudioTrack; -import dev.onvoid.webrtc.media.audio.AudioTrackSink; -import dev.onvoid.webrtc.media.audio.CustomAudioSource; -import dev.onvoid.webrtc.media.video.CustomVideoSource; -import dev.onvoid.webrtc.media.video.VideoDeviceSource; -import dev.onvoid.webrtc.media.video.VideoTrack; -import dev.onvoid.webrtc.media.video.VideoTrackSink; - -import dev.onvoid.webrtc.examples.web.connection.PeerConnectionManager; +import dev.onvoid.webrtc.client.CustomAudio; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.WebRtc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Manages media tracks for WebRTC connections, including audio and video. + * Creates the tracks the Java client sends and feeds them: a 440 Hz tone and an + * animated color pattern, generated in Java and pushed into the tracks. + *

+ * The tracks belong to the runtime that created them. They may be sent on any + * number of peers, which lets the client start a new peer for every participant + * that joins without creating new tracks. * * @author Alex Andres */ -public class MediaManager { +public class MediaManager implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(MediaManager.class); - /** The audio track managed by this class. */ - private AudioTrack audioTrack; - - /** The video track managed by this class. */ - private VideoTrack videoTrack; - - /** The custom audio source for generating audio frames. */ - private CustomAudioSource customAudioSource; - - /** The audio generator responsible for creating and pushing audio frames to the custom audio source. */ - private AudioGenerator audioGenerator; + /** The audio track, fed by the audio generator. */ + private final CustomAudio audio; - /** The custom video source for generating video frames. */ - private CustomVideoSource customVideoSource; - - /** The video generator responsible for creating and pushing video frames to the custom video source. */ - private VideoGenerator videoGenerator; - - - /** - * Creates a new MediaManager. - */ - public MediaManager() { - LOG.info("MediaManager created"); - } - - /** - * Creates and initializes audio and video tracks. - * - * @param peerConnectionManager The peer connection manager to use for creating tracks. - */ - public void createTracks(PeerConnectionManager peerConnectionManager) { - createAudioTrack(peerConnectionManager); - createVideoTrack(peerConnectionManager); - - // Add tracks to the peer connection. - List streamIds = List.of("stream0"); - - peerConnectionManager.addTrack(getAudioTrack(), streamIds); - peerConnectionManager.addTrack(getVideoTrack(), streamIds); - } + /** The video track, fed by the video generator. */ + private final CustomVideo video; - /** - * Creates and initializes tracks with a custom audio source and standard video track. - * - * @param peerConnectionManager The peer connection manager to use for creating tracks. - * @param useCustomAudio Whether to use a custom audio source that can push frames. - */ - public void createTracks(PeerConnectionManager peerConnectionManager, boolean useCustomAudio) { - if (useCustomAudio) { - createCustomAudioTrack(peerConnectionManager); - } - else { - createAudioTrack(peerConnectionManager); - } - - createVideoTrack(peerConnectionManager); - - // Add tracks to the peer connection. - List streamIds = List.of("stream0"); - - peerConnectionManager.addTrack(getAudioTrack(), streamIds); - peerConnectionManager.addTrack(getVideoTrack(), streamIds); - } - - /** - * Creates and initializes tracks with options for both custom audio and video sources. - * - * @param peerConnectionManager The peer connection manager to use for creating tracks. - * @param useCustomAudio Whether to use a custom audio source that can push frames. - * @param useCustomVideo Whether to use a custom video source that can push frames. - */ - public void createTracks(PeerConnectionManager peerConnectionManager, boolean useCustomAudio, boolean useCustomVideo) { - if (useCustomAudio) { - createCustomAudioTrack(peerConnectionManager); - } - else { - createAudioTrack(peerConnectionManager); - } - - if (useCustomVideo) { - createCustomVideoTrack(peerConnectionManager); - } - else { - createVideoTrack(peerConnectionManager); - } - - // Add tracks to the peer connection. - List streamIds = List.of("stream0"); - - peerConnectionManager.addTrack(getAudioTrack(), streamIds); - peerConnectionManager.addTrack(getVideoTrack(), streamIds); - } - - /** - * Creates an audio track with default options. - */ - private void createAudioTrack(PeerConnectionManager peerConnectionManager) { - AudioOptions audioOptions = new AudioOptions(); - audioOptions.echoCancellation = true; - audioOptions.autoGainControl = true; - audioOptions.noiseSuppression = true; + private final AudioGenerator audioGenerator; - audioTrack = peerConnectionManager.createAudioTrack(audioOptions, "audio0"); + private final VideoGenerator videoGenerator; - LOG.info("Audio track created"); - } /** - * Creates an audio track with a custom audio source that can push audio frames. - * Also starts the audio generator thread that pushes audio frames every 10ms. + * Creates the tracks and starts feeding them. * - * @param peerConnectionManager The peer connection manager to use for creating the track. + * @param rtc The runtime that creates the tracks. Its audio mode must allow pushed audio. */ - public void createCustomAudioTrack(PeerConnectionManager peerConnectionManager) { - customAudioSource = new CustomAudioSource(); + public MediaManager(WebRtc rtc) { + audio = rtc.media().customAudio("audio0"); + video = rtc.media().customVideo("video0"); - audioTrack = peerConnectionManager.createAudioTrack(customAudioSource, "audio0"); + audioGenerator = new AudioGenerator(audio); + videoGenerator = new VideoGenerator(video); - // Start the audio generator. - audioGenerator = new AudioGenerator(customAudioSource); audioGenerator.start(); - - LOG.info("Custom audio track created with audio generator"); - } - - /** - * Creates a video track using the default video device. - */ - private void createVideoTrack(PeerConnectionManager peerConnectionManager) { - VideoDeviceSource videoSource = new VideoDeviceSource(); - videoSource.start(); - - videoTrack = peerConnectionManager.createVideoTrack(videoSource, "video0"); - - LOG.info("Video track created"); - } - - /** - * Creates a video track with a custom video source that can push video frames. - * Also starts the video generator thread that pushes video frames at regular intervals. - * - * @param peerConnectionManager The peer connection manager to use for creating the track. - */ - public void createCustomVideoTrack(PeerConnectionManager peerConnectionManager) { - customVideoSource = new CustomVideoSource(); - - videoTrack = peerConnectionManager.createVideoTrack(customVideoSource, "video0"); - - // Start the video generator. - videoGenerator = new VideoGenerator(customVideoSource); videoGenerator.start(); - LOG.info("Custom video track created with video generator"); + LOG.info("Audio and video tracks created with generators"); } /** - * Gets the audio track. - * * @return The audio track. */ - public AudioTrack getAudioTrack() { - return audioTrack; + public CustomAudio audio() { + return audio; } /** - * Gets the video track. - * * @return The video track. */ - public VideoTrack getVideoTrack() { - return videoTrack; - } - - /** - * Adds a sink to the audio track. - * - * @param sink The sink to add. - */ - public void addAudioSink(AudioTrackSink sink) { - if (audioTrack != null) { - audioTrack.addSink(sink); - - LOG.info("Added sink to audio track"); - } - } - - /** - * Adds a sink to the video track. - * - * @param sink The sink to add. - */ - public void addVideoSink(VideoTrackSink sink) { - if (videoTrack != null) { - videoTrack.addSink(sink); - - LOG.info("Added sink to video track"); - } + public CustomVideo video() { + return video; } /** - * Disposes of all media resources. + * Stops the generators and closes the tracks, which stops sending them on every peer. */ - public void dispose() { - // Stop the audio generator if it's running. - if (audioGenerator != null) { - audioGenerator.stop(); - audioGenerator = null; - } - - // Stop the video generator if it's running. - if (videoGenerator != null) { - videoGenerator.stop(); - videoGenerator = null; - } - - if (audioTrack != null) { - audioTrack.dispose(); - audioTrack = null; - } - - if (videoTrack != null) { - videoTrack.dispose(); - videoTrack = null; - } + @Override + public void close() { + audioGenerator.stop(); + videoGenerator.stop(); - customAudioSource = null; - customVideoSource = null; + audio.close(); + video.close(); - LOG.info("Media resources disposed"); + LOG.info("Media resources closed"); } -} \ No newline at end of file +} diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/VideoGenerator.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/VideoGenerator.java index 950ed5da..28a6f7e6 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/VideoGenerator.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/media/VideoGenerator.java @@ -24,7 +24,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.client.CustomVideo; + import dev.onvoid.webrtc.media.video.NativeI420Buffer; import dev.onvoid.webrtc.media.video.VideoFrame; @@ -46,8 +47,8 @@ public class VideoGenerator { private static final Logger LOG = LoggerFactory.getLogger(VideoGenerator.class); - /** The custom video source that receives generated video frames. */ - private final CustomVideoSource customVideoSource; + /** The track that receives the generated frames. */ + private final CustomVideo track; /** Flag indicating whether the video generator is running. */ private final AtomicBoolean generatorRunning = new AtomicBoolean(false); @@ -69,17 +70,17 @@ public class VideoGenerator { /** - * Creates a new VideoGenerator that will push frames to the specified video source. + * Creates a new VideoGenerator that will push frames to the specified track. * - * @param videoSource The custom video source to receive the generated frames. + * @param track The track to receive the generated frames. */ - public VideoGenerator(CustomVideoSource videoSource) { - customVideoSource = videoSource; + public VideoGenerator(CustomVideo track) { + this.track = track; } /** * Starts the video frame generator thread that pushes video frames - * to the custom video source at the specified frame rate. + * to the track at the specified frame rate. */ public void start() { if (generatorRunning.get()) { @@ -99,8 +100,8 @@ public void start() { // Create a new video frame with a color pattern. VideoFrame frame = generateVideoFrame(); - // Push the frame to the custom video source. - customVideoSource.pushFrame(frame); + // Push the frame to the track. + track.push(frame); // Release the frame after pushing it. frame.release(); diff --git a/webrtc-examples/src/main/resources/web/WebRTCClient.js b/webrtc-examples/src/main/resources/web/WebRTCClient.js index 4bc5a683..1fc9b63d 100644 --- a/webrtc-examples/src/main/resources/web/WebRTCClient.js +++ b/webrtc-examples/src/main/resources/web/WebRTCClient.js @@ -151,6 +151,11 @@ class WebRTCClient { const userId = message.data.userInfo.userId; this.setStatus('User joined: ' + userId); + // A remote peer that joins anew calls with a new connection. + if (userId === this.peerConnectionManager.getRemotePeerId()) { + this.resetPeerConnection(); + } + // Add user to participants map if not already present. if (!this.participants.has(userId)) { const userInfo = message.data.userInfo || { userId: userId }; @@ -164,7 +169,11 @@ class WebRTCClient { handleUserLeft(message) { const userId = message.from; this.setStatus('User left: ' + userId); - + + if (userId === this.peerConnectionManager.getRemotePeerId()) { + this.resetPeerConnection(); + } + // Remove user from participants map if present if (this.participants.has(userId)) { this.participants.delete(userId); @@ -221,7 +230,11 @@ class WebRTCClient { } async startWebRTC() { - this.peerConnectionManager.initialize(); + // A later offer renegotiates the connection there is, for example when + // the Java client adds a track; only the first one creates it. + if (!this.peerConnectionManager.peerConnection) { + this.peerConnectionManager.initialize(); + } // Add local media if available // const mediaAdded = await this.peerConnectionManager.addLocalMedia(); @@ -240,6 +253,16 @@ class WebRTCClient { // } } + resetPeerConnection() { + this.peerConnectionManager.close(); + + if (this.remoteVideo.srcObject) { + this.remoteVideo.srcObject = null; + } + + this.videoContainer.style.display = 'none'; + } + handleTrack(event) { this.setStatus('Received remote track'); console.log('Track event:', event); From bd6c34dd496bb70d7911c77ccad66b76e739ed77 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:16:20 +0200 Subject: [PATCH 06/25] Check decoding against the handler in the slow frame test The test asserted more than 10 fps, which the H.264 encoder on macOS arm64 had not ramped up to yet. It now checks that the decoder gets well ahead of the slow handler within the same time. --- .../dev/onvoid/webrtc/client/MediaTests.java | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java index 72eca15d..e0541ecc 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java @@ -33,6 +33,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import dev.onvoid.webrtc.RTCStats; +import dev.onvoid.webrtc.RTCStatsType; import dev.onvoid.webrtc.media.video.NativeI420Buffer; import dev.onvoid.webrtc.media.video.VideoFrame; @@ -118,10 +120,41 @@ void slowFrameHandlerGetsNewest() throws Exception { await(enough); - // The connection kept up while the handler fell behind. - PeerStats stats = await(peers[1].stats()); - assertTrue(stats.inboundVideo().get(0).framesPerSecond() > 10, - () -> "Inbound video: " + stats.inboundVideo()); + // The decoder keeps going while the handler is busy: within the same + // time it gets well ahead of the handler, at whatever rate the encoder + // has ramped up to on this machine. + long decodedBefore = framesDecoded(await(peers[1].stats())); + int handledBefore = handled.get(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + long decodedMeanwhile; + int handledMeanwhile; + + do { + Thread.sleep(100); + decodedMeanwhile = framesDecoded(await(peers[1].stats())) - decodedBefore; + handledMeanwhile = handled.get() - handledBefore; + } + while ((decodedMeanwhile < 10 || decodedMeanwhile < handledMeanwhile + 5) && System.nanoTime() < deadline); + + long decoded = decodedMeanwhile; + int handledFrames = handledMeanwhile; + + assertTrue(decoded >= 10 && decoded >= handledFrames + 5, + () -> "Meanwhile decoded " + decoded + " frames and handled " + handledFrames); + } + + private static long framesDecoded(PeerStats stats) { + long frames = 0; + + for (RTCStats entry : stats.report().getStats().values()) { + Object decoded = entry.getAttributes().get("framesDecoded"); + + if (entry.getType() == RTCStatsType.INBOUND_RTP && decoded instanceof Number) { + frames += ((Number) decoded).longValue(); + } + } + + return frames; } @Test From f9aeab1a0d687efc822e1df4d501e388866e15a9 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:24:48 +0200 Subject: [PATCH 07/25] Bound complete ICE gathering by a timeout An unreachable STUN or TURN server held up the end of gathering past the operation timeout, so Gathering.COMPLETE failed on networks that block UDP. It now waits at most PeerOptions.gatheringTimeout, 3 seconds by default, and goes with the candidates gathered by then. --- design/client-api.md | 2 ++ docs/guide/client-api.md | 3 ++- .../dev/onvoid/webrtc/client/Gathering.java | 7 ++++- .../dev/onvoid/webrtc/client/PeerImpl.java | 8 ++++++ .../dev/onvoid/webrtc/client/PeerOptions.java | 27 +++++++++++++++++++ .../dev/onvoid/webrtc/client/PeerTests.java | 20 ++++++++++++++ 6 files changed, 65 insertions(+), 2 deletions(-) diff --git a/design/client-api.md b/design/client-api.md index 0fc98723..e52aa701 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -765,6 +765,8 @@ Where the code differs from the API above: `CustomAudio.queuedMillis()`/`clear()`, `Message.size()`, `Negotiator.answer(offer, gathering)`. - An impolite peer that ignored an offer reports the two-impolite-peers collision after 5 seconds without an answer. +- `Gathering.COMPLETE` waits at most `PeerOptions.Builder.gatheringTimeout` (3 seconds by default), + since an unreachable STUN or TURN server holds up the end of gathering far longer. Covered by tests (`PeerTests`, `MediaTests`, `SignalMessageTests`): - data channel round trip; diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 6b13bad4..ea959173 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -101,6 +101,7 @@ Peer peer = rtc.createPeer(PeerOptions.builder() | `polite(boolean)` | The role in perfect negotiation, see below. | | `negotiation(Negotiation)` | `AUTOMATIC` by default, or `MANUAL`, see [Manual Negotiation](#manual-negotiation). | | `receive(MediaKind...)` | Receives audio or video without sending any, as a viewer does. | +| `gatheringTimeout(Duration)` | How long `Gathering.COMPLETE` waits for ICE gathering, 3 seconds by default. See [Manual Negotiation](#manual-negotiation). | | `configure(Consumer)` | Anything else the [peer connection configuration](/guide/networking/peer-connection-config) has. | ### Signaling @@ -411,7 +412,7 @@ DataChannel positions = peer.createChannel("positions", ChannelOptions.builder() ## Manual Negotiation -WHIP, WHEP and most SFUs dictate their own offer and answer exchange. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. This WHEP client receives a stream: +WHIP, WHEP and most SFUs dictate their own offer and answer exchange. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. A STUN or TURN server that cannot be reached, as on networks that block UDP, holds up the end of gathering for a long time, so `COMPLETE` waits at most for the peer's gathering timeout and then goes with the candidates it has. This WHEP client receives a stream: ```java Peer viewer = rtc.createPeer(PeerOptions.builder() diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java index 86fb7d54..0087ed62 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Gathering.java @@ -24,7 +24,12 @@ public enum Gathering { /** At once. Candidates follow through the {@link Signaling}, if there is one. */ TRICKLE, - /** Once ICE gathering is complete, with all local candidates in the description. */ + /** + * Once ICE gathering is complete, with all local candidates in the + * description, as servers that take no trickled candidates need. It waits + * at most for the {@link PeerOptions.Builder#gatheringTimeout(java.time.Duration) + * gathering timeout}, and then hands back the candidates gathered so far. + */ COMPLETE } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java index 9385da45..1afcb458 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java @@ -813,6 +813,14 @@ private CompletableFuture waitForGathering() { CompletableFuture future = new CompletableFuture<>(); gatheringWaiters.add(future); + // A STUN or TURN server that cannot be reached holds up the end of + // gathering for long; the description then goes with what there is. + rtc.timer.schedule(() -> control.execute(() -> { + if (gatheringWaiters.remove(future)) { + future.complete(null); + } + }), options.gatheringTimeout.toMillis(), TimeUnit.MILLISECONDS); + return future; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java index 5295479a..94c86b32 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerOptions.java @@ -18,6 +18,7 @@ import static java.util.Objects.requireNonNull; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -58,6 +59,8 @@ public final class PeerOptions { final Consumer customizer; + final Duration gatheringTimeout; + private PeerOptions(Builder builder) { this.name = builder.name; @@ -69,6 +72,7 @@ private PeerOptions(Builder builder) { this.receive = Collections.unmodifiableSet(builder.receive.isEmpty() ? EnumSet.noneOf(MediaKind.class) : EnumSet.copyOf(builder.receive)); this.customizer = builder.customizer; + this.gatheringTimeout = builder.gatheringTimeout; } /** @@ -126,6 +130,8 @@ public static final class Builder { private Consumer customizer; + private Duration gatheringTimeout = Duration.ofSeconds(3); + private Builder() { } @@ -227,6 +233,27 @@ public Builder receive(MediaKind... kinds) { return this; } + /** + * Sets how long {@link Gathering#COMPLETE} waits for ICE gathering to + * complete; 3 seconds by default. A STUN or TURN server that cannot be + * reached holds up the end of gathering for much longer, and a + * description with the candidates gathered so far usually connects. + * + * @param timeout the longest wait. + * + * @return this builder. + */ + public Builder gatheringTimeout(Duration timeout) { + requireNonNull(timeout, "timeout"); + + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("The timeout must be positive, got " + timeout); + } + + this.gatheringTimeout = timeout; + return this; + } + /** * Changes anything else of the {@link RTCConfiguration} the peer is * created with, after this builder applied its own values. diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java index 6014c05d..af20bd9b 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -274,6 +275,25 @@ void manualNegotiation() throws Exception { assertThrows(IllegalStateException.class, () -> pair()[0].negotiator()); } + @Test + void unreachableStunServerBoundsGathering() throws Exception { + // 192.0.2.1 is reserved for documentation and never answers, which + // holds up the end of gathering far beyond the timeout. + Peer peer = rtc.createPeer(PeerOptions.builder() + .negotiation(Negotiation.MANUAL) + .iceServer("stun:192.0.2.1:3478") + .gatheringTimeout(Duration.ofMillis(500)) + .receive(MediaKind.VIDEO) + .build()); + + long start = System.nanoTime(); + RTCSessionDescription offer = await(peer.negotiator().createOffer(Gathering.COMPLETE)); + long millis = (System.nanoTime() - start) / 1_000_000; + + assertTrue(millis < 5000, () -> "Gathering took " + millis + " ms"); + assertTrue(offer.sdp.contains("typ host"), "The offer has no host candidates"); + } + @Test void closeFailsPendingWork() throws Exception { Peer peer = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build()); From 3bf42aa4712d8822166ff4f1b90266f1d126086a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:24:48 +0200 Subject: [PATCH 08/25] Port the WHEP example to the client API It negotiates by hand with complete gathering, receives video and audio, and ends the session with a DELETE of the resource the server created. --- docs/guide/examples.md | 17 +- .../onvoid/webrtc/examples/WhepExample.java | 377 +++++------------- 2 files changed, 120 insertions(+), 274 deletions(-) diff --git a/docs/guide/examples.md b/docs/guide/examples.md index d35615c5..d919f7d4 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -37,15 +37,20 @@ This example provides a foundation for building WebRTC applications that need to ## WHEP -The [`WhepExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java) demonstrates an implementation of WebRTC HTTP Egress Protocol (WHEP) client, which is a standardized protocol for WebRTC ingestion. +The [`WhepExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java) implements a client of the WebRTC HTTP Egress Protocol (WHEP), which receives a stream from a server, with the [Client API](/guide/client-api). **Key features demonstrated:** -- Setting up a WebRTC peer connection -- Creating and sending an SDP offer to a WHEP endpoint -- Receiving and processing an SDP answer -- Establishing media streaming over WebRTC +- Creating a receive-only peer that negotiates by hand +- Creating an offer that already holds the local candidates, as WHEP servers expect +- Sending the offer to a WHEP endpoint and applying the answer it returns +- Receiving the stream's video and audio +- Ending the session with an HTTP `DELETE` of the resource the server created -This example is useful for applications that need to receive media streams from WHEP-compatible servers, such as live streaming platforms. +This example is useful for applications that need to receive media streams from WHEP-compatible servers, such as live streaming platforms. The endpoint and the number of seconds to receive are optional: + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.WhepExample" -D"exec.args=http://localhost:8889/mystream/whep 10" +``` ## Codec List diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java index f8fdae3d..d591e1f1 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java @@ -21,310 +21,151 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -import dev.onvoid.webrtc.*; -import dev.onvoid.webrtc.media.MediaStream; -import dev.onvoid.webrtc.media.MediaStreamTrack; -import dev.onvoid.webrtc.media.video.VideoDeviceSource; -import dev.onvoid.webrtc.media.video.VideoTrack; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import dev.onvoid.webrtc.RTCSdpType; +import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.client.Gathering; +import dev.onvoid.webrtc.client.MediaKind; +import dev.onvoid.webrtc.client.Negotiation; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.WebRtc; /** - * Example implementation of WebRTC HTTP Egress Protocol (WHEP) client. + * Example implementation of a WebRTC HTTP Egress Protocol (WHEP) client, with the client API. *

* This class demonstrates: *

    - *
  • Setting up a WebRTC peer connection
    • - *
  • Creating and sending an SDP offer to a WHEP endpoint
    • - *
  • Receiving and processing an SDP answer
    • - *
  • Establishing media streaming over WebRTC
    • + *
  • Creating a receive-only peer that negotiates by hand
    • + *
  • Creating an offer that already holds all local candidates, as WHEP servers expect
    • + *
  • Sending the offer to a WHEP endpoint and applying the answer it returns
    • + *
  • Receiving the stream's video and audio
    • + *
  • Ending the session with an HTTP DELETE of the resource the server created
    • *
*

- * The example creates a receive-only peer connection that can accept - * incoming video streams from a WHEP-compatible server. + * Run it against a WHEP endpoint, such as one of MediaMTX, for a number of seconds: + * 

+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.WhepExample" -D"exec.args=http://localhost:8889/mystream/whep 10"
+ *
+ * + * @see WHEP and WHIP * - * @see WHEP Specification - * * @author Alex Andres */ public class WhepExample { - private static final String WHEP_ENDPOINT_URL = "http://localhost:8889/mystream/whep"; - - /** Factory for creating peer connections and media objects. */ - private PeerConnectionFactory factory; - - /** The WebRTC peer connection that handles media communication. */ - private RTCPeerConnection peerConnection; - - /** The local SDP offer to be sent to the remote endpoint. */ - private RTCSessionDescription localOffer; - - // Synchronization objects for async operations. - private final CountDownLatch offerCreatedLatch = new CountDownLatch(1); - private final CountDownLatch localDescriptionSetLatch = new CountDownLatch(1); - private final CountDownLatch remoteDescriptionSetLatch = new CountDownLatch(1); - - - public static void main(String[] args) { - WhepExample example = new WhepExample(); - - try { - example.run(); - } - catch (Exception e) { - Logger.getLogger("WHEPExample").log(Level.SEVERE, "Error running WHEP example", e); - } - finally { - example.cleanup(); - } - } - - public void run() throws Exception { - System.out.println("Starting WebRTC Peer Connection Example"); - - initializePeerConnectionFactory(); - createPeerConnection(); - createOffer(); - - // Wait for the offer to be created. - if (!offerCreatedLatch.await(5, TimeUnit.SECONDS)) { - throw new IllegalStateException("Timeout waiting for offer creation."); - } - - // Set the local description (the offer). - setLocalDescription(localOffer); - - // Wait for the local description to be set. - if (!localDescriptionSetLatch.await(5, TimeUnit.SECONDS)) { - throw new IllegalStateException("Timeout waiting for local description to be set."); - } - - System.out.println("Local offer created and set."); - //System.out.println("SDP Offer: " + localOffer.sdp); - System.out.println("Sending local offer to the remote endpoint."); - - String answerSdp = sendOfferEndpoint(localOffer.sdp); - - //System.out.println("SDP Answer: " + answerSdp); - - // Set the remote description (the answer). - setRemoteDescription(new RTCSessionDescription(RTCSdpType.ANSWER, answerSdp)); - - // Wait for the remote description to be set. - if (!remoteDescriptionSetLatch.await(5, TimeUnit.SECONDS)) { - throw new IllegalStateException("Timeout waiting for remote description to be set."); - } - - System.out.println("Remote answer set. Peer connection established!"); - System.out.println("Media should now be exchanged between peers."); - - // Wait a bit to see connection state changes. - Thread.sleep(10000); - - System.out.println("WebRTC Peer Connection Example completed."); - } - - private void initializePeerConnectionFactory() { - System.out.println("Initializing PeerConnectionFactory."); - factory = new PeerConnectionFactory(); - } - - private void createPeerConnection() { - System.out.println("Creating peer connection."); - - // Create ICE servers configuration. - RTCConfiguration config = new RTCConfiguration(); - - // Add Google's public STUN server. - RTCIceServer iceServer = new RTCIceServer(); - iceServer.urls.add("stun:stun.l.google.com:19302"); - config.iceServers.add(iceServer); - - // Create the peer connection with our observer. - peerConnection = factory.createPeerConnection(config, new PeerConnectionObserverImpl()); - - // Create a video track from a video device source (e.g., webcam). - // Since we are only receiving video in this example, the source will be a dummy video source. - VideoDeviceSource videoSource = new VideoDeviceSource(); - VideoTrack videoTrack = factory.createVideoTrack("videoTrack", videoSource); - videoTrack.addSink(videoFrame -> System.out.println("Received video frame: " + videoFrame)); - - // Only interested in receiving video, so we set up a transceiver for that. - RTCRtpTransceiverInit transceiverInit = new RTCRtpTransceiverInit(); - transceiverInit.direction = RTCRtpTransceiverDirection.RECV_ONLY; - - // Add the transceiver to the peer connection with the video track. - RTCRtpTransceiver transceiver = peerConnection.addTransceiver(videoTrack, transceiverInit); - - // Set up a sink to handle incoming video frames. - RTCRtpReceiver receiver = transceiver.getReceiver(); - MediaStreamTrack track = receiver.getTrack(); - if (track instanceof VideoTrack vTrack) { - vTrack.addSink(videoFrame -> { - System.out.println("Received video frame: " + videoFrame); - - // IMPORTANT: Always release the frame when done to prevent memory leaks - videoFrame.release(); + private static final String DEFAULT_ENDPOINT = "http://localhost:8889/mystream/whep"; + + private static final HttpClient HTTP = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + + public static void main(String[] args) throws Exception { + URI endpoint = URI.create(args.length > 0 ? args[0] : DEFAULT_ENDPOINT); + int seconds = args.length > 1 ? Integer.parseInt(args[1]) : 10; + + // The runtime plays the received audio to the speakers. + try (WebRtc rtc = WebRtc.create()) { + // A WHEP server dictates the exchange: one offer, one answer, over HTTP. + Peer viewer = rtc.createPeer(PeerOptions.builder() + .name("whep") + .iceServer("stun:stun.l.google.com:19302") + .negotiation(Negotiation.MANUAL) + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .build()); + + viewer.onStateChange(state -> System.out.println("Connection is " + state)); + viewer.onTrack(track -> { + System.out.println("Receiving " + track.kind()); + + if (track.isVideo()) { + AtomicInteger frames = new AtomicInteger(); + + track.asVideo().onFrame(frame -> { + if (frames.incrementAndGet() % 100 == 1) { + System.out.printf("Video frame %d: %dx%d%n", frames.get(), + frame.buffer.getWidth(), frame.buffer.getHeight()); + } + }); + } + else { + AtomicLong millis = new AtomicLong(); + + track.asAudio().onAudio(chunk -> { + if (millis.addAndGet(chunk.frames() * 1000L / chunk.sampleRate()) % 2000 == 0) { + System.out.printf("Received %d ms of audio%n", millis.get()); + } + }); + } }); - } - // The receiver and transceiver are query results this example owns; - // dispose them once the track has been retrieved. The track itself - // is unaffected and keeps delivering frames to its sink. - receiver.dispose(); - transceiver.dispose(); - } + // The main thread may wait for the futures; a handler must not. + RTCSessionDescription offer = viewer.negotiator() + .createOffer(Gathering.COMPLETE) + .get(10, TimeUnit.SECONDS); - private void createOffer() { - System.out.println("Creating offer."); + System.out.println("Sending the offer to " + endpoint); - // Create offer options (use default options). - RTCOfferOptions options = new RTCOfferOptions(); + HttpResponse response = post(endpoint, offer.sdp); + URI resource = response.headers().firstValue("Location").map(endpoint::resolve).orElse(null); - // Create the offer. - peerConnection.createOffer(options, new CreateSessionDescriptionObserver() { - @Override - public void onSuccess(RTCSessionDescription description) { - System.out.println("Offer created successfully."); - localOffer = description; - offerCreatedLatch.countDown(); - } + try { + viewer.negotiator() + .applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, response.body())) + .get(10, TimeUnit.SECONDS); + viewer.whenConnected().get(10, TimeUnit.SECONDS); - @Override - public void onFailure(String error) { - System.err.println("Failed to create offer: " + error); - offerCreatedLatch.countDown(); + Thread.sleep(seconds * 1000L); } - }); - } - - private void setLocalDescription(RTCSessionDescription description) { - System.out.println("Setting local description."); - - peerConnection.setLocalDescription(description, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - System.out.println("Local description set successfully."); - localDescriptionSetLatch.countDown(); + finally { + if (resource != null) { + end(resource); + } } - - @Override - public void onFailure(String error) { - System.err.println("Failed to set local description: " + error); - localDescriptionSetLatch.countDown(); - } - }); + } + // The runtime closed the peer and the factory. } - private String sendOfferEndpoint(String sdpOffer) throws Exception { - HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(10)) - .build(); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(WHEP_ENDPOINT_URL)) + /** + * Sends the offer and returns the server's answer. + */ + private static HttpResponse post(URI endpoint, String sdp) throws Exception { + HttpRequest request = HttpRequest.newBuilder(endpoint) .header("Content-Type", "application/sdp") - .POST(HttpRequest.BodyPublishers.ofString(sdpOffer)) + .POST(HttpRequest.BodyPublishers.ofString(sdp)) .timeout(Duration.ofSeconds(30)) .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = HTTP.send(request, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == 200 || response.statusCode() == 201) { - System.out.println("WHEP request successful"); - return response.body(); - } - else { - throw new RuntimeException("WHEP request failed with status: " + response.statusCode()); + if (response.statusCode() != 200 && response.statusCode() != 201) { + throw new IllegalStateException("The WHEP endpoint answered with status " + response.statusCode() + + ": " + response.body()); } - } - - private void setRemoteDescription(RTCSessionDescription description) { - System.out.println("Setting remote description."); - - peerConnection.setRemoteDescription(description, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - System.out.println("Remote description set successfully."); - remoteDescriptionSetLatch.countDown(); - } - @Override - public void onFailure(String error) { - System.err.println("Failed to set remote description: " + error); - remoteDescriptionSetLatch.countDown(); - } - }); + return response; } - private void cleanup() { - System.out.println("Cleaning up resources."); - - if (peerConnection != null) { - peerConnection.close(); - peerConnection = null; - } - - if (factory != null) { - factory.dispose(); - factory = null; - } - } - - - /** - * Implementation of PeerConnectionObserver to handle events from the peer connection. + * Ends the session on the server, which frees what it holds for this viewer. */ - private static class PeerConnectionObserverImpl implements PeerConnectionObserver { - - @Override - public void onIceCandidate(RTCIceCandidate candidate) { - System.out.println("ICE candidate: " + candidate.sdp); - // In a real application, we would send this candidate to the remote peer - } - - @Override - public void onConnectionChange(RTCPeerConnectionState state) { - System.out.println("Connection state changed: " + state); - } - - @Override - public void onIceConnectionChange(RTCIceConnectionState state) { - System.out.println("ICE connection state changed: " + state); - } - - @Override - public void onIceGatheringChange(RTCIceGatheringState state) { - System.out.println("ICE gathering state changed: " + state); - } - - @Override - public void onSignalingChange(RTCSignalingState state) { - System.out.println("Signaling state changed: " + state); - } + private static void end(URI resource) { + try { + HttpRequest request = HttpRequest.newBuilder(resource) + .DELETE() + .timeout(Duration.ofSeconds(5)) + .build(); - @Override - public void onDataChannel(RTCDataChannel dataChannel) { - System.out.println("Data channel created: " + dataChannel.getLabel()); - } + int status = HTTP.send(request, HttpResponse.BodyHandlers.discarding()).statusCode(); - @Override - public void onRenegotiationNeeded() { - System.out.println("Renegotiation needed."); + System.out.println("Ended the session, status " + status); } - - @Override - public void onAddTrack(RTCRtpReceiver receiver, MediaStream[] mediaStreams) { - System.out.println("Track added."); - - // The receiver is a query result the application owns; dispose it - // once it is no longer needed. - receiver.dispose(); + catch (Exception e) { + System.err.println("Ending the session failed: " + e); } } -} \ No newline at end of file +} From 62d4ac9e3dcb49b1a35a9176b6786ff3c7ee41c8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:26:06 +0200 Subject: [PATCH 09/25] Start the get-started guide with the client API A first call with the client API comes first; the core API walkthrough follows unchanged. The introduction names both APIs. --- docs/guide/get-started.md | 105 ++++++++++++++++++++++++++++++++----- docs/guide/introduction.md | 2 + 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/docs/guide/get-started.md b/docs/guide/get-started.md index 31388e32..0a3c285d 100644 --- a/docs/guide/get-started.md +++ b/docs/guide/get-started.md @@ -1,6 +1,6 @@ # Get Started -This guide will help you get started with webrtc-java quickly. We'll cover installation, basic setup, and simple examples to demonstrate core functionality. +This guide will help you get started with webrtc-java quickly. We'll cover installation, a first call with the client API, and the same steps with the core API. ## Supported Platforms Maven Central artifacts contain native libraries that can be loaded on the following platforms: @@ -114,11 +114,86 @@ implementation group: "dev.onvoid.webrtc", name: "webrtc-java", version: "{{ VER implementation group: "dev.onvoid.webrtc", name: "webrtc-java", version: "{{ VERSION }}", classifier: "linux-aarch32" ``` -## Basic Setup +## Your First Call + +webrtc-java has two APIs. The [Client API](/guide/client-api) in `dev.onvoid.webrtc.client` negotiates by itself, keeps your code off WebRTC's threads, and closes everything it created with one call. The core API in `dev.onvoid.webrtc` mirrors WebRTC's own classes one to one, for full control. Start with the client API; [Using the Core API](#using-the-core-api) below shows the same steps with the core API. + +### Create the Runtime + +A `WebRtc` runtime owns everything the client API creates. Closing it, here at the end of the try-with-resources block, closes it all: + +```java +import dev.onvoid.webrtc.client.*; + +try (WebRtc rtc = WebRtc.create()) { + // Peers, tracks and channels go here. +} +``` + +`WebRtc.create()` captures the microphone and plays to the speakers. `WebRtc.builder().audio(AudioMode.HEADLESS).build()` needs no audio hardware at all, which suits servers and tests. + +### Connect to a Peer + +Two peers find each other through signaling: they exchange offers, answers and ICE candidates over a channel you provide, such as a WebSocket. The library has no transport of its own; you hand it a function that sends, and pass it what the other side sent: + +```java +Peer peer = rtc.createPeer(PeerOptions.builder() + .iceServer("stun:stun.l.google.com:19302") + .polite(true) + .signaling(message -> socket.send(message.toJson())) + .build()); + +socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); +``` + +The peer negotiates by itself whenever its tracks or channels change. When both sides send an offer at the same moment, the polite one gives way, so set `polite(true)` on exactly one of the two peers. The JSON messages are the ones browsers use, so the other side can be a web page. + +### Send and Receive Media + +```java +peer.addTrack(rtc.media().microphone()); +peer.addTrack(rtc.media().camera()); + +peer.onTrack(track -> { + System.out.println("Receiving " + track.kind()); + + if (track.isVideo()) { + track.asVideo().onFrame(frame -> render(frame)); + } +}); + +peer.onStateChange(state -> System.out.println("Call is " + state)); +``` + +Handlers run on a thread of the peer's own, one at a time and in order. They may take their time and may call any method, so a slow handler never stalls WebRTC. + +### Exchange Messages + +```java +DataChannel chat = peer.createChannel("chat"); + +chat.onMessage(message -> System.out.println("Peer says: " + message.text())); +chat.send("Hello"); +``` + +Messages sent before the channel opens are sent once it does. + +### Hang Up + +`peer.close()` ends the call; closing the runtime ends every call and releases the camera, the microphone and the native resources. Operations return `CompletableFuture`s, which a thread such as `main` may wait for: + +```java +peer.whenConnected().get(10, TimeUnit.SECONDS); +// ... +peer.close(); +``` + +The [Client API](/guide/client-api) guide covers everything else: pushing your own audio and video, manual negotiation for WHIP and WHEP, statistics and errors. The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) runs a whole call between two peers in one process, without any hardware. + +## Using the Core API + +The core API leaves negotiation, threads and the lifetime of native objects to you. The steps below make the same call with it. -::: tip -This guide uses the core API, which mirrors WebRTC's own. The [Client API](/guide/client-api) builds on it: it negotiates by itself, keeps your handlers off WebRTC's threads, and closes everything it created with one call. Most applications are simpler with it. -::: ### Initialize the WebRTC Library @@ -153,7 +228,7 @@ RTCPeerConnection peerConnection = factory.createPeerConnection(config, new Peer }); ``` -## Signaling +### Signaling WebRTC requires a signaling mechanism to exchange connection information between peers. The library doesn't provide this, so you'll need to implement it using your preferred method (WebSockets, HTTP, etc.). @@ -219,7 +294,7 @@ signalingChannel.onIceCandidate(candidateMessage -> { }); ``` -### Letting the Peer Connection Create the Description +#### Letting the Peer Connection Create the Description `setLocalDescription` can also be called with only an observer. It then creates the offer or the answer, whichever the signaling state calls for, and applies it in one step, so nothing can change in between. That makes it the simpler way to implement "perfect negotiation". Read the result with `getLocalDescription()` once the observer reports success: @@ -237,7 +312,7 @@ peerConnection.setLocalDescription(new SetSessionDescriptionObserver() { }); ``` -### Adding Candidates and Knowing the Outcome +#### Adding Candidates and Knowing the Outcome `addIceCandidate(candidate)` reports nothing and fails silently when the remote description is not set yet. The form that takes an `AddIceCandidateObserver` is queued behind any pending offer/answer operation, as the standard requires, so a candidate can be added right after `setRemoteDescription()` without waiting for it, and the observer says whether it was accepted: @@ -267,9 +342,9 @@ public void onSelectedCandidatePairChanged(RTCCandidatePairChangeEvent event) { } ``` -## Media Streams +### Media Streams -### Accessing Media Devices +#### Accessing Media Devices To query media devices (cameras and microphones): @@ -291,7 +366,7 @@ for (AudioDevice device : audioDevices) { } ``` -### Creating Media Tracks +#### Creating Media Tracks To create audio and video tracks: @@ -321,7 +396,7 @@ AudioTrack audioTrack = factory.createAudioTrack("audio0", audioSource); Audio devices can be specified via the `AudioDeviceModule`, see the guide on [Audio Devices][audio-devices] for more details. ::: -### Adding Tracks to Peer Connection +#### Adding Tracks to Peer Connection ```java import java.util.ArrayList; @@ -335,7 +410,7 @@ RTCRtpSender videoSender = peerConnection.addTrack(videoTrack, streamIds); RTCRtpSender audioSender = peerConnection.addTrack(audioTrack, streamIds); ``` -### Receiving Tracks +#### Receiving Tracks A remote track arrives in `onTrack` with the transceiver that carries it. To learn when its media actually starts flowing, for example to show a video only once there is something to show, set an `RTCRtpReceiverObserver`; an `RTCRtpSenderObserver` does the same for the first packet a sender sends. Either is called right away when that has already happened: @@ -357,7 +432,7 @@ public void onTrack(RTCRtpTransceiver transceiver) { An observer belongs to the `RTCRtpReceiver` or `RTCRtpSender` object it was set through, and is removed when that object is disposed. `getId()` identifies a sender or receiver, and `RTCRtpSender.getStreams()` returns the stream IDs its track was added with. -## Cleanup +### Cleanup Always properly dispose of resources when you're done: @@ -385,6 +460,7 @@ factory.dispose(); Now that you have a basic understanding of webrtc-java, you can explore more advanced features: +- [Client API][client-api] - Everything the client API does, and how its threads work - [Data Channels][data-channels] - Learn more about data channels - [RTC Stats][rtc-stats] - Monitor connection quality - [Desktop Capture][desktop-capture] - Share screens and windows @@ -392,6 +468,7 @@ Now that you have a basic understanding of webrtc-java, you can explore more adv For a complete API reference, check the [JavaDoc][javadoc]. +[client-api]: /guide/client-api [audio-devices]: /guide/audio/audio-devices [data-channels]: /guide/data/data-channels [rtc-stats]: /guide/monitoring/rtc-stats diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 1065968a..8b35fede 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -4,6 +4,8 @@ webrtc-java is a Java wrapper for the [WebRTC Native API](https://webrtc.github. The library provides a comprehensive set of Java classes that map to the WebRTC C++ API, making it possible to establish peer-to-peer connections, transmit audio and video, share screens, and exchange arbitrary data between applications. +It has two APIs. The [Client API](/guide/client-api) negotiates connections by itself, keeps application code off WebRTC's threads, and releases everything it created with one call, which makes it the place to start. The core API maps WebRTC's classes one to one, for applications that need full control. + ## Overview For more detailed information, check out the documentation: From 071a232e1f486899a781029919592fc986064a9f Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:28:36 +0200 Subject: [PATCH 10/25] Send DTMF tones through client API senders Sender.sendTones completes once the tones have been sent, and onTone reports each tone on the peer's event thread. Tones still queued are replaced, and the earlier future is cancelled. --- design/client-api.md | 9 +- docs/guide/client-api.md | 10 ++ .../java/dev/onvoid/webrtc/client/Sender.java | 38 +++++ .../dev/onvoid/webrtc/client/SenderImpl.java | 148 ++++++++++++++++-- .../dev/onvoid/webrtc/client/MediaTests.java | 31 ++++ 5 files changed, 224 insertions(+), 12 deletions(-) diff --git a/design/client-api.md b/design/client-api.md index e52aa701..eb208aaa 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -495,7 +495,9 @@ public interface Sender { CompletableFuture replaceTrack(LocalTrack track); // no renegotiation, null sends nothing CompletableFuture configure(Encoding encoding); // max bitrate, max frame rate, scale down, active CompletableFuture remove(); // renegotiates in AUTOMATIC mode - DtmfSender dtmf(); // audio senders only + CompletableFuture sendTones(String tones); // DTMF, audio senders only + CompletableFuture sendTones(String tones, Duration duration, Duration gap); + Subscription onTone(Consumer handler); } public interface RemoteTrack { @@ -756,7 +758,10 @@ How the threading model is built: Where the code differs from the API above: -- `Sender.dtmf()` and `LocalVideo.switchCamera()` are left for later. +- `LocalVideo.switchCamera()` is left for later: a WebRTC track cannot change its source, so it + needs a new native track behind the same `LocalVideo`, replaced on every sender. +- DTMF is `Sender.sendTones()` and `Sender.onTone()` instead of a `DtmfSender` object; the future + completes once the tones have been sent. - `Devices.screens()` and `windows()` ask the system on every call; only the camera, microphone and speaker lists are snapshots. The first `devices()` call reads them once, synchronously. - `Peer.addTrack` without stream ids puts every such track of a peer into one shared stream, so a diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index ea959173..5ff7a2cd 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -310,6 +310,16 @@ camera.close(); // leaves every peer and stops th Closing a track stops sending it on every peer, without renegotiating, and then releases the capture. The senders stay, and can be given another track with `replaceTrack`. +An audio sender also sends DTMF tones, for telephone systems and voice menus. The future completes once the tones have been sent, and `onTone` reports each tone as it starts, then an empty string at the end: + +```java +audioSender.onTone(tone -> System.out.println(tone.isEmpty() ? "Done" : "Tone " + tone)); +audioSender.sendTones("1234#"); // 100 ms each +audioSender.sendTones("*99,1", Duration.ofMillis(200), Duration.ofMillis(100)); +``` + +The characters 0 to 9, A to D, `#` and `*` are tones, and a comma is a pause of 2 seconds. Tones still queued from an earlier call are replaced, and that call's future is cancelled. A video sender, or one whose remote peer accepts no telephone events, fails the future with an `IllegalStateException`. The [DTMF Sender](/guide/audio/dtmf-sender) guide has the details. + ### Devices `media.devices()` returns the cameras, microphones and speakers. The first call reads them from the system; after that, the lists are snapshots kept current in the background, so reading them is cheap on any thread. `screens()` and `windows()` ask the system on each call. diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java index d89d996e..dbcc9158 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Sender.java @@ -16,7 +16,9 @@ package dev.onvoid.webrtc.client; +import java.time.Duration; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; /** * Sends one track on one peer, made by {@link Peer#addTrack}. @@ -46,6 +48,42 @@ public interface Sender { */ CompletableFuture configure(Encoding encoding); + /** + * Sends DTMF tones on an audio sender, 100 ms each with 70 ms between + * them. See {@link #sendTones(String, Duration, Duration)}. + * + * @param tones the tones to send. + * + * @return a future that completes once the tones have been sent. + */ + CompletableFuture sendTones(String tones); + + /** + * Sends DTMF tones on an audio sender, as telephone systems expect. The + * characters 0 to 9, A to D, # and * are tones, and a comma is a pause of + * 2 seconds; other characters are skipped. Tones still queued from an + * earlier call are replaced, and the future of that call is cancelled. + * + * @param tones the tones to send. + * @param duration how long each tone sounds, 40 ms to 6 seconds. + * @param gap the pause between two tones, at least 30 ms. + * + * @return a future that completes once the tones have been sent. It fails + * with an {@link IllegalStateException} if the sender cannot send tones, + * because it sends video or the remote peer does not accept them. + */ + CompletableFuture sendTones(String tones, Duration duration, Duration gap); + + /** + * Calls the handler as each DTMF tone starts, and with an empty string once + * all queued tones have been sent. + * + * @param handler called on the peer's event thread. + * + * @return the subscription that removes the handler. + */ + Subscription onTone(Consumer handler); + /** * Stops sending. With {@link Negotiation#AUTOMATIC} the peer renegotiates. * The track stays open. diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java index 045913e0..e10e74ac 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java @@ -16,9 +16,15 @@ package dev.onvoid.webrtc.client; +import static java.util.Objects.requireNonNull; + +import java.time.Duration; import java.util.List; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import dev.onvoid.webrtc.RTCDtmfSender; import dev.onvoid.webrtc.RTCPeerConnection; import dev.onvoid.webrtc.RTCRtpEncodingParameters; import dev.onvoid.webrtc.RTCRtpSendParameters; @@ -46,11 +52,21 @@ final class SenderImpl implements Sender { private Throwable attachError; + private RTCDtmfSender dtmf; + + /** Completes once the tones sent last have been played. */ + private CompletableFuture pendingTones; + + // Confined to the peer's event executor. + + private final Handlers toneHandlers; + SenderImpl(PeerImpl peer, LocalTrackImpl track) { this.peer = peer; this.track = track; this.kind = track.kind(); + this.toneHandlers = new Handlers<>(peer.rtc::reportError); } void attach(RTCPeerConnection connection, List streamIds) { @@ -145,6 +161,82 @@ public CompletableFuture configure(Encoding encoding) { }); } + @Override + public CompletableFuture sendTones(String tones) { + return sendTones(tones, Duration.ofMillis(100), Duration.ofMillis(70)); + } + + @Override + public CompletableFuture sendTones(String tones, Duration duration, Duration gap) { + requireNonNull(tones, "tones"); + requireNonNull(duration, "duration"); + requireNonNull(gap, "gap"); + + long durationMs = duration.toMillis(); + long gapMs = gap.toMillis(); + + if (durationMs < 40 || durationMs > 6000) { + throw new IllegalArgumentException("A tone lasts 40 ms to 6 seconds, got " + duration); + } + if (gapMs < 30 || gapMs > Integer.MAX_VALUE) { + throw new IllegalArgumentException("The gap between tones is at least 30 ms, got " + gap); + } + + CompletableFuture sent = new CompletableFuture<>(); + + peer.control.execute(() -> { + try { + checkUsable(); + + if (kind != MediaKind.AUDIO) { + throw new IllegalStateException("Only an audio sender sends tones"); + } + if (dtmf == null) { + dtmf = sender.getDtmfSender(); + + if (dtmf == null) { + throw new IllegalStateException("The sender cannot send tones"); + } + + dtmf.registerObserver((tone, buffer) -> toneChanged(tone)); + } + if (!dtmf.canInsertDtmf()) { + throw new IllegalStateException("The sender cannot send tones now. It needs a track, " + + "and the remote peer must have accepted telephone events"); + } + if (!dtmf.insertDtmf(tones, (int) durationMs, (int) gapMs)) { + throw new IllegalStateException("WebRTC refused the tones \"" + tones + "\""); + } + + if (pendingTones != null) { + pendingTones.completeExceptionally(new CancellationException("Replaced by later tones")); + } + + if (hasTone(tones)) { + pendingTones = sent; + } + else { + // Nothing to play, so no event would complete the future. + pendingTones = null; + sent.complete(null); + } + } + catch (Throwable e) { + sent.completeExceptionally(e instanceof IllegalStateException + ? e : PeerImpl.asWebRtcException(e, "Sending tones failed")); + } + }); + + return peer.toPublic(sent); + } + + @Override + public Subscription onTone(Consumer handler) { + requireNonNull(handler, "handler"); + + return peer.subscribe(() -> toneHandlers.add(handler)); + } + @Override public CompletableFuture remove() { return run(() -> { @@ -200,6 +292,16 @@ void closeFromPeer() { private void release() { removed = true; + if (dtmf != null) { + dtmf.unregisterObserver(); + dtmf = null; + } + if (pendingTones != null) { + pendingTones.completeExceptionally(peer.isClosed() + ? peer.closedError() : new IllegalStateException("The sender was removed")); + pendingTones = null; + } + if (sender != null) { sender.dispose(); sender = null; @@ -224,21 +326,47 @@ private LocalTrackImpl checkTrack(LocalTrack replacement) { return (LocalTrackImpl) replacement; } + /** Called on a WebRTC thread, so it only posts. */ + private void toneChanged(String tone) { + String current = tone == null ? "" : tone; + + peer.control.execute(() -> { + if (current.isEmpty() && pendingTones != null) { + pendingTones.complete(null); + pendingTones = null; + } + }); + peer.events.execute(() -> toneHandlers.accept(current)); + } + + private void checkUsable() { + if (peer.isClosed()) { + throw peer.closedError(); + } + if (attachError != null) { + throw PeerImpl.asWebRtcException(attachError, "The track was never added"); + } + if (removed) { + throw new IllegalStateException("The sender was removed"); + } + } + + private static boolean hasTone(String tones) { + for (int i = 0; i < tones.length(); i++) { + if ("0123456789ABCDabcd#*,".indexOf(tones.charAt(i)) >= 0) { + return true; + } + } + + return false; + } + private CompletableFuture run(Runnable operation) { CompletableFuture result = new CompletableFuture<>(); peer.control.execute(() -> { try { - if (peer.isClosed()) { - throw peer.closedError(); - } - if (attachError != null) { - throw PeerImpl.asWebRtcException(attachError, "The track was never added"); - } - if (removed) { - throw new IllegalStateException("The sender was removed"); - } - + checkUsable(); operation.run(); result.complete(null); } diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java index e0541ecc..62132e1f 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java @@ -17,6 +17,7 @@ package dev.onvoid.webrtc.client; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,8 +27,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -260,6 +266,31 @@ private static long residentBytes(Path status) throws Exception { throw new IllegalStateException("No VmRSS in " + status); } + @Test + void tonesSent() throws Exception { + Peer[] peers = pair(); + CustomAudio voice = rtc.media().customAudio("voice"); + Sender audioSender = peers[0].addTrack(voice); + Sender videoSender = peers[0].addTrack(rtc.media().customVideo("video")); + List tones = Collections.synchronizedList(new ArrayList<>()); + + audioSender.onTone(tones::add); + await(peers[0].whenConnected()); + + // Replaced before it sounds: its future is cancelled. + CompletableFuture replaced = audioSender.sendTones("999999", Duration.ofMillis(500), Duration.ofMillis(100)); + await(audioSender.sendTones("1#", Duration.ofMillis(100), Duration.ofMillis(70))); + + assertTrue(replaced.isCancelled() || replaced.isCompletedExceptionally()); + // The last event is the empty tone, which the handler gets before the future completes. + assertEquals("", tones.get(tones.size() - 1)); + assertTrue(String.join("", tones).endsWith("1#"), () -> "Tones: " + tones); + + ExecutionException refused = assertThrows(ExecutionException.class, + () -> videoSender.sendTones("1").get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, refused.getCause()); + } + @Test void audioModeChecked() { assertThrows(ConfigurationException.class, () -> rtc.media().microphone()); From 1284e0af2c04fca271dcea5c961521ff47b9ee1a Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:29:48 +0200 Subject: [PATCH 11/25] Bound the messages a data channel keeps for its handler Messages that arrive while a channel has no message handler are kept up to 16 MiB; past that the oldest are dropped, and the peer's error handlers hear of it once. --- design/client-api.md | 2 + docs/guide/client-api.md | 2 +- .../dev/onvoid/webrtc/client/DataChannel.java | 6 ++- .../onvoid/webrtc/client/DataChannelImpl.java | 23 ++++++++++- .../dev/onvoid/webrtc/client/PeerTests.java | 40 +++++++++++++++++++ 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/design/client-api.md b/design/client-api.md index eb208aaa..03b76f46 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -770,6 +770,8 @@ Where the code differs from the API above: `CustomAudio.queuedMillis()`/`clear()`, `Message.size()`, `Negotiator.answer(offer, gathering)`. - An impolite peer that ignored an offer reports the two-impolite-peers collision after 5 seconds without an answer. +- Messages a data channel keeps while it has no message handler are capped at 16 MiB; past that the + oldest are dropped and the peer's `onError` handlers hear of it once. - `Gathering.COMPLETE` waits at most `PeerOptions.Builder.gatheringTimeout` (3 seconds by default), since an unreachable STUN or TURN server holds up the end of gathering far longer. diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 5ff7a2cd..0ff0c576 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -397,7 +397,7 @@ peer.onChannel(channel -> { // channels the remote peer opens ``` - A channel may be used right away: messages sent before it opens are held until it does. -- Messages that arrive before the first `onMessage` handler are kept for it, so none are lost while a handler is being set up. +- Messages that arrive while a channel has no `onMessage` handler are kept for the next one, so none are lost while a handler is being set up. Up to 16 MiB are kept; beyond that the oldest are dropped, and the peer's `onError` handlers are told. - `send` takes a `String`, a `byte[]` or a `ByteBuffer`, and copies the data before it returns. - A `Message` holds a copy of the data. `isText()` tells text from binary, `text()` decodes it as UTF-8, and `bytes()` returns a copy. diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java index 66d7fa4e..96301b2e 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannel.java @@ -86,8 +86,10 @@ public interface DataChannel extends AutoCloseable { long bufferedAmount(); /** - * Calls the handler for each message received. Messages that arrive before - * the first handler is added are kept for it. + * Calls the handler for each message received. Messages that arrive while + * the channel has no handler are kept for the next one, up to 16 MiB; + * beyond that the oldest are dropped, which is reported to the peer's + * {@link Peer#onError} handlers. * * @param handler called on the peer's event thread. * diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java index 102a4820..74de5401 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DataChannelImpl.java @@ -44,6 +44,9 @@ final class DataChannelImpl implements DataChannel { /** Bytes the native channel may hold before sends wait in Java. */ static final long HIGH_WATER_MARK = 1024 * 1024; + /** Bytes of received messages kept while there is no message handler. */ + static final long EARLY_LIMIT = 16 * 1024 * 1024; + private final PeerImpl peer; private final String label; @@ -70,7 +73,11 @@ final class DataChannelImpl implements DataChannel { private ChannelState deliveredState = ChannelState.CONNECTING; - private final List early = new ArrayList<>(); + private final ArrayDeque early = new ArrayDeque<>(); + + private long earlyBytes; + + private boolean earlyDropReported; private final Handlers messageHandlers; @@ -173,6 +180,7 @@ public Subscription onMessage(Consumer handler) { if (!early.isEmpty()) { List kept = new ArrayList<>(early); early.clear(); + earlyBytes = 0; for (Message message : kept) { peer.call(handler, message); @@ -372,6 +380,7 @@ else if (newState == ChannelState.CLOSED) { messageHandlers.clear(); stateHandlers.clear(); early.clear(); + earlyBytes = 0; } } @@ -382,6 +391,18 @@ private void deliverMessage(Message message) { if (messageHandlers.isEmpty()) { early.add(message); + earlyBytes += message.size(); + + // Kept for a handler that may never come, so their memory is bounded. + while (earlyBytes > EARLY_LIMIT && early.size() > 1) { + earlyBytes -= early.poll().size(); + + if (!earlyDropReported) { + earlyDropReported = true; + peer.deliverError(new WebRtcException("The data channel " + label + " dropped received messages: " + + "more than " + (EARLY_LIMIT >> 20) + " MiB arrived while it had no message handler")); + } + } } else { messageHandlers.accept(message); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java index af20bd9b..de7d234c 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java @@ -94,6 +94,46 @@ void earlyMessagesKept() throws Exception { assertEquals(Arrays.asList("one", "two"), messages); } + @Test + void earlyMessagesBounded() throws Exception { + Peer[] peers = pair(); + CompletableFuture remote = new CompletableFuture<>(); + CompletableFuture dropped = new CompletableFuture<>(); + + peers[1].onChannel(remote::complete); + peers[1].onError(dropped::complete); + + // About 20 MiB while the remote channel has no message handler. + DataChannel channel = peers[0].createChannel("bulk"); + int count = 100; + byte[] payload = new byte[200 * 1024]; + + for (int i = 0; i < count; i++) { + payload[0] = (byte) i; + channel.send(payload); + } + await(channel.send(new byte[] { (byte) 0xFF })); + + assertTrue(await(dropped).getMessage().contains("dropped received messages")); + + List received = Collections.synchronizedList(new ArrayList<>()); + CompletableFuture last = new CompletableFuture<>(); + await(remote).onMessage(message -> { + byte[] bytes = message.bytes(); + if (bytes.length == 1) { + last.complete(null); + } + else { + received.add(bytes[0] & 0xFF); + } + }); + await(last); + + // The newest messages were kept, in order. + assertTrue(received.size() < count, () -> "Kept " + received.size() + " messages"); + assertEquals(count - 1, (int) received.get(received.size() - 1)); + } + @Test void eventsInOrderOneAtATime() throws Exception { Peer[] peers = pair(); From 761bcb5de308ae859a0ca74a74b65072b5394b54 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:34:10 +0200 Subject: [PATCH 12/25] Make the offer collision tests collide every time Both peers created a channel at once, but one offer could arrive before the other was made, so nothing collided and the impolite test timed out. A test signaling now holds the messages until both peers have sent an offer. --- .../onvoid/webrtc/client/HeldSignaling.java | 121 ++++++++++++++++++ .../dev/onvoid/webrtc/client/PeerTests.java | 22 ++-- 2 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/HeldSignaling.java diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/HeldSignaling.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/HeldSignaling.java new file mode 100644 index 00000000..2b14ac37 --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/HeldSignaling.java @@ -0,0 +1,121 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +/** + * Two linked signaling ends that hold every message until both peers have sent + * an offer. Releasing them then makes the offers collide, whatever the timing. + */ +final class HeldSignaling { + + private final End first = new End(); + + private final End second = new End(); + + private boolean released; + + + HeldSignaling() { + first.remote = second; + second.remote = first; + } + + Signaling first() { + return first; + } + + Signaling second() { + return second; + } + + /** + * Waits until both peers have sent an offer, then delivers what they sent and + * lets later messages through. + */ + void releaseWhenBothOffered(long timeoutSeconds) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + + synchronized (this) { + while (!(first.offered && second.offered)) { + long left = deadline - System.nanoTime(); + + if (left <= 0) { + throw new IllegalStateException("Both peers did not send an offer"); + } + + TimeUnit.NANOSECONDS.timedWait(this, left); + } + + released = true; + + first.flush(); + second.flush(); + } + } + + + + private final class End implements Signaling { + + private final List held = new ArrayList<>(); + + private End remote; + + private Consumer inbound; + + private boolean offered; + + + @Override + public void send(SignalMessage message) { + synchronized (HeldSignaling.this) { + if (message.type() == SignalMessage.Type.OFFER) { + offered = true; + HeldSignaling.this.notifyAll(); + } + + if (released) { + remote.inbound.accept(message); + } + else { + held.add(message); + } + } + } + + @Override + public void attach(Consumer inbound) { + synchronized (HeldSignaling.this) { + this.inbound = inbound; + } + } + + /** Delivers what this end sent to the other end's peer. */ + private void flush() { + for (SignalMessage message : held) { + remote.inbound.accept(message); + } + + held.clear(); + } + } +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java index de7d234c..feccec33 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java @@ -258,19 +258,19 @@ void blockedHandlerDelaysOnlyItsPeer() throws Exception { @Test void politePeerResolvesCollision() throws Exception { - Peer[] peers = pair(); + HeldSignaling link = new HeldSignaling(); + Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); + Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); CompletableFuture atCaller = new CompletableFuture<>(); CompletableFuture atCallee = new CompletableFuture<>(); - // Both create a channel at once, so both send an offer at once. - DataChannel fromCaller = peers[0].createChannel("from-caller"); - DataChannel fromCallee = peers[1].createChannel("from-callee"); - - peers[0].onChannel(channel -> channel.onMessage(message -> atCaller.complete(message.text()))); - peers[1].onChannel(channel -> channel.onMessage(message -> atCallee.complete(message.text()))); + caller.onChannel(channel -> channel.onMessage(message -> atCaller.complete(message.text()))); + callee.onChannel(channel -> channel.onMessage(message -> atCallee.complete(message.text()))); - fromCaller.send("hello callee"); - fromCallee.send("hello caller"); + // Both send an offer before either sees the other's. + caller.createChannel("from-caller").send("hello callee"); + callee.createChannel("from-callee").send("hello caller"); + link.releaseWhenBothOffered(TIMEOUT_SECONDS); assertEquals("hello callee", await(atCallee)); assertEquals("hello caller", await(atCaller)); @@ -279,7 +279,7 @@ void politePeerResolvesCollision() throws Exception { @Test void impoliteCollisionReported() throws Exception { - SignalingPair link = Signaling.loopback(); + HeldSignaling link = new HeldSignaling(); Peer first = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); Peer second = rtc.createPeer(PeerOptions.builder().signaling(link.second()).build()); @@ -287,8 +287,10 @@ void impoliteCollisionReported() throws Exception { first.onError(error::complete); second.onError(error::complete); + // Both send an offer before either sees the other's, and neither gives way. first.createChannel("a"); second.createChannel("b"); + link.releaseWhenBothOffered(TIMEOUT_SECONDS); WebRtcException reported = error.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); From ba5d501c2ad439f526b35a46ca02d8c8edd9f985 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:34:10 +0200 Subject: [PATCH 13/25] Add VideoView, a Swing component for client API video It converts frames on their own thread, paints on the EDT, keeps the aspect ratio and rotation, and can mirror a local preview. --- design/client-api.md | 4 +- docs/guide/client-api.md | 21 ++ .../onvoid/webrtc/client/swing/VideoView.java | 200 ++++++++++++++++++ webrtc/src/main/java/module-info.java | 1 + .../onvoid/webrtc/client/VideoViewTests.java | 159 ++++++++++++++ 5 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/swing/VideoView.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/VideoViewTests.java diff --git a/design/client-api.md b/design/client-api.md index 03b76f46..431369e8 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -627,7 +627,7 @@ Peer peer = rtc.createPeer(options); peer.onStateChange(state -> statusLabel.setText(state.toString())); // on the EDT peer.onTrack(track -> { if (track.isVideo()) { - track.asVideo().onFrame(remoteView); // VideoView, phase 2: converts off the EDT, paints on it + track.asVideo().onFrame(remoteView); // VideoView: converts off the EDT, paints on it } }); @@ -799,5 +799,5 @@ Every test also fails if anything reached the runtime's error handler, including Next: - a guide page and an example; -- `Whip`/`Whep` and `VideoView`; +- `Whip`/`Whep` (`VideoView` is done, in `dev.onvoid.webrtc.client.swing`); - porting the existing examples (phases 2 and 3). diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 0ff0c576..49e3dbe5 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -217,6 +217,27 @@ hangUpButton.addActionListener(event -> peer.close()); // never blocks the EDT The order of each peer's events is kept with any executor, including a thread pool. Frame handlers stay on their own threads even then, since frames come far too often for a UI thread. +`VideoView` in `dev.onvoid.webrtc.client.swing` shows video in a Swing window. It converts each frame on the frame's own thread and leaves only the painting to the EDT. It keeps the picture's aspect ratio and rotation, and can mirror a local camera preview: + +```java +import dev.onvoid.webrtc.client.swing.VideoView; + +VideoView remoteView = new VideoView(); +VideoView preview = new VideoView(); +preview.setMirrored(true); + +peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(remoteView); + } +}); +camera.onFrame(preview); + +frame.add(remoteView, BorderLayout.CENTER); +``` + +`snapshot()` returns a copy of the picture shown, for saving a still or for tests. + ### Don't Wait in a Handler A future of the client API completes on the event thread of its peer. A handler that waits for one would wait for its own thread, so `join()` and `get()` there throw a `BlockingCallException` instead of hanging. Chain the next step instead: diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/swing/VideoView.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/swing/VideoView.java new file mode 100644 index 00000000..1aaa4821 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/swing/VideoView.java @@ -0,0 +1,200 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client.swing; + +import java.awt.Color; +import java.awt.Dimension; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.geom.AffineTransform; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferInt; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.function.Consumer; + +import javax.swing.JComponent; + +import dev.onvoid.webrtc.media.FourCC; +import dev.onvoid.webrtc.media.video.VideoBufferConverter; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoFrameBuffer; + +/** + * A Swing component that shows video frames, such as those of a remote track: + * 
{@code
+ * VideoView view = new VideoView();
+ * track.asVideo().onFrame(view);
+ * }
+ * Frames are converted on the thread that delivers them, so the event dispatch + * thread only paints. The picture keeps its aspect ratio and its rotation, with + * the component's background around it. + */ +public class VideoView extends JComponent implements Consumer { + + private static final long serialVersionUID = 1L; + + /** Guards the images and the pixel buffer. */ + private final Object lock = new Object(); + + /** The picture shown. */ + private transient BufferedImage front; + + /** The picture the next frame is converted into. */ + private transient BufferedImage back; + + private transient ByteBuffer pixels; + + private int rotation; + + private volatile boolean mirrored; + + + /** + * Creates a view with a black background and a preferred size of 640 by + * 360 pixels. + */ + public VideoView() { + setBackground(Color.BLACK); + setOpaque(true); + setPreferredSize(new Dimension(640, 360)); + } + + /** + * Shows a frame. Called with the frames of a track, from any thread; the + * frame is not released, which its subscription does. + * + * @param frame the frame to show. + */ + @Override + public void accept(VideoFrame frame) { + VideoFrameBuffer buffer = frame.buffer; + int width = buffer.getWidth(); + int height = buffer.getHeight(); + + synchronized (lock) { + if (back == null || back.getWidth() != width || back.getHeight() != height) { + back = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + } + if (pixels == null || pixels.capacity() != width * height * 4) { + pixels = ByteBuffer.allocateDirect(width * height * 4).order(ByteOrder.LITTLE_ENDIAN); + } + + try { + // libyuv's ARGB is B, G, R, A in memory, which read as + // little-endian ints is what TYPE_INT_RGB holds. + pixels.clear(); + VideoBufferConverter.convertFromI420(buffer, pixels, FourCC.ARGB); + pixels.rewind(); + pixels.asIntBuffer().get(((DataBufferInt) back.getRaster().getDataBuffer()).getData()); + } + catch (Exception e) { + throw new IllegalStateException("The frame could not be converted", e); + } + + BufferedImage shown = front; + front = back; + back = shown; + rotation = frame.rotation; + } + + repaint(); + } + + /** + * Removes the picture, leaving the background. + */ + public void clear() { + synchronized (lock) { + front = null; + back = null; + pixels = null; + } + + repaint(); + } + + /** + * @param mirrored true to flip the picture horizontally, as a local camera + * preview usually is. + */ + public void setMirrored(boolean mirrored) { + this.mirrored = mirrored; + repaint(); + } + + /** + * @return true when the picture is flipped horizontally. + */ + public boolean isMirrored() { + return mirrored; + } + + /** + * @return a copy of the picture shown, as the frame was, without rotation + * or mirroring; null when there is none. + */ + public BufferedImage snapshot() { + synchronized (lock) { + if (front == null) { + return null; + } + + BufferedImage copy = new BufferedImage(front.getWidth(), front.getHeight(), BufferedImage.TYPE_INT_RGB); + copy.setData(front.getRaster()); + + return copy; + } + } + + @Override + protected void paintComponent(Graphics graphics) { + graphics.setColor(getBackground()); + graphics.fillRect(0, 0, getWidth(), getHeight()); + + synchronized (lock) { + if (front == null) { + return; + } + + boolean sideways = rotation == 90 || rotation == 270; + int imageWidth = sideways ? front.getHeight() : front.getWidth(); + int imageHeight = sideways ? front.getWidth() : front.getHeight(); + double scale = Math.min((double) getWidth() / imageWidth, (double) getHeight() / imageHeight); + + Graphics2D g = (Graphics2D) graphics.create(); + + try { + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + + // Around the center of the component: scale, rotate, mirror, then + // draw the image centered on the origin. + AffineTransform transform = new AffineTransform(); + transform.translate(getWidth() / 2.0, getHeight() / 2.0); + transform.scale(mirrored ? -scale : scale, scale); + transform.rotate(Math.toRadians(rotation)); + transform.translate(-front.getWidth() / 2.0, -front.getHeight() / 2.0); + + g.drawImage(front, transform, null); + } + finally { + g.dispose(); + } + } + } +} diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java index 77a60a8e..ec5e78d8 100644 --- a/webrtc/src/main/java/module-info.java +++ b/webrtc/src/main/java/module-info.java @@ -7,6 +7,7 @@ exports dev.onvoid.webrtc; exports dev.onvoid.webrtc.client; + exports dev.onvoid.webrtc.client.swing; exports dev.onvoid.webrtc.logging; exports dev.onvoid.webrtc.media; exports dev.onvoid.webrtc.media.audio; diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/VideoViewTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/VideoViewTests.java new file mode 100644 index 00000000..7a373bab --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/VideoViewTests.java @@ -0,0 +1,159 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.image.BufferedImage; +import java.nio.ByteBuffer; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import dev.onvoid.webrtc.client.swing.VideoView; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +import org.junit.jupiter.api.Test; + +class VideoViewTests extends ClientTestBase { + + @Test + void showsFrame() { + VideoView view = new VideoView(); + assertNull(view.snapshot()); + + view.accept(frame(64, 48, 128, 128)); + + BufferedImage snapshot = view.snapshot(); + assertNotNull(snapshot); + assertEquals(64, snapshot.getWidth()); + assertEquals(48, snapshot.getHeight()); + assertGray(new Color(snapshot.getRGB(32, 24))); + + view.clear(); + assertNull(view.snapshot()); + } + + @Test + void keepsAspectRatio() { + VideoView view = new VideoView(); + view.setSize(200, 100); + view.accept(frame(64, 48, 128, 128)); + + // A 4:3 picture in a 2:1 view leaves bars on the sides. + BufferedImage painted = paint(view); + assertEquals(Color.BLACK, new Color(painted.getRGB(5, 50))); + assertGray(new Color(painted.getRGB(100, 50))); + } + + @Test + void mirrors() { + VideoView view = new VideoView(); + view.setSize(64, 48); + view.accept(frame(64, 48, 16, 235)); + + assertTrue(brightness(paint(view).getRGB(8, 24)) < 64); + + view.setMirrored(true); + + // The bright right half is now on the left. + assertTrue(brightness(paint(view).getRGB(8, 24)) > 192); + } + + @Test + void showsRemoteVideo() throws Exception { + Peer[] peers = pair(); + CustomVideo video = rtc.media().customVideo("video"); + VideoView view = new VideoView(); + + peers[1].onTrack(track -> track.asVideo().onFrame(view)); + peers[0].addTrack(video); + + ScheduledExecutorService feeder = Executors.newSingleThreadScheduledExecutor(); + + try { + feeder.scheduleAtFixedRate(() -> { + VideoFrame frame = frame(320, 240, 128, 128); + video.push(frame); + frame.release(); + }, 0, 33, TimeUnit.MILLISECONDS); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (view.snapshot() == null && System.nanoTime() < deadline) { + Thread.sleep(50); + } + } + finally { + feeder.shutdownNow(); + } + + BufferedImage snapshot = view.snapshot(); + assertNotNull(snapshot, "No frame reached the view"); + assertEquals(320, snapshot.getWidth()); + } + + /** + * A frame whose left half has luma {@code left} and right half {@code right}, with + * neutral chroma. + */ + private static VideoFrame frame(int width, int height, int left, int right) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(width, height); + ByteBuffer y = buffer.getDataY(); + + for (int row = 0; row < height; row++) { + for (int column = 0; column < width; column++) { + y.put(row * buffer.getStrideY() + column, (byte) (column < width / 2 ? left : right)); + } + } + + fill(buffer.getDataU()); + fill(buffer.getDataV()); + + return new VideoFrame(buffer, System.nanoTime()); + } + + private static void fill(ByteBuffer plane) { + for (int i = 0; i < plane.capacity(); i++) { + plane.put(i, (byte) 128); + } + } + + private static BufferedImage paint(VideoView view) { + BufferedImage image = new BufferedImage(view.getWidth(), view.getHeight(), BufferedImage.TYPE_INT_RGB); + Graphics2D g = image.createGraphics(); + view.paint(g); + g.dispose(); + + return image; + } + + private static void assertGray(Color color) { + assertTrue(Math.abs(color.getRed() - color.getGreen()) < 8 && Math.abs(color.getGreen() - color.getBlue()) < 8 + && color.getGreen() > 100 && color.getGreen() < 160, () -> "Not mid gray: " + color); + } + + private static int brightness(int rgb) { + return new Color(rgb).getGreen(); + } + +} From a0390241e1d04d861c17e8ffd7956cae145328e3 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 19 Oct 2026 08:39:14 +0200 Subject: [PATCH 14/25] Add WHIP and WHEP sessions to the client API Whip.to(endpoint).publish(rtc) and Whep.from(endpoint).play(rtc) negotiate a peer over HTTP and complete once it is connected. Closing a session deletes it on the server; a failed setup leaves nothing behind. The WHEP example uses Whep now. --- design/client-api.md | 4 +- docs/guide/client-api.md | 33 +- docs/guide/examples.md | 8 +- .../onvoid/webrtc/examples/WhepExample.java | 110 ++----- .../dev/onvoid/webrtc/client/HttpSession.java | 284 ++++++++++++++++++ .../java/dev/onvoid/webrtc/client/Whep.java | 177 +++++++++++ .../java/dev/onvoid/webrtc/client/Whip.java | 177 +++++++++++ .../webrtc/client/HttpSessionTests.java | 144 +++++++++ .../webrtc/client/TestHttpEndpoint.java | 170 +++++++++++ 9 files changed, 1013 insertions(+), 94 deletions(-) create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/HttpSession.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Whep.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/Whip.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/HttpSessionTests.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/TestHttpEndpoint.java diff --git a/design/client-api.md b/design/client-api.md index 431369e8..0a8cde8e 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -799,5 +799,7 @@ Every test also fails if anything reached the runtime's error handler, including Next: - a guide page and an example; -- `Whip`/`Whep` (`VideoView` is done, in `dev.onvoid.webrtc.client.swing`); +- phase 2 is done except the media module bridge and recording: `Whip.to(...).publish(rtc)` and + `Whep.from(...).play(rtc)` return sessions over `HttpURLConnection` (Java 8), and `VideoView` is in + `dev.onvoid.webrtc.client.swing`; - porting the existing examples (phases 2 and 3). diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 49e3dbe5..3c291b57 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -441,9 +441,40 @@ DataChannel positions = peer.createChannel("positions", ChannelOptions.builder() `negotiatedId(int)` creates a channel both peers set up themselves with the same id, without announcing it; such a channel never shows up in `onChannel`. The [Data Channels](/guide/data/data-channels) guide explains the options in detail. +## WHIP and WHEP + +Media servers and streaming services take streams with the WebRTC HTTP Ingestion Protocol ([WHIP](https://www.rfc-editor.org/rfc/rfc9725)) and hand them out with its egress counterpart, WHEP. Both exchange one offer and one answer over HTTP. `Whip` publishes tracks, and `Whep` receives a stream: + +```java +Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) + .bearerToken(streamKey) + .track(rtc.media().camera()) + .track(rtc.media().microphone()) + .publish(rtc) + .get(15, TimeUnit.SECONDS); + +Whep.Session viewer = Whep.from(URI.create("https://example.com/live/whep")) + .play(rtc) + .get(15, TimeUnit.SECONDS); + +viewer.peer().onTrack(track -> System.out.println("Receiving " + track.kind())); +``` + +The offer goes to the endpoint with the local candidates, as WHIP and WHEP servers expect, and the future completes once the peer is connected. If the endpoint refuses the offer, or the peer does not connect within the timeout (15 seconds by default), the future fails, and the session is deleted on the server again. `onTrack` hands over tracks that already arrived, so a handler added once the future completed misses none. + +`session.close()` deletes the session on the server, which frees what the server holds for it, and closes the peer. The tracks a WHIP session published stay open. `session.peer()` is a regular peer, for stats and anything else. + +| Builder method | What it sets | +|---|---| +| `bearerToken(String)` | The token sent as `Authorization: Bearer`; for many services, the stream key. | +| `iceServer(String...)` | A STUN or TURN server; endpoints usually name their own in the answer. | +| `track(LocalTrack)` | A track to publish, WHIP only. The tracks share one stream. | +| `receive(MediaKind...)` | What to receive, WHEP only; video and audio by default. | +| `timeout(Duration)` | How long the session may take to be connected. | + ## Manual Negotiation -WHIP, WHEP and most SFUs dictate their own offer and answer exchange. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. A STUN or TURN server that cannot be reached, as on networks that block UDP, holds up the end of gathering for a long time, so `COMPLETE` waits at most for the peer's gathering timeout and then goes with the candidates it has. This WHEP client receives a stream: +SFUs and other servers dictate an offer and answer exchange of their own. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`; `Whip` and `Whep` are built on it. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. A STUN or TURN server that cannot be reached, as on networks that block UDP, holds up the end of gathering for a long time, so `COMPLETE` waits at most for the peer's gathering timeout and then goes with the candidates it has. This WHEP client does by hand what `Whep` does: ```java Peer viewer = rtc.createPeer(PeerOptions.builder() diff --git a/docs/guide/examples.md b/docs/guide/examples.md index d919f7d4..f93550d7 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -40,11 +40,9 @@ This example provides a foundation for building WebRTC applications that need to The [`WhepExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java) implements a client of the WebRTC HTTP Egress Protocol (WHEP), which receives a stream from a server, with the [Client API](/guide/client-api). **Key features demonstrated:** -- Creating a receive-only peer that negotiates by hand -- Creating an offer that already holds the local candidates, as WHEP servers expect -- Sending the offer to a WHEP endpoint and applying the answer it returns -- Receiving the stream's video and audio -- Ending the session with an HTTP `DELETE` of the resource the server created +- Receiving a stream from a WHEP endpoint with `Whep` +- Logging the stream's video frames and audio +- Ending the session, which deletes it on the server This example is useful for applications that need to receive media streams from WHEP-compatible servers, such as live streaming platforms. The endpoint and the number of seconds to receive are optional: diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java index d591e1f1..15255e97 100644 --- a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java @@ -17,35 +17,28 @@ package dev.onvoid.webrtc.examples; import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.time.Duration; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import dev.onvoid.webrtc.RTCSdpType; -import dev.onvoid.webrtc.RTCSessionDescription; -import dev.onvoid.webrtc.client.Gathering; -import dev.onvoid.webrtc.client.MediaKind; -import dev.onvoid.webrtc.client.Negotiation; import dev.onvoid.webrtc.client.Peer; -import dev.onvoid.webrtc.client.PeerOptions; import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.client.Whep; /** - * Example implementation of a WebRTC HTTP Egress Protocol (WHEP) client, with the client API. + * Example of a WebRTC HTTP Egress Protocol (WHEP) client, with the client API. *

* This class demonstrates: *

    - *
  • Creating a receive-only peer that negotiates by hand
    • - *
  • Creating an offer that already holds all local candidates, as WHEP servers expect
    • - *
  • Sending the offer to a WHEP endpoint and applying the answer it returns
    • - *
  • Receiving the stream's video and audio
    • - *
  • Ending the session with an HTTP DELETE of the resource the server created
    • + *
  • Receiving a stream from a WHEP endpoint with {@link Whep}
    • + *
  • Logging the stream's video frames and audio
    • + *
  • Ending the session, which deletes it on the server
    • *
*

+ * {@link Whep} sends an offer with all local candidates, applies the answer and waits until + * the peer is connected. The Client API guide shows the same steps by hand, through the + * peer's negotiator, for servers that speak a protocol of their own. + *

* Run it against a WHEP endpoint, such as one of MediaMTX, for a number of seconds: * 

  * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.WhepExample" -D"exec.args=http://localhost:8889/mystream/whep 10"
@@ -59,10 +52,6 @@ public class WhepExample {
 
     private static final String DEFAULT_ENDPOINT = "http://localhost:8889/mystream/whep";
 
-    private static final HttpClient HTTP = HttpClient.newBuilder()
-            .connectTimeout(Duration.ofSeconds(10))
-            .build();
-
 
     public static void main(String[] args) throws Exception {
         URI endpoint = URI.create(args.length > 0 ? args[0] : DEFAULT_ENDPOINT);
@@ -70,15 +59,18 @@ public static void main(String[] args) throws Exception {
 
         // The runtime plays the received audio to the speakers.
         try (WebRtc rtc = WebRtc.create()) {
-            // A WHEP server dictates the exchange: one offer, one answer, over HTTP.
-            Peer viewer = rtc.createPeer(PeerOptions.builder()
-                    .name("whep")
+            System.out.println("Requesting the stream from " + endpoint);
+
+            // The main thread may wait for the session; a handler must not.
+            Whep.Session session = Whep.from(endpoint)
                     .iceServer("stun:stun.l.google.com:19302")
-                    .negotiation(Negotiation.MANUAL)
-                    .receive(MediaKind.VIDEO, MediaKind.AUDIO)
-                    .build());
+                    .play(rtc)
+                    .get(15, TimeUnit.SECONDS);
 
+            Peer viewer = session.peer();
             viewer.onStateChange(state -> System.out.println("Connection is " + state));
+
+            // Tracks that arrived before this handler are handed to it too.
             viewer.onTrack(track -> {
                 System.out.println("Receiving " + track.kind());
 
@@ -103,69 +95,13 @@ public static void main(String[] args) throws Exception {
                 }
             });
 
-            // The main thread may wait for the futures; a handler must not.
-            RTCSessionDescription offer = viewer.negotiator()
-                    .createOffer(Gathering.COMPLETE)
-                    .get(10, TimeUnit.SECONDS);
-
-            System.out.println("Sending the offer to " + endpoint);
+            Thread.sleep(seconds * 1000L);
 
-            HttpResponse response = post(endpoint, offer.sdp);
-            URI resource = response.headers().firstValue("Location").map(endpoint::resolve).orElse(null);
+            // Deletes the session on the server, then closes the peer.
+            session.closeAsync().get(5, TimeUnit.SECONDS);
 
-            try {
-                viewer.negotiator()
-                        .applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, response.body()))
-                        .get(10, TimeUnit.SECONDS);
-                viewer.whenConnected().get(10, TimeUnit.SECONDS);
-
-                Thread.sleep(seconds * 1000L);
-            }
-            finally {
-                if (resource != null) {
-                    end(resource);
-                }
-            }
-        }
-        // The runtime closed the peer and the factory.
-    }
-
-    /**
-     * Sends the offer and returns the server's answer.
-     */
-    private static HttpResponse post(URI endpoint, String sdp) throws Exception {
-        HttpRequest request = HttpRequest.newBuilder(endpoint)
-                .header("Content-Type", "application/sdp")
-                .POST(HttpRequest.BodyPublishers.ofString(sdp))
-                .timeout(Duration.ofSeconds(30))
-                .build();
-
-        HttpResponse response = HTTP.send(request, HttpResponse.BodyHandlers.ofString());
-
-        if (response.statusCode() != 200 && response.statusCode() != 201) {
-            throw new IllegalStateException("The WHEP endpoint answered with status " + response.statusCode()
-                    + ": " + response.body());
-        }
-
-        return response;
-    }
-
-    /**
-     * Ends the session on the server, which frees what it holds for this viewer.
-     */
-    private static void end(URI resource) {
-        try {
-            HttpRequest request = HttpRequest.newBuilder(resource)
-                    .DELETE()
-                    .timeout(Duration.ofSeconds(5))
-                    .build();
-
-            int status = HTTP.send(request, HttpResponse.BodyHandlers.discarding()).statusCode();
-
-            System.out.println("Ended the session, status " + status);
-        }
-        catch (Exception e) {
-            System.err.println("Ending the session failed: " + e);
+            System.out.println("Ended the session " + session.resource());
         }
+        // The runtime closed the factory.
     }
 }
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/HttpSession.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/HttpSession.java
new file mode 100644
index 00000000..c0843c36
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/HttpSession.java
@@ -0,0 +1,284 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static java.util.Objects.requireNonNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import dev.onvoid.webrtc.RTCSdpType;
+import dev.onvoid.webrtc.RTCSessionDescription;
+
+/**
+ * A WHIP or WHEP session: a peer whose offer and answer went over HTTP, and the
+ * resource the server created for it. Closing the session deletes the resource
+ * and closes the peer.
+ */
+abstract class HttpSession implements AutoCloseable {
+
+	private final Peer peer;
+
+	private final URI resource;
+
+	private final String bearerToken;
+
+	private final Duration timeout;
+
+	private final WebRtc rtc;
+
+	private final AtomicBoolean closing = new AtomicBoolean();
+
+	private final CompletableFuture closed = new CompletableFuture<>();
+
+
+	HttpSession(WebRtc rtc, Peer peer, URI resource, String bearerToken, Duration timeout) {
+		this.rtc = rtc;
+		this.peer = peer;
+		this.resource = resource;
+		this.bearerToken = bearerToken;
+		this.timeout = timeout;
+	}
+
+	/**
+	 * @return the peer of the session, which negotiates by hand.
+	 */
+	public Peer peer() {
+		return peer;
+	}
+
+	/**
+	 * @return the resource the server created for the session, or null if it
+	 * named none.
+	 */
+	public URI resource() {
+		return resource;
+	}
+
+	/**
+	 * Deletes the session's resource on the server, which frees what the
+	 * server holds for it, and closes the peer.
+	 *
+	 * @return a future that completes once both are done.
+	 */
+	public CompletableFuture closeAsync() {
+		if (closing.compareAndSet(false, true)) {
+			CompletableFuture.runAsync(() -> {
+				try {
+					if (resource != null) {
+						delete(resource, bearerToken, timeout);
+					}
+				}
+				catch (Exception e) {
+					rtc.reportError(new WebRtcException("Deleting the session " + resource + " failed", e));
+				}
+			}, rtc.controlPool)
+					.thenCompose(ignored -> peer.closeAsync())
+					.whenComplete((ignored, error) -> closed.complete(null));
+		}
+
+		return closed;
+	}
+
+	/**
+	 * Ends the session, as {@link #closeAsync()} does, without waiting.
+	 */
+	@Override
+	public void close() {
+		closeAsync();
+	}
+
+	/**
+	 * Negotiates a manual peer over HTTP: creates the offer with all candidates,
+	 * posts it, applies the answer and waits until the peer is connected.
+	 *
+	 * @return a future with the resource the server named, or null.
+	 */
+	static CompletableFuture negotiate(WebRtc rtc, Peer peer, URI endpoint, String bearerToken,
+			Duration timeout) {
+		CompletableFuture result = new CompletableFuture<>();
+		AtomicReference resource = new AtomicReference<>();
+		AtomicBoolean finished = new AtomicBoolean();
+
+		// Either way the session ends, once: connected, failed or timed out.
+		Consumer fail = error -> {
+			if (!finished.compareAndSet(false, true)) {
+				return;
+			}
+
+			// Leaves nothing behind, on the server or here.
+			CompletableFuture.runAsync(() -> {
+				try {
+					URI created = resource.get();
+
+					if (created != null) {
+						delete(created, bearerToken, timeout);
+					}
+				}
+				catch (Exception e) {
+					// The failure that ended the session is the one to report.
+				}
+
+				peer.close();
+				result.completeExceptionally(PeerImpl.asWebRtcException(error,
+						"The session with " + endpoint + " could not be set up"));
+			}, rtc.controlPool);
+		};
+
+		peer.negotiator().createOffer(Gathering.COMPLETE)
+				.thenApplyAsync(offer -> {
+					Response response = post(endpoint, offer.sdp, bearerToken, timeout);
+					resource.set(response.location);
+					return response.body;
+				}, rtc.controlPool)
+				.thenCompose(answer -> peer.negotiator()
+						.applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, answer)))
+				.thenCompose(applied -> peer.whenConnected())
+				.whenComplete((connected, error) -> {
+					if (error != null) {
+						fail.accept(error);
+					}
+					else if (finished.compareAndSet(false, true)) {
+						result.complete(resource.get());
+					}
+				});
+
+		rtc.timer.schedule(() -> fail.accept(new WebRtcException("The session with " + endpoint
+				+ " was not connected within " + timeout)), timeout.toMillis(), TimeUnit.MILLISECONDS);
+
+		return result;
+	}
+
+	private static Response post(URI endpoint, String sdp, String bearerToken, Duration timeout) {
+		HttpURLConnection connection = null;
+
+		try {
+			connection = open(endpoint, "POST", bearerToken, timeout);
+			connection.setDoOutput(true);
+			connection.setRequestProperty("Content-Type", "application/sdp");
+			connection.setRequestProperty("Accept", "application/sdp");
+
+			byte[] body = sdp.getBytes(StandardCharsets.UTF_8);
+			connection.setFixedLengthStreamingMode(body.length);
+
+			try (OutputStream out = connection.getOutputStream()) {
+				out.write(body);
+			}
+
+			int status = connection.getResponseCode();
+
+			if (status != HttpURLConnection.HTTP_CREATED && status != HttpURLConnection.HTTP_OK) {
+				throw new WebRtcException("The endpoint " + endpoint + " answered the offer with status " + status
+						+ ": " + read(connection.getErrorStream()));
+			}
+
+			String location = connection.getHeaderField("Location");
+
+			return new Response(read(connection.getInputStream()),
+					location == null ? null : endpoint.resolve(location));
+		}
+		catch (IOException e) {
+			throw new WebRtcException("Sending the offer to " + endpoint + " failed: " + e.getMessage(), e);
+		}
+		finally {
+			if (connection != null) {
+				connection.disconnect();
+			}
+		}
+	}
+
+	private static void delete(URI resource, String bearerToken, Duration timeout) throws IOException {
+		HttpURLConnection connection = open(resource, "DELETE", bearerToken, timeout);
+
+		try {
+			connection.getResponseCode();
+		}
+		finally {
+			connection.disconnect();
+		}
+	}
+
+	private static HttpURLConnection open(URI uri, String method, String bearerToken, Duration timeout)
+			throws IOException {
+		HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
+		connection.setRequestMethod(method);
+		connection.setConnectTimeout((int) Math.min(Integer.MAX_VALUE, timeout.toMillis()));
+		connection.setReadTimeout((int) Math.min(Integer.MAX_VALUE, timeout.toMillis()));
+
+		if (bearerToken != null) {
+			connection.setRequestProperty("Authorization", "Bearer " + bearerToken);
+		}
+
+		return connection;
+	}
+
+	private static String read(InputStream in) throws IOException {
+		if (in == null) {
+			return "";
+		}
+
+		try (InputStream stream = in) {
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			byte[] buffer = new byte[8192];
+			int count;
+
+			while ((count = stream.read(buffer)) != -1) {
+				out.write(buffer, 0, count);
+			}
+
+			return new String(out.toByteArray(), StandardCharsets.UTF_8);
+		}
+	}
+
+	static URI checkEndpoint(URI endpoint) {
+		requireNonNull(endpoint, "endpoint");
+
+		String scheme = endpoint.getScheme();
+
+		if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
+			throw new IllegalArgumentException("The endpoint must be an http or https URL, got " + endpoint);
+		}
+
+		return endpoint;
+	}
+
+
+
+	private static final class Response {
+
+		final String body;
+
+		final URI location;
+
+
+		Response(String body, URI location) {
+			this.body = body;
+			this.location = location;
+		}
+	}
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Whep.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Whep.java
new file mode 100644
index 00000000..cde25cb0
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Whep.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static java.util.Objects.requireNonNull;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Receives a stream from a server with the WebRTC HTTP Egress Protocol (WHEP),
+ * as media servers such as MediaMTX, Janus and Cloudflare Stream offer it:
+ * 
{@code
+ * Whep.Session session = Whep.from(URI.create("https://example.com/live/whep"))
+ *         .play(rtc)
+ *         .get(15, TimeUnit.SECONDS);
+ *
+ * session.peer().onTrack(track -> ...);
+ * ...
+ * session.close();
+ * }

+ * The offer goes to the endpoint with all local candidates, as WHEP servers
+ * expect, and the session's future completes once the peer is connected.
+ * {@link Peer#onTrack} hands over tracks that arrived before a handler was
+ * added, so registering it after the future completed misses none.
+ *
+ * @see WHIP and the WHEP draft that builds on it
+ */
+public final class Whep {
+
+	private Whep() {
+	}
+
+	/**
+	 * @param endpoint the WHEP endpoint, an http or https URL.
+	 *
+	 * @return a builder that receives video and audio from the endpoint.
+	 */
+	public static Builder from(URI endpoint) {
+		return new Builder(HttpSession.checkEndpoint(endpoint));
+	}
+
+
+
+	/** A stream received over WHEP. Closing it ends the session on the server. */
+	public static final class Session extends HttpSession {
+
+		Session(WebRtc rtc, Peer peer, URI resource, String bearerToken, Duration timeout) {
+			super(rtc, peer, resource, bearerToken, timeout);
+		}
+	}
+
+
+
+	/** Builds a WHEP {@link Session}. */
+	public static final class Builder {
+
+		private final URI endpoint;
+
+		private final List iceServers = new ArrayList<>();
+
+		private final Set receive = EnumSet.of(MediaKind.VIDEO, MediaKind.AUDIO);
+
+		private String bearerToken;
+
+		private Duration timeout = Duration.ofSeconds(15);
+
+
+		private Builder(URI endpoint) {
+			this.endpoint = endpoint;
+		}
+
+		/**
+		 * @param token the token the endpoint authorizes the request with,
+		 *              sent as {@code Authorization: Bearer }.
+		 *
+		 * @return this builder.
+		 */
+		public Builder bearerToken(String token) {
+			this.bearerToken = requireNonNull(token, "token");
+			return this;
+		}
+
+		/**
+		 * @param urls a STUN or TURN server without credentials. The endpoint
+		 *             usually names its own in its answer.
+		 *
+		 * @return this builder.
+		 */
+		public Builder iceServer(String... urls) {
+			iceServers.add(urls.clone());
+			return this;
+		}
+
+		/**
+		 * @param kinds what to receive; video and audio by default.
+		 *
+		 * @return this builder.
+		 */
+		public Builder receive(MediaKind... kinds) {
+			if (kinds.length == 0) {
+				throw new IllegalArgumentException("Receive at least one kind of media");
+			}
+
+			receive.clear();
+
+			for (MediaKind kind : kinds) {
+				receive.add(requireNonNull(kind, "kind"));
+			}
+			return this;
+		}
+
+		/**
+		 * @param timeout how long the session may take to be connected; 15
+		 *                seconds by default.
+		 *
+		 * @return this builder.
+		 */
+		public Builder timeout(Duration timeout) {
+			requireNonNull(timeout, "timeout");
+
+			if (timeout.isNegative() || timeout.isZero()) {
+				throw new IllegalArgumentException("The timeout must be positive, got " + timeout);
+			}
+
+			this.timeout = timeout;
+			return this;
+		}
+
+		/**
+		 * Creates a receive-only peer of the runtime and negotiates it with the
+		 * endpoint.
+		 *
+		 * @param rtc the runtime that owns the peer.
+		 *
+		 * @return a future with the session once its peer is connected. It
+		 * fails if the endpoint refuses the offer or the peer does not connect
+		 * in time, with nothing left behind on the server.
+		 */
+		public CompletableFuture play(WebRtc rtc) {
+			requireNonNull(rtc, "rtc");
+
+			PeerOptions.Builder options = PeerOptions.builder()
+					.name("whep-" + endpoint.getHost())
+					.negotiation(Negotiation.MANUAL)
+					.receive(receive.toArray(new MediaKind[0]));
+
+			for (String[] urls : iceServers) {
+				options.iceServer(urls);
+			}
+
+			PeerImpl peer = (PeerImpl) rtc.createPeer(options.build());
+
+			return peer.toPublic(HttpSession.negotiate(rtc, peer, endpoint, bearerToken, timeout)
+					.thenApply(resource -> new Session(rtc, peer, resource, bearerToken, timeout)));
+		}
+	}
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Whip.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Whip.java
new file mode 100644
index 00000000..044b2b0e
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Whip.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static java.util.Objects.requireNonNull;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Publishes tracks to a server with the WebRTC HTTP Ingestion Protocol (WHIP,
+ * RFC 9725), as media servers and streaming services take it:
+ * 
{@code
+ * Whip.Session session = Whip.to(URI.create("https://example.com/live/whip"))
+ *         .bearerToken(streamKey)
+ *         .track(rtc.media().camera())
+ *         .track(rtc.media().microphone())
+ *         .publish(rtc)
+ *         .get(15, TimeUnit.SECONDS);
+ * ...
+ * session.close();
+ * }

+ * The offer goes to the endpoint with all local candidates, as WHIP servers
+ * expect, and the session's future completes once the peer is connected.
+ * Closing the session ends it on the server; the tracks stay open.
+ *
+ * @see RFC 9725: WHIP
+ */
+public final class Whip {
+
+	private Whip() {
+	}
+
+	/**
+	 * @param endpoint the WHIP endpoint, an http or https URL.
+	 *
+	 * @return a builder that publishes to the endpoint.
+	 */
+	public static Builder to(URI endpoint) {
+		return new Builder(HttpSession.checkEndpoint(endpoint));
+	}
+
+
+
+	/** Tracks published over WHIP. Closing it ends the session on the server. */
+	public static final class Session extends HttpSession {
+
+		Session(WebRtc rtc, Peer peer, URI resource, String bearerToken, Duration timeout) {
+			super(rtc, peer, resource, bearerToken, timeout);
+		}
+	}
+
+
+
+	/** Builds a WHIP {@link Session}. */
+	public static final class Builder {
+
+		private final URI endpoint;
+
+		private final List iceServers = new ArrayList<>();
+
+		private final List tracks = new ArrayList<>();
+
+		private String bearerToken;
+
+		private Duration timeout = Duration.ofSeconds(15);
+
+
+		private Builder(URI endpoint) {
+			this.endpoint = endpoint;
+		}
+
+		/**
+		 * @param track a track of the runtime to publish. Tracks share one
+		 *              stream, so the server keeps audio and video in sync.
+		 *
+		 * @return this builder.
+		 */
+		public Builder track(LocalTrack track) {
+			tracks.add(requireNonNull(track, "track"));
+			return this;
+		}
+
+		/**
+		 * @param token the token the endpoint authorizes the request with,
+		 *              sent as {@code Authorization: Bearer }; for many
+		 *              services, the stream key.
+		 *
+		 * @return this builder.
+		 */
+		public Builder bearerToken(String token) {
+			this.bearerToken = requireNonNull(token, "token");
+			return this;
+		}
+
+		/**
+		 * @param urls a STUN or TURN server without credentials.
+		 *
+		 * @return this builder.
+		 */
+		public Builder iceServer(String... urls) {
+			iceServers.add(urls.clone());
+			return this;
+		}
+
+		/**
+		 * @param timeout how long the session may take to be connected; 15
+		 *                seconds by default.
+		 *
+		 * @return this builder.
+		 */
+		public Builder timeout(Duration timeout) {
+			requireNonNull(timeout, "timeout");
+
+			if (timeout.isNegative() || timeout.isZero()) {
+				throw new IllegalArgumentException("The timeout must be positive, got " + timeout);
+			}
+
+			this.timeout = timeout;
+			return this;
+		}
+
+		/**
+		 * Creates a peer of the runtime that sends the tracks and negotiates it
+		 * with the endpoint.
+		 *
+		 * @param rtc the runtime that owns the peer and the tracks.
+		 *
+		 * @return a future with the session once its peer is connected. It
+		 * fails if the endpoint refuses the offer or the peer does not connect
+		 * in time, with nothing left behind on the server.
+		 *
+		 * @throws IllegalStateException if no track was given.
+		 */
+		public CompletableFuture publish(WebRtc rtc) {
+			requireNonNull(rtc, "rtc");
+
+			if (tracks.isEmpty()) {
+				throw new IllegalStateException("Give at least one track to publish");
+			}
+
+			PeerOptions.Builder options = PeerOptions.builder()
+					.name("whip-" + endpoint.getHost())
+					.negotiation(Negotiation.MANUAL);
+
+			for (String[] urls : iceServers) {
+				options.iceServer(urls);
+			}
+
+			PeerImpl peer = (PeerImpl) rtc.createPeer(options.build());
+
+			for (LocalTrack track : tracks) {
+				peer.addTrack(track);
+			}
+
+			return peer.toPublic(HttpSession.negotiate(rtc, peer, endpoint, bearerToken, timeout)
+					.thenApply(resource -> new Session(rtc, peer, resource, bearerToken, timeout)));
+		}
+	}
+}
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/HttpSessionTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/HttpSessionTests.java
new file mode 100644
index 00000000..246d5fd8
--- /dev/null
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/HttpSessionTests.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import dev.onvoid.webrtc.RTCSdpType;
+import dev.onvoid.webrtc.RTCSessionDescription;
+import dev.onvoid.webrtc.media.video.NativeI420Buffer;
+import dev.onvoid.webrtc.media.video.VideoFrame;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class HttpSessionTests extends ClientTestBase {
+
+	private ScheduledExecutorService feeder;
+
+	private CustomVideo video;
+
+
+	@BeforeEach
+	void feedVideo() {
+		video = rtc.media().customVideo("video");
+		feeder = Executors.newSingleThreadScheduledExecutor();
+		feeder.scheduleAtFixedRate(() -> {
+			NativeI420Buffer buffer = NativeI420Buffer.allocate(160, 120);
+			ByteBuffer y = buffer.getDataY();
+			for (int i = 0; i < y.capacity(); i++) {
+				y.put(i, (byte) i);
+			}
+			VideoFrame frame = new VideoFrame(buffer, System.nanoTime());
+			video.push(frame);
+			frame.release();
+		}, 0, 33, TimeUnit.MILLISECONDS);
+	}
+
+	@AfterEach
+	void stopFeeding() throws InterruptedException {
+		feeder.shutdownNow();
+		feeder.awaitTermination(5, TimeUnit.SECONDS);
+	}
+
+	@Test
+	void whepPlays() throws Exception {
+		Peer[] server = new Peer[1];
+
+		try (TestHttpEndpoint endpoint = new TestHttpEndpoint(offer -> {
+			// The server's side sends the stream.
+			server[0] = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build());
+			server[0].addTrack(video);
+			return answer(server[0], offer);
+		}, 201)) {
+			Whep.Session session = await(Whep.from(endpoint.uri()).bearerToken("secret").play(rtc));
+
+			CompletableFuture width = new CompletableFuture<>();
+			session.peer().onTrack(track -> track.asVideo().onFrame(frame -> width.complete(frame.buffer.getWidth())));
+
+			assertEquals(160, (int) await(width));
+			assertEquals("Bearer secret", endpoint.requests.get(0).authorization);
+			assertEquals(URI.create(endpoint.uri().resolve("/live/session/1").toString()), session.resource());
+
+			await(session.closeAsync());
+
+			assertEquals("DELETE /live/session/1", endpoint.requests.get(1).toString());
+			assertEquals(PeerState.CLOSED, session.peer().state());
+		}
+	}
+
+	@Test
+	void whipPublishes() throws Exception {
+		CompletableFuture published = new CompletableFuture<>();
+
+		try (TestHttpEndpoint endpoint = new TestHttpEndpoint(offer -> {
+			// The server's side receives the stream.
+			Peer server = rtc.createPeer(PeerOptions.builder().negotiation(Negotiation.MANUAL).build());
+			server.onTrack(published::complete);
+			return answer(server, offer);
+		}, 201)) {
+			Whip.Session session = await(Whip.to(endpoint.uri()).track(video).publish(rtc));
+
+			assertEquals(MediaKind.VIDEO, await(published).kind());
+
+			// Before the endpoint goes, which would fail the DELETE.
+			await(session.closeAsync());
+			assertEquals("DELETE /live/session/1", endpoint.requests.get(1).toString());
+		}
+	}
+
+	@Test
+	void refusedOfferLeavesNothing() throws Exception {
+		try (TestHttpEndpoint endpoint = new TestHttpEndpoint(offer -> "", 403)) {
+			ExecutionException failed = assertThrows(ExecutionException.class,
+					() -> Whep.from(endpoint.uri()).play(rtc).get(TIMEOUT_SECONDS, TimeUnit.SECONDS));
+
+			assertInstanceOf(WebRtcException.class, failed.getCause());
+			assertTrue(failed.getCause().getMessage().contains("403"), failed.getCause().getMessage());
+
+			long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+			while (!rtc.peers().isEmpty() && System.nanoTime() < deadline) {
+				Thread.sleep(50);
+			}
+			assertTrue(rtc.peers().isEmpty(), () -> "Peers left: " + rtc.peers());
+		}
+	}
+
+	private static String answer(Peer peer, String offer) {
+		try {
+			return peer.negotiator()
+					.answer(new RTCSessionDescription(RTCSdpType.OFFER, offer), Gathering.COMPLETE)
+					.get(TIMEOUT_SECONDS, TimeUnit.SECONDS).sdp;
+		}
+		catch (Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+}
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/TestHttpEndpoint.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/TestHttpEndpoint.java
new file mode 100644
index 00000000..dc316ae0
--- /dev/null
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/TestHttpEndpoint.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Function;
+
+/**
+ * Just enough HTTP for WHIP and WHEP tests. A POST is answered with what the
+ * handler returns for the offer, and creates a resource; a DELETE of that
+ * resource returns 200.
+ */
+final class TestHttpEndpoint implements AutoCloseable {
+
+	/** A request the endpoint received. */
+	static final class Request {
+
+		final String method;
+
+		final String path;
+
+		final String authorization;
+
+
+		Request(String method, String path, String authorization) {
+			this.method = method;
+			this.path = path;
+			this.authorization = authorization;
+		}
+
+		@Override
+		public String toString() {
+			return method + " " + path;
+		}
+	}
+
+	final List requests = new CopyOnWriteArrayList<>();
+
+	private final ServerSocket server;
+
+	private final Function answerer;
+
+	private final int status;
+
+
+	/**
+	 * @param answerer turns an offer into an answer.
+	 * @param status   the status of the answer, 201 for success.
+	 */
+	TestHttpEndpoint(Function answerer, int status) throws IOException {
+		this.answerer = answerer;
+		this.status = status;
+		this.server = new ServerSocket(0, 10, InetAddress.getLoopbackAddress());
+
+		Thread thread = new Thread(this::serve, "test-http-endpoint");
+		thread.setDaemon(true);
+		thread.start();
+	}
+
+	URI uri() {
+		return URI.create("http://127.0.0.1:" + server.getLocalPort() + "/live/endpoint");
+	}
+
+	@Override
+	public void close() throws IOException {
+		server.close();
+	}
+
+	private void serve() {
+		while (!server.isClosed()) {
+			try (Socket socket = server.accept()) {
+				handle(socket);
+			}
+			catch (IOException e) {
+				// Closed.
+			}
+		}
+	}
+
+	private void handle(Socket socket) throws IOException {
+		InputStream in = socket.getInputStream();
+		String[] requestLine = readLine(in).split(" ");
+		String authorization = null;
+		int length = 0;
+		String line;
+
+		while (!(line = readLine(in)).isEmpty()) {
+			String name = line.substring(0, line.indexOf(':')).trim().toLowerCase(Locale.ROOT);
+			String value = line.substring(line.indexOf(':') + 1).trim();
+
+			if (name.equals("content-length")) {
+				length = Integer.parseInt(value);
+			}
+			else if (name.equals("authorization")) {
+				authorization = value;
+			}
+		}
+
+		byte[] body = new byte[length];
+		int read = 0;
+		while (read < length) {
+			read += in.read(body, read, length - read);
+		}
+
+		requests.add(new Request(requestLine[0], requestLine[1], authorization));
+
+		OutputStream out = socket.getOutputStream();
+
+		if (requestLine[0].equals("POST")) {
+			if (status != 201) {
+				respond(out, status + " Refused", "", "not allowed");
+				return;
+			}
+
+			String answer = answerer.apply(new String(body, StandardCharsets.UTF_8));
+			respond(out, "201 Created", "Location: /live/session/1\r\n", answer);
+		}
+		else {
+			respond(out, "200 OK", "", "");
+		}
+	}
+
+	private static void respond(OutputStream out, String status, String headers, String body) throws IOException {
+		byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+		String head = "HTTP/1.1 " + status + "\r\nContent-Type: application/sdp\r\n" + headers
+				+ "Content-Length: " + bytes.length + "\r\nConnection: close\r\n\r\n";
+
+		out.write(head.getBytes(StandardCharsets.US_ASCII));
+		out.write(bytes);
+		out.flush();
+	}
+
+	private static String readLine(InputStream in) throws IOException {
+		ByteArrayOutputStream line = new ByteArrayOutputStream();
+		int c;
+
+		while ((c = in.read()) != -1 && c != '\n') {
+			if (c != '\r') {
+				line.write(c);
+			}
+		}
+
+		return new String(line.toByteArray(), StandardCharsets.US_ASCII);
+	}
+}

From a824cd99b86139c4efb0ab4977c2221c1f932346 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 17:39:36 +0200
Subject: [PATCH 15/25] Keep the JVM crash report of failed native test runs

A native crash ends the test JVM with an hs_err report beside the module,
which the upload left out. The client tests also note each test in the
surefire reports before it starts, so the last line names the one that ran.
---
 .github/actions/test-natives/action.yml       |  6 +++++-
 .../onvoid/webrtc/client/ClientTestBase.java  | 19 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/.github/actions/test-natives/action.yml b/.github/actions/test-natives/action.yml
index 84b5ad76..62aab303 100644
--- a/.github/actions/test-natives/action.yml
+++ b/.github/actions/test-natives/action.yml
@@ -151,6 +151,10 @@ runs:
       uses: actions/upload-artifact@v7
       with:
         name: test-reports-native-${{ inputs.platform-name }}
-        path: '**/target/surefire-reports/'
+        # A native crash ends the test JVM with a report of its own beside the
+        # module, which names the thread that crashed and what it ran.
+        path: |
+          **/target/surefire-reports/
+          **/hs_err_pid*.log
         if-no-files-found: ignore
         retention-days: 7
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java
index 0580e3a0..a9720af6 100644
--- a/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/ClientTestBase.java
@@ -18,6 +18,14 @@
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.time.LocalTime;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -25,6 +33,7 @@
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 
@@ -43,7 +52,15 @@ abstract class ClientTestBase {
 
 
 	@BeforeEach
-	void createRuntime() {
+	void createRuntime(TestInfo test) throws IOException {
+		// A native crash ends the test JVM before any report names the test
+		// that ran; the last line of this file does.
+		Path log = Paths.get("target", "surefire-reports", "client-tests.log");
+		Files.createDirectories(log.getParent());
+		Files.write(log, Collections.singletonList(LocalTime.now() + " " + getClass().getSimpleName()
+				+ "." + test.getTestMethod().map(Method::getName).orElse("?")),
+				StandardOpenOption.CREATE, StandardOpenOption.APPEND);
+
 		rtc = WebRtc.builder()
 				.audio(AudioMode.HEADLESS)
 				.errorHandler(errors::add)

From 7c52018cb708fb3c103f50793c65d5bbe5b695c6 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 18:37:25 +0200
Subject: [PATCH 16/25] Never dispose a runtime's factory under a closing peer

A peer whose Signaling.attach failed was closed in the background and
dropped from the runtime, so closing the runtime could dispose the factory,
and WebRTC's threads with it, while that peer's close() still ran on them.
The macOS runners crashed in RTCPeerConnection.close() that way.

createPeer now closes a failed peer before it throws. Closing the runtime
waits until no peer or track is left, and leaves the factory undisposed
if something did not close in time.
---
 .../java/dev/onvoid/webrtc/client/WebRtc.java | 64 ++++++++++++++-----
 1 file changed, 47 insertions(+), 17 deletions(-)

diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
index 28fce478..a43a58f7 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
@@ -35,6 +35,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 import dev.onvoid.webrtc.PeerConnectionFactory;
 import dev.onvoid.webrtc.media.audio.AudioDeviceModule;
@@ -235,8 +236,10 @@ public Peer createPeer(PeerOptions options) {
 			peer.open();
 		}
 		catch (RuntimeException e) {
+			// Closed before the error goes out, so that nothing of the failed
+			// peer still runs when the runtime closes and frees WebRTC's threads.
 			peers.remove(peer);
-			peer.closeInternal();
+			peer.closeInternal().join();
 			throw e;
 		}
 
@@ -330,15 +333,25 @@ boolean isClosing() {
 
 	private void shutdown() {
 		try {
-			List> peerCloses = new ArrayList<>();
+			// Peers and tracks created while the runtime closes are closed as
+			// well, so each is waited for until none is left.
+			boolean released = awaitAll(() -> {
+				List> closing = new ArrayList<>();
 
-			for (PeerImpl peer : new ArrayList<>(peers)) {
-				peerCloses.add(peer.closeInternal());
+				for (PeerImpl peer : new ArrayList<>(peers)) {
+					closing.add(peer.closeInternal());
+				}
+				return closing;
+			}, "peers") && awaitAll(media::closeAll, "local tracks");
+
+			if (!released) {
+				// Something may still run on WebRTC's threads, and disposing
+				// the factory would free them under it. Leaking is the lesser harm.
+				reportError(new WebRtcException("The factory of the runtime was left undisposed, "
+						+ "since not everything it created closed within " + closeTimeout));
+				return;
 			}
 
-			await(peerCloses, "peers");
-			await(media.closeAll(), "local tracks");
-
 			media.dispose();
 
 			try {
@@ -366,17 +379,34 @@ private void shutdown() {
 		}
 	}
 
-	private void await(List> futures, String what) {
-		try {
-			CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
-					.get(closeTimeout.toMillis(), TimeUnit.MILLISECONDS);
-		}
-		catch (InterruptedException e) {
-			Thread.currentThread().interrupt();
-		}
-		catch (Exception e) {
-			reportError(new WebRtcException("Closing the " + what + " did not finish within " + closeTimeout, e));
+	/**
+	 * Closes what the supplier names, again and again, until it names nothing.
+	 *
+	 * @return false if a round did not finish within the close timeout.
+	 */
+	private boolean awaitAll(Supplier>> closes, String what) {
+		for (int round = 0; round < 10; round++) {
+			List> futures = closes.get();
+
+			if (futures.isEmpty()) {
+				return true;
+			}
+
+			try {
+				CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
+						.get(closeTimeout.toMillis(), TimeUnit.MILLISECONDS);
+			}
+			catch (InterruptedException e) {
+				Thread.currentThread().interrupt();
+				return false;
+			}
+			catch (Exception e) {
+				reportError(new WebRtcException("Closing the " + what + " did not finish within " + closeTimeout, e));
+				return false;
+			}
 		}
+
+		return closes.get().isEmpty();
 	}
 
 	private void shutdownPools() {

From e584e3cf588034fa64b29de6049d7fa11107e9e4 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 19:22:58 +0200
Subject: [PATCH 17/25] Point the README to the client API

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 21a9c23d..18487848 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,7 @@ The library provides a comprehensive set of Java classes that map to the WebRTC
 ## Features
 
 - **Complete WebRTC API implementation** - Includes peer connections, media devices, data channels, and more
+- **Client API** - A high-level layer that negotiates by itself, keeps your code off WebRTC's threads, and closes everything with one call; with WHIP and WHEP sessions and a Swing video view
 - **Cross-platform support** - Works on Windows, macOS, and Linux (x64, ARM, ARM64)
 - **Media capabilities** - Audio and video capture from cameras and microphones
 - **Desktop capture** - Screen and application window sharing
@@ -34,6 +35,7 @@ The library provides a comprehensive set of Java classes that map to the WebRTC
 For more detailed information, check out the documentation:
 
 - [Quickstart](https://jrtc.dev/guide/get-started) - Get up and running quickly with webrtc-java
+- [Client API](https://jrtc.dev/guide/client-api) - Calls, media and data channels without the threading pitfalls
 - [Guides](https://jrtc.dev/guide/) - Comprehensive documentation on using the library
 - [Examples](https://jrtc.dev/guide/examples) - Sample code demonstrating various features
 - [Media Files](https://jrtc.dev/guide/media/media-files) - Sending video and audio files with the media module

From 1341eca347736a2dadc55a92f2ff067f0f33bbab Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 19:24:48 +0200
Subject: [PATCH 18/25] Put sources of the core API on client API tracks

LocalMedia.videoTrack and audioTrack wrap an existing source, such as those
a MediaFileSource of the media module feeds. The source stays the caller's
and is not disposed with the track.
---
 design/client-api.md                          |  6 ++-
 docs/guide/client-api.md                      | 15 ++++++
 .../dev/onvoid/webrtc/client/LocalMedia.java  | 35 +++++++++++++
 .../onvoid/webrtc/client/LocalMediaImpl.java  | 36 +++++++++++++
 .../dev/onvoid/webrtc/client/MediaTests.java  | 50 +++++++++++++++++++
 5 files changed, 141 insertions(+), 1 deletion(-)

diff --git a/design/client-api.md b/design/client-api.md
index 0a8cde8e..244e4418 100644
--- a/design/client-api.md
+++ b/design/client-api.md
@@ -799,7 +799,11 @@ Every test also fails if anything reached the runtime's error handler, including
 
 Next:
 - a guide page and an example;
-- phase 2 is done except the media module bridge and recording: `Whip.to(...).publish(rtc)` and
+- `LocalMedia.videoTrack(label, source)` and `audioTrack(label, source)` wrap sources of the core
+  API, such as those of the media module's `MediaFileSource`, which stay the caller's. Recording a
+  peer with `MediaRecorder` needs its native senders and receivers, which the client API keeps to
+  itself; that bridge is still open.
+- phase 2 is done except recording: `Whip.to(...).publish(rtc)` and
   `Whep.from(...).play(rtc)` return sessions over `HttpURLConnection` (Java 8), and `VideoView` is in
   `dev.onvoid.webrtc.client.swing`;
 - porting the existing examples (phases 2 and 3).
diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md
index 3c291b57..fc47ee23 100644
--- a/docs/guide/client-api.md
+++ b/docs/guide/client-api.md
@@ -282,6 +282,7 @@ Sender audioSender = peer.addTrack(microphone);
 | `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. |
 | `customVideo(String)` | Frames you push. |
 | `customAudio(String)` | Audio you push. `CUSTOM` and `HEADLESS` modes only. |
+| `videoTrack(String, VideoTrackSource)`, `audioTrack(String, AudioTrackSource)` | A source of the core API, such as one the [media module](/guide/media/media-files) feeds. The source stays yours. |
 
 ```java
 Devices devices = media.devices();
@@ -292,6 +293,20 @@ Tracks added without stream ids share one stream per peer, so that a browser pla
 
 One track may be sent on any number of peers. Opening a camera that is already open shares its capture when the options are the same, and fails with a `MediaException` when they differ.
 
+### Sending a Media File
+
+The media module plays files and RTSP streams into sources of the core API. `videoTrack` and `audioTrack` put them on peers like any other track:
+
+```java
+MediaFileSource file = new MediaFileSource(Paths.get("movie.mp4"));
+
+peer.addTrack(rtc.media().videoTrack("movie", file.getVideoSource()));
+peer.addTrack(rtc.media().audioTrack("movie", file.getAudioSource()));
+file.play();
+```
+
+The sources belong to the `MediaFileSource`, not to the tracks: close the runtime, or the tracks, before the file. The file's audio source is pushed audio, so the runtime needs the `CUSTOM` or `HEADLESS` audio mode.
+
 ### Pushing Video and Audio
 
 ```java
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
index 90858337..0808725e 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
@@ -17,6 +17,9 @@
 package dev.onvoid.webrtc.client;
 
 import dev.onvoid.webrtc.media.audio.AudioOptions;
+import dev.onvoid.webrtc.media.audio.AudioTrackSource;
+import dev.onvoid.webrtc.media.audio.CustomAudioSource;
+import dev.onvoid.webrtc.media.video.VideoTrackSource;
 
 /**
  * Creates the tracks a runtime sends, from {@link WebRtc#media()}. Every track
@@ -95,6 +98,38 @@ public interface LocalMedia {
 	 */
 	CustomVideo customVideo(String label);
 
+	/**
+	 * Creates a track over a video source of the low-level API, such as the
+	 * video source of a {@code MediaFileSource} of the media module. The
+	 * source stays the caller's: closing the track does not dispose it, and it
+	 * must outlive the track.
+	 *
+	 * @param label  the track label.
+	 * @param source the source that feeds the track.
+	 *
+	 * @return the track.
+	 */
+	LocalVideo videoTrack(String label, VideoTrackSource source);
+
+	/**
+	 * Creates a track over an audio source of the low-level API, such as the
+	 * audio source of a {@code MediaFileSource} of the media module. The
+	 * source stays the caller's: closing the track does not dispose it, and it
+	 * must outlive the track.
+	 *
+	 * @param label  the track label.
+	 * @param source the source that feeds the track. A {@link CustomAudioSource}
+	 *               needs a runtime built with {@link AudioMode#CUSTOM} or
+	 *               {@link AudioMode#HEADLESS}, any other source {@link
+	 *               AudioMode#DEVICES}.
+	 *
+	 * @return the track.
+	 *
+	 * @throws ConfigurationException if the source does not fit the runtime's
+	 *                                {@link AudioMode}.
+	 */
+	LocalAudio audioTrack(String label, AudioTrackSource source);
+
 	/**
 	 * Creates an audio track fed with audio the application pushes.
 	 *
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
index 6d62ec09..ff0bef7f 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
@@ -40,6 +40,7 @@
 import dev.onvoid.webrtc.media.video.VideoDevice;
 import dev.onvoid.webrtc.media.video.VideoDeviceSource;
 import dev.onvoid.webrtc.media.video.VideoTrack;
+import dev.onvoid.webrtc.media.video.VideoTrackSource;
 
 /**
  * The {@link LocalMedia} of a {@link WebRtc} runtime. It keeps every track it
@@ -201,6 +202,41 @@ public CustomVideo customVideo(String label) {
 		}
 	}
 
+	@Override
+	public LocalVideo videoTrack(String label, VideoTrackSource source) {
+		requireNonNull(label, "label");
+		requireNonNull(source, "source");
+		rtc.checkOpen();
+
+		VideoTrack track = rtc.factory.createVideoTrack(label, source);
+
+		// The source is the caller's, and outlives the track.
+		return register(new LocalVideoImpl(this, track, () -> { }));
+	}
+
+	@Override
+	public LocalAudio audioTrack(String label, AudioTrackSource source) {
+		requireNonNull(label, "label");
+		requireNonNull(source, "source");
+		rtc.checkOpen();
+
+		boolean pushed = source instanceof CustomAudioSource;
+
+		if (pushed && rtc.audioMode == AudioMode.DEVICES) {
+			throw new ConfigurationException("A CustomAudioSource needs a runtime built with AudioMode.CUSTOM or "
+					+ "AudioMode.HEADLESS; this one captures the microphone");
+		}
+		if (!pushed && rtc.audioMode != AudioMode.DEVICES) {
+			throw new ConfigurationException("A captured audio source needs a runtime built with AudioMode.DEVICES; "
+					+ "this one was built with AudioMode." + rtc.audioMode);
+		}
+
+		AudioTrack track = rtc.factory.createAudioTrack(label, source);
+
+		// The source is the caller's, and outlives the track.
+		return register(new LocalAudioImpl(this, track, () -> { }));
+	}
+
 	@Override
 	public CustomAudio customAudio(String label) {
 		requireNonNull(label, "label");
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java
index 62132e1f..7a1c456e 100644
--- a/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/MediaTests.java
@@ -41,6 +41,8 @@
 
 import dev.onvoid.webrtc.RTCStats;
 import dev.onvoid.webrtc.RTCStatsType;
+import dev.onvoid.webrtc.media.audio.CustomAudioSource;
+import dev.onvoid.webrtc.media.video.CustomVideoSource;
 import dev.onvoid.webrtc.media.video.NativeI420Buffer;
 import dev.onvoid.webrtc.media.video.VideoFrame;
 
@@ -291,6 +293,54 @@ void tonesSent() throws Exception {
 		assertInstanceOf(IllegalStateException.class, refused.getCause());
 	}
 
+	@Test
+	void tracksOverOwnSources() throws Exception {
+		Peer[] peers = pair();
+		CustomVideoSource videoSource = new CustomVideoSource();
+		CustomAudioSource audioSource = new CustomAudioSource();
+
+		try {
+			LocalVideo video = rtc.media().videoTrack("own-video", videoSource);
+			LocalAudio audio = rtc.media().audioTrack("own-audio", audioSource);
+			CompletableFuture width = new CompletableFuture<>();
+			CompletableFuture ended = new CompletableFuture<>();
+
+			peers[1].onTrack(track -> {
+				if (track.isVideo()) {
+					track.asVideo().onFrame(frame -> width.complete(frame.buffer.getWidth()));
+				}
+			});
+			peers[0].addTrack(video);
+			peers[0].addTrack(audio);
+
+			feeder.scheduleAtFixedRate(() -> {
+				VideoFrame frame = new VideoFrame(NativeI420Buffer.allocate(160, 120), System.nanoTime());
+				videoSource.pushFrame(frame);
+				frame.release();
+			}, 0, 33, TimeUnit.MILLISECONDS);
+
+			assertEquals(160, (int) await(width));
+
+			video.onEnded(() -> ended.complete(null));
+			video.close();
+			await(ended);
+
+			// The source is still the test's, and still takes frames.
+			VideoFrame frame = new VideoFrame(NativeI420Buffer.allocate(160, 120), System.nanoTime());
+			videoSource.pushFrame(frame);
+			frame.release();
+		}
+		finally {
+			feeder.shutdownNow();
+			feeder.awaitTermination(5, TimeUnit.SECONDS);
+
+			// Sources outlive the tracks over them.
+			rtc.close();
+			videoSource.dispose();
+			audioSource.dispose();
+		}
+	}
+
 	@Test
 	void audioModeChecked() {
 		assertThrows(ConfigurationException.class, () -> rtc.media().microphone());

From 3d1ad0600fe32fc283edbd623a49ca0adae8a159 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 19:51:29 +0200
Subject: [PATCH 19/25] Wait for whenClosed in the runtime close test

WebRtc.close() returns once every peer is closed; the whenClosed futures
complete afterwards on the peers' event threads, which a slow macOS runner
had not reached yet. The test now waits for them, and the javadoc and the
guide say what close() guarantees.
---
 docs/guide/client-api.md                                   | 2 +-
 webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java  | 5 +++++
 .../src/test/java/dev/onvoid/webrtc/client/PeerTests.java  | 7 +++++--
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md
index fc47ee23..b9f8ae0d 100644
--- a/docs/guide/client-api.md
+++ b/docs/guide/client-api.md
@@ -551,7 +551,7 @@ You close what you created, and closing an owner closes what it owns:
 
 `Peer.close()` and every other `close()` except the runtime's return at once, may be called more than once, and are fine inside a handler. After a peer closed, its only event is the `CLOSED` state. Operations that were pending fail with a `PeerClosedException`, and so do those called later. `closeAsync()` returns a future for callers that want to wait.
 
-`WebRtc.close()` is the one close that waits, for up to the close timeout, which makes it fit for try-with-resources.
+`WebRtc.close()` is the one close that waits, for up to the close timeout, which makes it fit for try-with-resources. Once it returns, every peer's `state()` is `CLOSED`; the `whenClosed` futures and `CLOSED` events follow on the peers' event threads, since waiting for them could never end when `close()` is called on an event thread.
 
 ## Errors
 
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
index a43a58f7..341237b7 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/WebRtc.java
@@ -292,6 +292,11 @@ public CompletableFuture closeAsync() {
 	 * Closes every peer, then every track, then the factory, and stops the
 	 * threads of this API. Waits for that up to the close timeout, which is
 	 * the one wait in this API. May be called more than once.
+	 * 

+	 * Once it returns, every peer's {@link Peer#state()} is {@link
+	 * PeerState#CLOSED}. The peers' {@code whenClosed} futures and their
+	 * {@code CLOSED} events follow on their event threads; waiting for those
+	 * here could never end when this is called on an event thread.
 	 */
 	@Override
 	public void close() {
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java
index feccec33..2c46a478 100644
--- a/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/PeerTests.java
@@ -363,9 +363,12 @@ void runtimeCloseClosesPeers() throws Exception {
 
 		rtc.close();
 
-		assertTrue(peers[0].whenClosed().isDone());
-		assertTrue(peers[1].whenClosed().isDone());
+		// Closed once close() returns; the futures and the CLOSED events follow
+		// on the event threads.
+		assertEquals(PeerState.CLOSED, peers[0].state());
 		assertEquals(PeerState.CLOSED, peers[1].state());
+		await(peers[0].whenClosed());
+		await(peers[1].whenClosed());
 		assertThrows(PeerClosedException.class, this::pair);
 	}
 

From 95a719b2028f78f0b2d811bc96c5537622f383b8 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 20:26:53 +0200
Subject: [PATCH 20/25] Report the last DTMF tone before the tones' future
 completes

The tone went to the event executor straight from WebRTC's thread, and the
future through the control executor, so on a fast control thread the future
could complete first. Both now go through the control executor, in order.
---
 .../src/main/java/dev/onvoid/webrtc/client/SenderImpl.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
index e10e74ac..af97184c 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
@@ -331,12 +331,15 @@ private void toneChanged(String tone) {
 		String current = tone == null ? "" : tone;
 
 		peer.control.execute(() -> {
+			// The handlers hear of the tone before the future of the tones
+			// completes, which also goes through the event executor.
+			peer.events.execute(() -> toneHandlers.accept(current));
+
 			if (current.isEmpty() && pendingTones != null) {
 				pendingTones.complete(null);
 				pendingTones = null;
 			}
 		});
-		peer.events.execute(() -> toneHandlers.accept(current));
 	}
 
 	private void checkUsable() {

From 35fae9b586828fcbd5f30efcb8a2d4f71ef595c7 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Mon, 19 Oct 2026 23:50:58 +0200
Subject: [PATCH 21/25] Test sending a media file through client API tracks

The guide's media file snippet now runs as a test of the media module. It
also showed that the runtime has to exist before the MediaFileSource, since
the runtime loads the native library the file's sources need; the guide
says so now.
---
 docs/guide/client-api.md                      |   3 +-
 .../media/player/ClientApiMediaFileTest.java  | 111 ++++++++++++++++++
 2 files changed, 113 insertions(+), 1 deletion(-)
 create mode 100644 webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/ClientApiMediaFileTest.java

diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md
index b9f8ae0d..c6713bf9 100644
--- a/docs/guide/client-api.md
+++ b/docs/guide/client-api.md
@@ -298,6 +298,7 @@ One track may be sent on any number of peers. Opening a camera that is already o
 The media module plays files and RTSP streams into sources of the core API. `videoTrack` and `audioTrack` put them on peers like any other track:
 
 ```java
+WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build();   // first: it loads the native library
 MediaFileSource file = new MediaFileSource(Paths.get("movie.mp4"));
 
 peer.addTrack(rtc.media().videoTrack("movie", file.getVideoSource()));
@@ -305,7 +306,7 @@ peer.addTrack(rtc.media().audioTrack("movie", file.getAudioSource()));
 file.play();
 ```
 
-The sources belong to the `MediaFileSource`, not to the tracks: close the runtime, or the tracks, before the file. The file's audio source is pushed audio, so the runtime needs the `CUSTOM` or `HEADLESS` audio mode.
+Create the runtime before the file: the runtime loads the native library that the file's sources need. The sources belong to the `MediaFileSource`, not to the tracks, so close the runtime, or the tracks, before the file. The file's audio source is pushed audio, so the runtime needs the `CUSTOM` or `HEADLESS` audio mode.
 
 ### Pushing Video and Audio
 
diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/ClientApiMediaFileTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/ClientApiMediaFileTest.java
new file mode 100644
index 00000000..d1d8ce0c
--- /dev/null
+++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/player/ClientApiMediaFileTest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.media.player;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import dev.onvoid.webrtc.client.AudioMode;
+import dev.onvoid.webrtc.client.Peer;
+import dev.onvoid.webrtc.client.PeerOptions;
+import dev.onvoid.webrtc.client.Signaling;
+import dev.onvoid.webrtc.client.SignalingPair;
+import dev.onvoid.webrtc.client.WebRtc;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+
+/**
+ * Sends a media file through tracks of the client API, as its guide shows.
+ *
+ * @author Alex Andres
+ */
+@Execution(ExecutionMode.SAME_THREAD)
+class ClientApiMediaFileTest {
+
+	private static final String ASSET = "/media-test.webm";
+
+	@Test
+	void sendsFileToPeer() throws Exception {
+		List errors = new CopyOnWriteArrayList<>();
+		AtomicInteger frames = new AtomicInteger();
+		AtomicInteger chunks = new AtomicInteger();
+		CountDownLatch received = new CountDownLatch(2);
+
+		// The runtime comes first, since it loads the native library the file's
+		// sources need. The file's audio is pushed audio, hence no microphone.
+		WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).errorHandler(errors::add).build();
+
+		try (MediaFileSource file = new MediaFileSource(asset())) {
+			try {
+				SignalingPair link = Signaling.loopback();
+				Peer sender = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build());
+				Peer receiver = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build());
+
+				receiver.onTrack(track -> {
+					if (track.isVideo()) {
+						track.asVideo().onFrame(frame -> {
+							if (frames.incrementAndGet() == 10) {
+								received.countDown();
+							}
+						});
+					}
+					else {
+						track.asAudio().onAudio(chunk -> {
+							if (chunks.incrementAndGet() == 50) {
+								received.countDown();
+							}
+						});
+					}
+				});
+
+				sender.addTrack(rtc.media().videoTrack("movie", file.getVideoSource()));
+				sender.addTrack(rtc.media().audioTrack("movie", file.getAudioSource()));
+				file.play();
+
+				assertTrue(received.await(15, TimeUnit.SECONDS),
+						"video frames: " + frames.get() + ", audio chunks: " + chunks.get());
+			}
+			finally {
+				// The runtime and its tracks close before the file and its sources.
+				rtc.close();
+			}
+		}
+
+		assertEquals(0, errors.size(), () -> "Errors: " + errors);
+	}
+
+	private static Path asset() throws Exception {
+		URL url = ClientApiMediaFileTest.class.getResource(ASSET);
+
+		assertNotNull(url, "Test asset " + ASSET + " is missing");
+
+		return Paths.get(url.toURI());
+	}
+
+}

From f88d96253cb07371f9a469bec10a98769532fd3b Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Tue, 20 Oct 2026 00:14:21 +0200
Subject: [PATCH 22/25] Switch cameras in a client API camera track

camera() returns a LocalCamera, whose switchCamera puts a new native track
over the other camera in place of the old one, on every sender, without
renegotiating. Cameras that cannot capture at once, like the two sensors of
a laptop camera, are handed over by closing the old one first, and a failed
switch keeps the camera the track had. Tested with a real camera; the test
skips without one.
---
 design/client-api.md                          |   8 +-
 docs/guide/client-api.md                      |  18 +-
 .../dev/onvoid/webrtc/client/LocalCamera.java |  47 +++++
 .../onvoid/webrtc/client/LocalCameraImpl.java | 178 ++++++++++++++++++
 .../dev/onvoid/webrtc/client/LocalMedia.java  |   4 +-
 .../onvoid/webrtc/client/LocalMediaImpl.java  |  18 +-
 .../onvoid/webrtc/client/LocalTrackImpl.java  |  59 +++++-
 .../onvoid/webrtc/client/LocalVideoImpl.java  |  27 ++-
 .../dev/onvoid/webrtc/client/SenderImpl.java  |  26 +++
 .../webrtc/client/CameraIntegrationTest.java  | 150 +++++++++++++++
 10 files changed, 512 insertions(+), 23 deletions(-)
 create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCamera.java
 create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCameraImpl.java
 create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/client/CameraIntegrationTest.java

diff --git a/design/client-api.md b/design/client-api.md
index 244e4418..152885b7 100644
--- a/design/client-api.md
+++ b/design/client-api.md
@@ -758,8 +758,12 @@ How the threading model is built:
 
 Where the code differs from the API above:
 
-- `LocalVideo.switchCamera()` is left for later: a WebRTC track cannot change its source, so it
-  needs a new native track behind the same `LocalVideo`, replaced on every sender.
+- `camera()` returns a `LocalCamera`, a `LocalVideo` with `device()` and `switchCamera(device)`;
+  other video tracks cannot switch. A WebRTC track cannot change its source, so a switch puts a new
+  native track over the new camera behind the same `LocalCamera`, on every sender. It opens the new
+  camera first, and when that fails, as with two sensors of one camera that cannot stream at once,
+  closes the current one before opening the new one, and reopens the current one if the new one
+  fails too. `CameraIntegrationTest` covers it with a real camera and skips without one.
 - DTMF is `Sender.sendTones()` and `Sender.onTone()` instead of a `DtmfSender` object; the future
   completes once the tones have been sent.
 - `Devices.screens()` and `windows()` ask the system on every call; only the camera, microphone and
diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md
index c6713bf9..b87d4462 100644
--- a/docs/guide/client-api.md
+++ b/docs/guide/client-api.md
@@ -264,7 +264,7 @@ A UI thread used as event executor is the same: waiting on it for a future of th
 ```java
 LocalMedia media = rtc.media();
 
-LocalVideo camera = media.camera(CameraOptions.builder()
+LocalCamera camera = media.camera(CameraOptions.builder()
         .device(media.devices().cameras().get(0))
         .resolution(1280, 720)
         .frameRate(30)
@@ -277,7 +277,7 @@ Sender audioSender = peer.addTrack(microphone);
 
 | Method | Track |
 |---|---|
-| `camera()`, `camera(CameraOptions)` | A camera. The supported capability nearest to the requested resolution and frame rate is used. |
+| `camera()`, `camera(CameraOptions)` | A camera, as a `LocalCamera` that can switch cameras. The supported capability nearest to the requested resolution and frame rate is used. |
 | `microphone()`, `microphone(AudioOptions)` | The microphone, with echo cancellation, noise suppression and gain control unless `AudioOptions` say otherwise. `DEVICES` mode only. |
 | `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. |
 | `customVideo(String)` | Frames you push. |
@@ -357,6 +357,20 @@ audioSender.sendTones("*99,1", Duration.ofMillis(200), Duration.ofMillis(100));
 
 The characters 0 to 9, A to D, `#` and `*` are tones, and a comma is a pause of 2 seconds. Tones still queued from an earlier call are replaced, and that call's future is cancelled. A video sender, or one whose remote peer accepts no telephone events, fails the future with an `IllegalStateException`. The [DTMF Sender](/guide/audio/dtmf-sender) guide has the details.
 
+### Switching Cameras
+
+A `LocalCamera` switches to another camera without renegotiating. Every peer that sends it keeps sending it, and preview handlers keep getting frames:
+
+```java
+LocalCamera camera = media.camera();
+peer.addTrack(camera);
+
+camera.switchCamera(media.devices().cameras().get(1))
+        .thenRun(() -> System.out.println("Now sending " + camera.device().getName()));
+```
+
+The new camera keeps the resolution and frame rate of the old one, as far as it supports them. The switch opens the new camera before it closes the old one, so that the peers see no gap. Some devices cannot capture with two cameras at once, such as the two sensors of a laptop camera; then the old camera closes first, and the switch takes about half a second. If the new camera cannot be opened, the future fails with a `MediaException`, and the track keeps sending the camera it had.
+
 ### Devices
 
 `media.devices()` returns the cameras, microphones and speakers. The first call reads them from the system; after that, the lists are snapshots kept current in the background, so reading them is cheap on any thread. `screens()` and `windows()` ask the system on each call.
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCamera.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCamera.java
new file mode 100644
index 00000000..421097c3
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCamera.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import java.util.concurrent.CompletableFuture;
+
+import dev.onvoid.webrtc.media.video.VideoDevice;
+
+/**
+ * A camera this side sends, made by {@link LocalMedia#camera()}.
+ */
+public interface LocalCamera extends LocalVideo {
+
+	/**
+	 * @return the camera captured now.
+	 */
+	VideoDevice device();
+
+	/**
+	 * Captures another camera, with the same resolution and frame rate, in
+	 * this track. Every peer that sends the track keeps sending it, without
+	 * renegotiating, and local preview handlers keep getting frames. The
+	 * camera captured before is closed unless another track still uses it.
+	 *
+	 * @param device one of {@link Devices#cameras()}.
+	 *
+	 * @return a future that completes once every peer sends the new camera. It
+	 * fails with a {@link MediaException} if the camera cannot be opened, or is
+	 * already open with other options, and the track keeps its camera then.
+	 */
+	CompletableFuture switchCamera(VideoDevice device);
+
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCameraImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCameraImpl.java
new file mode 100644
index 00000000..16a5492e
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalCameraImpl.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import dev.onvoid.webrtc.media.video.VideoDevice;
+import dev.onvoid.webrtc.media.video.VideoTrack;
+
+/**
+ * A camera track. A WebRTC track cannot change its source, so switching the
+ * camera puts a new native track over the new camera's capture in place of the
+ * old one, on the track and on every sender, and only then lets the old one go.
+ */
+final class LocalCameraImpl extends LocalVideoImpl implements LocalCamera {
+
+	/** Guarded by this. */
+	private LocalMediaImpl.SharedCamera camera;
+
+
+	LocalCameraImpl(LocalMediaImpl media, VideoTrack track, LocalMediaImpl.SharedCamera camera) {
+		super(media, track, () -> media.releaseCamera(camera));
+
+		this.camera = camera;
+	}
+
+	@Override
+	public synchronized VideoDevice device() {
+		return camera.options.device;
+	}
+
+	@Override
+	public CompletableFuture switchCamera(VideoDevice device) {
+		requireNonNull(device, "device");
+
+		CompletableFuture switched = new CompletableFuture<>();
+
+		// Opening a camera can take a moment, so it happens off the caller's thread.
+		media.executor.execute(() -> {
+			try {
+				switchTo(device, switched);
+			}
+			catch (Throwable e) {
+				switched.completeExceptionally(e instanceof WebRtcException
+						? e : new MediaException("Switching to the camera " + device.getName() + " failed", e));
+			}
+		});
+
+		return switched;
+	}
+
+	private void switchTo(VideoDevice device, CompletableFuture switched) {
+		LocalMediaImpl.SharedCamera current;
+
+		synchronized (this) {
+			current = camera;
+		}
+
+		if (current.options.device.equals(device)) {
+			switched.complete(null);
+			return;
+		}
+
+		CameraOptions target = current.options.withDevice(device);
+		LocalMediaImpl.SharedCamera next;
+		boolean currentReleased = false;
+		MediaException failure = null;
+
+		try {
+			// Opened first, so that peers see no gap.
+			next = media.openCamera(target);
+		}
+		catch (MediaException beside) {
+			// Some cameras cannot capture at the same time, such as two
+			// sensors of one device. Unless another track shares the current
+			// camera, it is let go first, and comes back if the new one fails.
+			if (!media.isOnlyUser(current)) {
+				throw beside;
+			}
+
+			media.releaseCamera(current);
+			currentReleased = true;
+
+			try {
+				next = media.openCamera(target);
+			}
+			catch (MediaException instead) {
+				failure = instead;
+				next = media.openCamera(current.options);
+			}
+		}
+
+		replace(next, currentReleased);
+
+		if (failure != null) {
+			switched.completeExceptionally(failure);
+		}
+		else {
+			switched.complete(null);
+		}
+	}
+
+	/**
+	 * Puts a track over the camera in place of the current one, on this
+	 * track and on every sender, and lets the old one go once no sender sends
+	 * it any more.
+	 */
+	private void replace(LocalMediaImpl.SharedCamera next, boolean oldCameraReleased) {
+		VideoTrack nextTrack;
+
+		try {
+			nextTrack = rtc.factory.createVideoTrack("camera-" + UUID.randomUUID(), next.source);
+		}
+		catch (RuntimeException e) {
+			media.releaseCamera(next);
+			throw new MediaException("The track of the camera " + next.options.device.getName()
+					+ " could not be created", e);
+		}
+
+		LocalTrackImpl.Replaced replaced;
+
+		synchronized (this) {
+			replaced = replaceNative(nextTrack, () -> media.releaseCamera(next));
+
+			if (replaced != null) {
+				moveSinks((VideoTrack) replaced.track);
+				camera = next;
+			}
+		}
+
+		if (replaced == null) {
+			// Closed meanwhile.
+			nextTrack.dispose();
+			media.releaseCamera(next);
+			throw new MediaException("The camera track is closed");
+		}
+
+		List> sending = new ArrayList<>();
+
+		for (SenderImpl sender : replaced.senders) {
+			sending.add(sender.trackReplaced(this));
+		}
+
+		try {
+			CompletableFuture.allOf(sending.toArray(new CompletableFuture[0]))
+					.get(rtc.operationTimeout.toMillis(), TimeUnit.MILLISECONDS);
+		}
+		catch (Exception e) {
+			rtc.reportError(new WebRtcException("Not every peer took the new camera in time", e));
+		}
+
+		replaced.track.dispose();
+
+		if (!oldCameraReleased) {
+			replaced.release.run();
+		}
+	}
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
index 0808725e..5bbf2b4b 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMedia.java
@@ -42,7 +42,7 @@ public interface LocalMedia {
 	 *
 	 * @throws MediaException if there is no camera or it cannot be opened.
 	 */
-	LocalVideo camera();
+	LocalCamera camera();
 
 	/**
 	 * Opens a camera. A camera that is already open is shared when the options
@@ -55,7 +55,7 @@ public interface LocalMedia {
 	 * @throws MediaException if the camera cannot be opened, or is already open
 	 *                        with other options.
 	 */
-	LocalVideo camera(CameraOptions options);
+	LocalCamera camera(CameraOptions options);
 
 	/**
 	 * Captures the microphone selected in {@link Devices}, with echo
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
index ff0bef7f..2e9d8927 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalMediaImpl.java
@@ -84,12 +84,12 @@ public synchronized Devices devices() {
 	}
 
 	@Override
-	public LocalVideo camera() {
+	public LocalCamera camera() {
 		return camera(CameraOptions.builder().build());
 	}
 
 	@Override
-	public LocalVideo camera(CameraOptions options) {
+	public LocalCamera camera(CameraOptions options) {
 		requireNonNull(options, "options");
 		rtc.checkOpen();
 
@@ -111,7 +111,7 @@ public LocalVideo camera(CameraOptions options) {
 		try {
 			VideoTrack track = rtc.factory.createVideoTrack(newId("camera"), camera.source);
 
-			return register(new LocalVideoImpl(this, track, () -> releaseCamera(camera)));
+			return register(new LocalCameraImpl(this, track, camera));
 		}
 		catch (RuntimeException e) {
 			releaseCamera(camera);
@@ -293,7 +293,7 @@ private  T register(T track) {
 		return track;
 	}
 
-	private SharedCamera openCamera(CameraOptions options) {
+	SharedCamera openCamera(CameraOptions options) {
 		String key = options.device.getDescriptor();
 
 		synchronized (cameras) {
@@ -337,7 +337,13 @@ private SharedCamera openCamera(CameraOptions options) {
 		}
 	}
 
-	private void releaseCamera(SharedCamera camera) {
+	boolean isOnlyUser(SharedCamera camera) {
+		synchronized (cameras) {
+			return camera.users == 1;
+		}
+	}
+
+	void releaseCamera(SharedCamera camera) {
 		synchronized (cameras) {
 			if (--camera.users > 0) {
 				return;
@@ -384,7 +390,7 @@ private static String newId(String kind) {
 
 
 
-	private static final class SharedCamera {
+	static final class SharedCamera {
 
 		final String key;
 
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java
index c71bae0b..0ba77a13 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalTrackImpl.java
@@ -38,13 +38,15 @@ abstract class LocalTrackImpl implements LocalTrack {
 
 	final LocalMediaImpl media;
 
-	private final MediaStreamTrack track;
+	/** Replaced when a camera switches; guarded by this, read by senders. */
+	private volatile MediaStreamTrack track;
 
 	private final MediaKind kind;
 
 	private final String id;
 
-	private final Runnable releaseSource;
+	/** Guarded by this. */
+	private Runnable releaseSource;
 
 	private final CompletableFuture closed = new CompletableFuture<>();
 
@@ -136,6 +138,27 @@ MediaStreamTrack nativeTrack() {
 		return track;
 	}
 
+	/**
+	 * Puts another native track in place of the current one, keeping whether
+	 * it is enabled.
+	 *
+	 * @return the track and release replaced, and the senders that send it now;
+	 * null once the track ended.
+	 */
+	synchronized Replaced replaceNative(MediaStreamTrack next, Runnable nextRelease) {
+		if (ended) {
+			return null;
+		}
+
+		Replaced replaced = new Replaced(track, releaseSource, new ArrayList<>(senders));
+
+		next.setEnabled(enabled);
+		track = next;
+		releaseSource = nextRelease;
+
+		return replaced;
+	}
+
 	synchronized boolean register(SenderImpl sender) {
 		if (ended) {
 			return false;
@@ -192,8 +215,17 @@ void beforeRelease() {
 	private void release() {
 		try {
 			beforeRelease();
-			track.dispose();
-			releaseSource.run();
+
+			MediaStreamTrack last;
+			Runnable lastRelease;
+
+			synchronized (this) {
+				last = track;
+				lastRelease = releaseSource;
+			}
+
+			last.dispose();
+			lastRelease.run();
 		}
 		catch (Throwable e) {
 			rtc.reportError(e);
@@ -220,4 +252,23 @@ private void release() {
 
 		closed.complete(null);
 	}
+
+
+
+	/** What {@link #replaceNative} replaced. */
+	static final class Replaced {
+
+		final MediaStreamTrack track;
+
+		final Runnable release;
+
+		final List senders;
+
+
+		Replaced(MediaStreamTrack track, Runnable release, List senders) {
+			this.track = track;
+			this.release = release;
+			this.senders = senders;
+		}
+	}
 }
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java
index fcb09530..56835996 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/LocalVideoImpl.java
@@ -30,8 +30,6 @@
  */
 class LocalVideoImpl extends LocalTrackImpl implements LocalVideo {
 
-	private final VideoTrack track;
-
 	// Guarded by this.
 
 	private final List dispatchers = new ArrayList<>();
@@ -39,8 +37,6 @@ class LocalVideoImpl extends LocalTrackImpl implements LocalVideo {
 
 	LocalVideoImpl(LocalMediaImpl media, VideoTrack track, Runnable releaseSource) {
 		super(media, track, MediaKind.VIDEO, releaseSource);
-
-		this.track = track;
 	}
 
 	@Override
@@ -56,7 +52,7 @@ public Subscription onFrame(Consumer handler) {
 			}
 
 			dispatchers.add(dispatcher);
-			track.addSink(dispatcher);
+			videoTrack().addSink(dispatcher);
 		}
 
 		return () -> {
@@ -64,7 +60,7 @@ public Subscription onFrame(Consumer handler) {
 
 			synchronized (LocalVideoImpl.this) {
 				if (dispatchers.remove(dispatcher) && isLive()) {
-					track.removeSink(dispatcher);
+					videoTrack().removeSink(dispatcher);
 				}
 			}
 		};
@@ -80,8 +76,25 @@ void beforeRelease() {
 		}
 
 		for (FrameDispatcher dispatcher : attached) {
-			track.removeSink(dispatcher);
+			videoTrack().removeSink(dispatcher);
 			dispatcher.close();
 		}
 	}
+
+	/**
+	 * Moves the preview handlers from a track that was replaced to the
+	 * current one. Called under this object's lock.
+	 */
+	void moveSinks(VideoTrack from) {
+		VideoTrack to = videoTrack();
+
+		for (FrameDispatcher dispatcher : dispatchers) {
+			from.removeSink(dispatcher);
+			to.addSink(dispatcher);
+		}
+	}
+
+	private VideoTrack videoTrack() {
+		return (VideoTrack) nativeTrack();
+	}
 }
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
index af97184c..844581a1 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
@@ -281,6 +281,32 @@ CompletableFuture trackClosed(LocalTrackImpl closedTrack) {
 		return done;
 	}
 
+	/**
+	 * Sends the native track a local track uses now, after it replaced the one
+	 * before, without renegotiating.
+	 *
+	 * @return a future that completes once the sender sends the new track.
+	 */
+	CompletableFuture trackReplaced(LocalTrackImpl replacedIn) {
+		CompletableFuture done = new CompletableFuture<>();
+
+		peer.control.execute(() -> {
+			try {
+				if (!removed && sender != null && track == replacedIn && !peer.isClosed()) {
+					sender.replaceTrack(replacedIn.nativeTrack());
+				}
+			}
+			catch (Throwable e) {
+				peer.rtc.reportError(e);
+			}
+			finally {
+				done.complete(null);
+			}
+		});
+
+		return done;
+	}
+
 	/**
 	 * Lets go of the native sender while the peer closes, on its control
 	 * executor.
diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/client/CameraIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/client/CameraIntegrationTest.java
new file mode 100644
index 00000000..069b5729
--- /dev/null
+++ b/webrtc/src/test/java/dev/onvoid/webrtc/client/CameraIntegrationTest.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import dev.onvoid.webrtc.media.video.VideoDevice;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Captures real cameras, so it skips on machines without one, as CI runners
+ * are.
+ */
+class CameraIntegrationTest extends ClientTestBase {
+
+	@Test
+	void cameraReachesPeer() throws Exception {
+		List cameras = cameras(1);
+		Peer[] peers = pair();
+		LocalCamera camera = rtc.media().camera(CameraOptions.builder()
+				.device(cameras.get(0))
+				.resolution(640, 480)
+				.frameRate(30)
+				.build());
+		CompletableFuture remote = new CompletableFuture<>();
+		CompletableFuture preview = new CompletableFuture<>();
+
+		peers[1].onTrack(track -> track.asVideo().onFrame(frame -> remote.complete(frame.buffer.getWidth())));
+		camera.onFrame(frame -> preview.complete(frame.buffer.getWidth()));
+		peers[0].addTrack(camera);
+
+		assertEquals(cameras.get(0), camera.device());
+		assertEquals(640, (int) await(preview));
+		assertEquals(640, (int) await(remote));
+	}
+
+	@Test
+	void sharedCapture() throws Exception {
+		VideoDevice device = cameras(1).get(0);
+		CameraOptions options = CameraOptions.builder().device(device).resolution(640, 480).build();
+
+		LocalCamera first = rtc.media().camera(options);
+		LocalCamera second = rtc.media().camera(options);
+
+		// One device, one set of options at a time.
+		assertThrows(MediaException.class, () -> rtc.media().camera(CameraOptions.builder()
+				.device(device).resolution(320, 240).build()));
+
+		first.close();
+
+		// The capture stays open for the track still using it.
+		CompletableFuture frame = new CompletableFuture<>();
+		second.onFrame(video -> frame.complete(video.buffer.getWidth()));
+		assertEquals(640, (int) await(frame));
+	}
+
+	@Test
+	void switchesCamera() throws Exception {
+		List cameras = cameras(2);
+		Peer[] one = pair();
+		Peer[] two = pair();
+		LocalCamera camera = rtc.media().camera(CameraOptions.builder()
+				.device(cameras.get(0))
+				.resolution(1920, 1080)
+				.frameRate(30)
+				.build());
+		// Senders scale the frames to what their encoders want, but keep the
+		// aspect ratio, which tells the cameras apart.
+		AtomicReference previewAspect = new AtomicReference<>(0.0);
+		AtomicInteger atOne = new AtomicInteger();
+		AtomicInteger atTwo = new AtomicInteger();
+
+		camera.onFrame(frame -> previewAspect.set((double) frame.buffer.getWidth() / frame.buffer.getHeight()));
+		one[1].onTrack(track -> track.asVideo().onFrame(frame -> atOne.incrementAndGet()));
+		two[1].onTrack(track -> track.asVideo().onFrame(frame -> atTwo.incrementAndGet()));
+		Sender senderOne = one[0].addTrack(camera);
+		two[0].addTrack(camera);
+
+		waitFor(() -> atOne.get() > 5 && atTwo.get() > 5, "frames before the switch");
+		double aspectBefore = previewAspect.get();
+
+		await(camera.switchCamera(cameras.get(1)));
+
+		int oneBefore = atOne.get();
+		int twoBefore = atTwo.get();
+
+		// Both peers keep receiving, and the preview shows the other camera.
+		waitFor(() -> atOne.get() > oneBefore + 10 && atTwo.get() > twoBefore + 10, "frames after the switch");
+		waitFor(() -> Math.abs(previewAspect.get() - aspectBefore) > 0.1, "the other camera's frames in the preview");
+		assertEquals(cameras.get(1), camera.device());
+		assertEquals(camera, senderOne.track());
+
+		// Back to the first, which the switch let go of.
+		await(camera.switchCamera(cameras.get(0)));
+		waitFor(() -> Math.abs(previewAspect.get() - aspectBefore) < 0.05, "the first camera's frames again");
+
+		// A camera that cannot be opened fails the switch, and the track keeps its camera.
+		VideoDevice missing = new VideoDevice("Missing Camera", "/dev/video-missing") { };
+		ExecutionException failed = assertThrows(ExecutionException.class,
+				() -> camera.switchCamera(missing).get(TIMEOUT_SECONDS, TimeUnit.SECONDS));
+		assertInstanceOf(MediaException.class, failed.getCause());
+		assertEquals(cameras.get(0), camera.device());
+
+		int oneAfter = atOne.get();
+		waitFor(() -> atOne.get() > oneAfter + 10, "frames after the failed switch");
+	}
+
+	private List cameras(int needed) {
+		List cameras = rtc.media().devices().cameras();
+		assumeTrue(cameras.size() >= needed, "Needs " + needed + " camera(s), found " + cameras.size());
+		return cameras;
+	}
+
+	private static void waitFor(java.util.function.BooleanSupplier condition, String what) throws InterruptedException {
+		long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS);
+
+		while (!condition.getAsBoolean()) {
+			if (System.nanoTime() > deadline) {
+				throw new AssertionError("Timed out waiting for " + what);
+			}
+			Thread.sleep(50);
+		}
+	}
+
+}

From 2a3d4e2aecea4964f1853e669144c27d19a36b46 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Tue, 20 Oct 2026 00:35:07 +0200
Subject: [PATCH 23/25] Record client API senders and remote tracks

MediaRecorder.addTrack takes a Sender or a RemoteTrack. The client API hands
the media module their encoded frames through a package exported to it
alone, taken on the peer's control executor, where key frames are asked for
too; once the sender or receiver is gone those requests do nothing, so
closing the peer mid-recording just ends the track in the file.
---
 design/client-api.md                          |   9 +-
 docs/guide/client-api.md                      |  16 ++
 docs/guide/media/media-recording.md           |   9 +
 .../webrtc/media/recorder/MediaRecorder.java  |  72 +++++
 .../media/recorder/ClientApiRecorderTest.java | 252 ++++++++++++++++++
 .../webrtc/client/EncodedFramesAccess.java    |  46 ++++
 .../dev/onvoid/webrtc/client/PeerImpl.java    |   7 +
 .../onvoid/webrtc/client/RemoteTrackImpl.java |  37 +++
 .../dev/onvoid/webrtc/client/SenderImpl.java  |  33 +++
 .../webrtc/client/internal/EncodedFrames.java | 139 ++++++++++
 webrtc/src/main/java/module-info.java         |   4 +
 11 files changed, 620 insertions(+), 4 deletions(-)
 create mode 100644 webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
 create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/EncodedFramesAccess.java
 create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/client/internal/EncodedFrames.java

diff --git a/design/client-api.md b/design/client-api.md
index 152885b7..ae8d265c 100644
--- a/design/client-api.md
+++ b/design/client-api.md
@@ -804,10 +804,11 @@ Every test also fails if anything reached the runtime's error handler, including
 Next:
 - a guide page and an example;
 - `LocalMedia.videoTrack(label, source)` and `audioTrack(label, source)` wrap sources of the core
-  API, such as those of the media module's `MediaFileSource`, which stay the caller's. Recording a
-  peer with `MediaRecorder` needs its native senders and receivers, which the client API keeps to
-  itself; that bridge is still open.
-- phase 2 is done except recording: `Whip.to(...).publish(rtc)` and
+  API, such as those of the media module's `MediaFileSource`, which stay the caller's. `MediaRecorder`
+  takes client API senders and remote tracks: `dev.onvoid.webrtc.client.internal.EncodedFrames`,
+  exported to the media module only, hands it their encoded frames, taken on the peer's control
+  executor, where key frame requests run too, and do nothing once the sender or receiver is gone.
+- phase 2 is done: `Whip.to(...).publish(rtc)` and
   `Whep.from(...).play(rtc)` return sessions over `HttpURLConnection` (Java 8), and `VideoView` is in
   `dev.onvoid.webrtc.client.swing`;
 - porting the existing examples (phases 2 and 3).
diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md
index b87d4462..0155c69a 100644
--- a/docs/guide/client-api.md
+++ b/docs/guide/client-api.md
@@ -432,6 +432,22 @@ Peer viewer = rtc.createPeer(PeerOptions.builder()
         .build());
 ```
 
+### Recording
+
+The media module's `MediaRecorder` records senders and remote tracks into a file as they are, without re-encoding:
+
+```java
+try (MediaRecorder recorder = new MediaRecorder(Paths.get("call.mkv"))) {
+    recorder.addTrack(videoSender);
+    recorder.addTrack(remoteVideo);
+    recorder.start();
+    // ...
+    recorder.stop();
+}
+```
+
+Closing the peer, or removing a sender, during a recording ends that track in the file, and `stop()` still finishes it. The [Media Recording](/guide/media/media-recording) guide covers containers and the recorder's events.
+
 ## Data Channels
 
 ```java
diff --git a/docs/guide/media/media-recording.md b/docs/guide/media/media-recording.md
index fbb543d7..aa69c0cc 100644
--- a/docs/guide/media/media-recording.md
+++ b/docs/guide/media/media-recording.md
@@ -36,6 +36,15 @@ A sender's frames are recorded as they leave the encoder, a receiver's as they g
 Keep the senders and receivers undisposed while recording. The recorder asks them for key frames, and a disposed one cannot be asked.
 :::
 
+With the [Client API](/guide/client-api), a recorder takes its senders and remote tracks directly:
+
+```java
+recorder.addTrack(videoSender);          // a Sender from peer.addTrack
+recorder.addTrack(remoteTrack);          // a RemoteTrack from peer.onTrack
+```
+
+The peer keeps its native senders and receivers to itself and asks them for key frames on its own thread, so nothing needs to be kept undisposed: removing a sender or closing the peer ends the track in the recording, and `stop()` still finishes the file.
+
 ## Containers
 
 The container follows the file name:
diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java
index 803d9a2f..eee94cb5 100644
--- a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java
+++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/recorder/MediaRecorder.java
@@ -21,13 +21,20 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import dev.onvoid.webrtc.RTCRtpReceiver;
 import dev.onvoid.webrtc.RTCRtpSender;
+import dev.onvoid.webrtc.client.RemoteTrack;
+import dev.onvoid.webrtc.client.Sender;
+import dev.onvoid.webrtc.client.internal.EncodedFrames;
 import dev.onvoid.webrtc.internal.NativeApi;
 import dev.onvoid.webrtc.media.player.FFmpeg;
 
@@ -165,6 +172,46 @@ public void addTrack(RTCRtpReceiver receiver) {
 		}
 	}
 
+	/**
+	 * Records what a sender of the client API sends, as it comes out of the
+	 * encoder. Key frames are asked for on the sender's peer, which keeps
+	 * the sender to itself; removing the sender or closing the peer ends the
+	 * track in the recording.
+	 *
+	 * @param sender The sender to record.
+	 *
+	 * @throws IllegalStateException    If the recorder was started or closed.
+	 * @throws IllegalArgumentException If the sender was removed, or its peer
+	 *                                  closed.
+	 */
+	public void addTrack(Sender sender) {
+		Objects.requireNonNull(sender, "Sender is null");
+
+		synchronized (lock) {
+			checkState(MediaRecorderState.IDLE);
+			addTrack(sourceOf(EncodedFrames.of(sender)));
+		}
+	}
+
+	/**
+	 * Records what a remote track of the client API receives, as it goes into
+	 * the decoder. Key frames are asked for on the track's peer; the track
+	 * ending, or the peer closing, ends the track in the recording.
+	 *
+	 * @param track The remote track to record.
+	 *
+	 * @throws IllegalStateException    If the recorder was started or closed.
+	 * @throws IllegalArgumentException If the track ended.
+	 */
+	public void addTrack(RemoteTrack track) {
+		Objects.requireNonNull(track, "Track is null");
+
+		synchronized (lock) {
+			checkState(MediaRecorderState.IDLE);
+			addTrack(sourceOf(EncodedFrames.of(track)));
+		}
+	}
+
 	/**
 	 * Starts recording the tracks added so far.
 	 *
@@ -268,6 +315,31 @@ public void close() {
 	}
 
 	/** Called with the lock held, in the idle state. */
+	private void addTrack(EncodedFrames.Source source) {
+		addTrack(source.frames(), source::requestKeyFrame);
+	}
+
+	/**
+	 * Waits for the client API's peer to hand over the encoded frames. Its
+	 * control thread never takes this recorder's lock, so waiting under it
+	 * cannot deadlock.
+	 */
+	private static EncodedFrames.Source sourceOf(CompletableFuture future) {
+		try {
+			return future.get(10, TimeUnit.SECONDS);
+		}
+		catch (InterruptedException e) {
+			Thread.currentThread().interrupt();
+			throw new IllegalStateException("Interrupted while adding the track", e);
+		}
+		catch (ExecutionException e) {
+			throw new IllegalArgumentException(e.getCause().getMessage(), e.getCause());
+		}
+		catch (TimeoutException e) {
+			throw new IllegalStateException("The peer did not hand over the track in time", e);
+		}
+	}
+
 	private void addTrack(long frames, Runnable keyFrameRequest) {
 		if (frames == 0) {
 			throw new IllegalArgumentException("The sender or receiver was disposed");
diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
new file mode 100644
index 00000000..8cd3ac07
--- /dev/null
+++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
@@ -0,0 +1,252 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.media.recorder;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import dev.onvoid.webrtc.client.AudioMode;
+import dev.onvoid.webrtc.client.CustomAudio;
+import dev.onvoid.webrtc.client.CustomVideo;
+import dev.onvoid.webrtc.client.PcmFormat;
+import dev.onvoid.webrtc.client.Peer;
+import dev.onvoid.webrtc.client.PeerOptions;
+import dev.onvoid.webrtc.client.RemoteTrack;
+import dev.onvoid.webrtc.client.Sender;
+import dev.onvoid.webrtc.client.Signaling;
+import dev.onvoid.webrtc.client.SignalingPair;
+import dev.onvoid.webrtc.client.WebRtc;
+import dev.onvoid.webrtc.media.player.MediaInfo;
+import dev.onvoid.webrtc.media.player.MediaReader;
+import dev.onvoid.webrtc.media.video.NativeI420Buffer;
+import dev.onvoid.webrtc.media.video.VideoFrame;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+
+/**
+ * Records a call of the client API, from its senders and from its remote
+ * tracks.
+ *
+ * @author Alex Andres
+ */
+@Execution(ExecutionMode.SAME_THREAD)
+class ClientApiRecorderTest {
+
+	private static final int WIDTH = 320;
+
+	private static final int HEIGHT = 240;
+
+	private static final long RECORD_MS = 2000;
+
+	@TempDir
+	Path tempDir;
+
+	private final List errors = new CopyOnWriteArrayList<>();
+
+	private WebRtc rtc;
+
+	private ScheduledExecutorService feeder;
+
+	private Peer caller;
+
+	private Sender videoSender;
+
+	private Sender audioSender;
+
+	private final CompletableFuture remoteVideo = new CompletableFuture<>();
+
+	private final CompletableFuture remoteAudio = new CompletableFuture<>();
+
+
+	@BeforeEach
+	void startCall() throws Exception {
+		rtc = WebRtc.builder().audio(AudioMode.HEADLESS).errorHandler(errors::add).build();
+		feeder = Executors.newSingleThreadScheduledExecutor();
+
+		SignalingPair link = Signaling.loopback();
+		caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build());
+		Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build());
+
+		callee.onTrack(track -> (track.isVideo() ? remoteVideo : remoteAudio).complete(track));
+
+		CustomVideo video = rtc.media().customVideo("video");
+		CustomAudio audio = rtc.media().customAudio("audio");
+		videoSender = caller.addTrack(video);
+		audioSender = caller.addTrack(audio);
+
+		feeder.scheduleAtFixedRate(() -> {
+			NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT);
+			ByteBuffer y = buffer.getDataY();
+			for (int i = 0; i < y.capacity(); i++) {
+				y.put(i, (byte) (i + System.nanoTime() / 10_000_000));
+			}
+			VideoFrame frame = new VideoFrame(buffer, System.nanoTime());
+			video.push(frame);
+			frame.release();
+		}, 0, 33, TimeUnit.MILLISECONDS);
+		feeder.scheduleAtFixedRate(() -> audio.push(new byte[48000 / 10 * 2], PcmFormat.of(48000, 1)),
+				0, 100, TimeUnit.MILLISECONDS);
+
+		caller.whenConnected().get(15, TimeUnit.SECONDS);
+		remoteVideo.get(15, TimeUnit.SECONDS);
+		remoteAudio.get(15, TimeUnit.SECONDS);
+
+		// Well into the call, so that the recorder has to ask for a key frame.
+		Thread.sleep(1500);
+	}
+
+	@AfterEach
+	void endCall() throws Exception {
+		feeder.shutdownNow();
+		feeder.awaitTermination(5, TimeUnit.SECONDS);
+		rtc.close();
+
+		assertTrue(errors.isEmpty(), () -> "Errors: " + errors);
+	}
+
+	@Test
+	void recordsRemoteTracks() throws Exception {
+		Path file = tempDir.resolve("received.mkv");
+
+		record(file, recorder -> {
+			recorder.addTrack(remoteVideo.join());
+			recorder.addTrack(remoteAudio.join());
+		});
+
+		MediaInfo info = readInfo(file);
+
+		assertEquals("vp8", info.getVideoCodec());
+		assertEquals(WIDTH, info.getVideoWidth());
+		assertEquals(HEIGHT, info.getVideoHeight());
+		assertEquals("opus", info.getAudioCodec());
+	}
+
+	@Test
+	void recordsSenders() throws Exception {
+		Path file = tempDir.resolve("sent.webm");
+
+		record(file, recorder -> {
+			recorder.addTrack(videoSender);
+			recorder.addTrack(audioSender);
+		});
+
+		MediaInfo info = readInfo(file);
+
+		assertEquals("vp8", info.getVideoCodec());
+		assertEquals("opus", info.getAudioCodec());
+	}
+
+	@Test
+	void peerClosedWhileRecording() throws Exception {
+		Path file = tempDir.resolve("cut.mkv");
+		CountDownLatch started = new CountDownLatch(1);
+
+		try (MediaRecorder recorder = new MediaRecorder(file)) {
+			recorder.setListener(new MediaRecorderListener() {
+
+				@Override
+				public void onStarted() {
+					started.countDown();
+				}
+			});
+
+			recorder.addTrack(remoteVideo.join());
+			recorder.addTrack(videoSender);
+			recorder.start();
+
+			assertTrue(started.await(10, TimeUnit.SECONDS), "not started");
+			Thread.sleep(500);
+
+			// The tracks end in the recording; the file still finishes.
+			caller.closeAsync().get(10, TimeUnit.SECONDS);
+			Thread.sleep(500);
+
+			assertTrue(recorder.stop());
+		}
+
+		assertEquals("vp8", readInfo(file).getVideoCodec());
+	}
+
+	@Test
+	void removedSenderRefused() throws Exception {
+		videoSender.remove().get(10, TimeUnit.SECONDS);
+
+		try (MediaRecorder recorder = new MediaRecorder(tempDir.resolve("refused.mkv"))) {
+			assertThrows(IllegalArgumentException.class, () -> recorder.addTrack(videoSender));
+		}
+	}
+
+	private interface Tracks {
+
+		void addTo(MediaRecorder recorder);
+
+	}
+
+	private static void record(Path file, Tracks tracks) throws Exception {
+		CountDownLatch started = new CountDownLatch(1);
+		AtomicReference error = new AtomicReference<>();
+
+		try (MediaRecorder recorder = new MediaRecorder(file)) {
+			recorder.setListener(new MediaRecorderListener() {
+
+				@Override
+				public void onStarted() {
+					started.countDown();
+				}
+
+				@Override
+				public void onError(String message) {
+					error.set(message);
+				}
+			});
+
+			tracks.addTo(recorder);
+			recorder.start();
+
+			assertTrue(started.await(10, TimeUnit.SECONDS), "not started");
+			Thread.sleep(RECORD_MS);
+			assertTrue(recorder.stop());
+		}
+
+		assertNull(error.get());
+	}
+
+	private static MediaInfo readInfo(Path file) throws Exception {
+		try (MediaReader reader = new MediaReader(file)) {
+			return reader.getInfo();
+		}
+	}
+
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/EncodedFramesAccess.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/EncodedFramesAccess.java
new file mode 100644
index 00000000..d0641efc
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/EncodedFramesAccess.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client;
+
+import java.util.concurrent.CompletableFuture;
+
+import dev.onvoid.webrtc.client.internal.EncodedFrames;
+
+/**
+ * The client API's side of {@link EncodedFrames}, registered when the first
+ * peer class loads.
+ */
+final class EncodedFramesAccess implements EncodedFrames.Access {
+
+	@Override
+	public CompletableFuture of(Sender sender) {
+		if (!(sender instanceof SenderImpl)) {
+			throw new IllegalArgumentException("Not a sender of the client API: " + sender);
+		}
+
+		return ((SenderImpl) sender).encodedFrames();
+	}
+
+	@Override
+	public CompletableFuture of(RemoteTrack track) {
+		if (!(track instanceof RemoteTrackImpl)) {
+			throw new IllegalArgumentException("Not a remote track of the client API: " + track);
+		}
+
+		return ((RemoteTrackImpl) track).encodedFrames();
+	}
+}
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java
index 1afcb458..947b343c 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/PeerImpl.java
@@ -57,6 +57,7 @@
 import dev.onvoid.webrtc.RTCSessionDescription;
 import dev.onvoid.webrtc.RTCSignalingState;
 import dev.onvoid.webrtc.SetSessionDescriptionObserver;
+import dev.onvoid.webrtc.client.internal.EncodedFrames;
 import dev.onvoid.webrtc.media.MediaStream;
 import dev.onvoid.webrtc.media.MediaStreamTrack;
 import dev.onvoid.webrtc.media.MediaType;
@@ -78,6 +79,12 @@
  */
 final class PeerImpl implements Peer {
 
+	static {
+		// Senders and remote tracks come from peers, so whoever holds one
+		// finds the access to their encoded frames registered.
+		EncodedFrames.register(new EncodedFramesAccess());
+	}
+
 	/** How long an impolite peer waits for an answer after it ignored an offer. */
 	private static final long COLLISION_TIMEOUT_MS = 5000;
 
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java
index c072da80..09669e34 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/RemoteTrackImpl.java
@@ -20,9 +20,12 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.function.Consumer;
 
 import dev.onvoid.webrtc.RTCRtpReceiver;
+import dev.onvoid.webrtc.client.internal.EncodedFrames;
+import dev.onvoid.webrtc.internal.NativeApi;
 import dev.onvoid.webrtc.media.MediaStreamTrack;
 import dev.onvoid.webrtc.media.MediaStreamTrackMuteListener;
 
@@ -148,6 +151,40 @@ void onControl(Runnable change) {
 		});
 	}
 
+	/**
+	 * Hands a native extension the encoded frames of this track's receiver,
+	 * taken on the peer's control executor, where key frames are asked for
+	 * too.
+	 */
+	CompletableFuture encodedFrames() {
+		CompletableFuture result = new CompletableFuture<>();
+
+		peer.control.execute(() -> {
+			try {
+				if (ended) {
+					throw new IllegalStateException("The track " + id + " ended");
+				}
+
+				long frames = NativeApi.encodedFramesOf(receiver);
+
+				if (frames == 0) {
+					throw new IllegalStateException("The track " + id + " ended");
+				}
+
+				result.complete(new EncodedFrames.Source(frames, () -> onControl(() -> {
+					if (kind == MediaKind.VIDEO) {
+						receiver.requestKeyFrame();
+					}
+				})));
+			}
+			catch (Throwable e) {
+				result.completeExceptionally(e);
+			}
+		});
+
+		return result;
+	}
+
 	/** Detaches everything from the native track, on the control executor. */
 	void end() {
 		if (ended) {
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
index 844581a1..b634389c 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/SenderImpl.java
@@ -29,6 +29,8 @@
 import dev.onvoid.webrtc.RTCRtpEncodingParameters;
 import dev.onvoid.webrtc.RTCRtpSendParameters;
 import dev.onvoid.webrtc.RTCRtpSender;
+import dev.onvoid.webrtc.client.internal.EncodedFrames;
+import dev.onvoid.webrtc.internal.NativeApi;
 
 /**
  * The {@link Sender} of a {@link PeerImpl}. The native sender is used on the
@@ -307,6 +309,37 @@ CompletableFuture trackReplaced(LocalTrackImpl replacedIn) {
 		return done;
 	}
 
+	/**
+	 * Hands a native extension the encoded frames of this sender, taken on the
+	 * peer's control executor, where key frames are asked for too.
+	 */
+	CompletableFuture encodedFrames() {
+		CompletableFuture result = new CompletableFuture<>();
+
+		peer.control.execute(() -> {
+			try {
+				checkUsable();
+
+				long frames = NativeApi.encodedFramesOf(sender);
+
+				if (frames == 0) {
+					throw new IllegalStateException("The sender was removed");
+				}
+
+				result.complete(new EncodedFrames.Source(frames, () -> peer.control.execute(() -> {
+					if (kind == MediaKind.VIDEO && !removed && sender != null && !peer.isClosed()) {
+						sender.generateKeyFrame();
+					}
+				})));
+			}
+			catch (Throwable e) {
+				result.completeExceptionally(e);
+			}
+		});
+
+		return result;
+	}
+
 	/**
 	 * Lets go of the native sender while the peer closes, on its control
 	 * executor.
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/internal/EncodedFrames.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/internal/EncodedFrames.java
new file mode 100644
index 00000000..e7c8266c
--- /dev/null
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/internal/EncodedFrames.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.onvoid.webrtc.client.internal;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.concurrent.CompletableFuture;
+
+import dev.onvoid.webrtc.client.RemoteTrack;
+import dev.onvoid.webrtc.client.Sender;
+
+/**
+ * Lets a native extension, such as the media module's recorder, observe the
+ * encoded frames of a client API sender or remote track. Not API for
+ * applications: the package is exported to the media module only.
+ * 

+ * The client API keeps its native senders and receivers to itself, and uses
+ * them on the control executor of their peer. What this hands out keeps to
+ * that: the handle is taken there, and key frames are asked for there, once
+ * the sender or receiver is gone not at all.
+ */
+public final class EncodedFrames {
+
+	/** The client API's side, registered by its peers. */
+	public interface Access {
+
+		/**
+		 * @param sender a sender of the client API.
+		 *
+		 * @return a future with the source of the sender's encoded frames.
+		 */
+		CompletableFuture of(Sender sender);
+
+		/**
+		 * @param track a remote track of the client API.
+		 *
+		 * @return a future with the source of the track's encoded frames.
+		 */
+		CompletableFuture of(RemoteTrack track);
+
+	}
+
+	/**
+	 * Where a sender's or receiver's encoded frames come from.
+	 */
+	public static final class Source {
+
+		private final long frames;
+
+		private final Runnable keyFrameRequest;
+
+
+		/**
+		 * @param frames          the handle of {@code NativeApi.encodedFramesOf},
+		 *                        whose reference the receiver of this source
+		 *                        drops.
+		 * @param keyFrameRequest asks for a key frame, from any thread.
+		 */
+		public Source(long frames, Runnable keyFrameRequest) {
+			this.frames = frames;
+			this.keyFrameRequest = requireNonNull(keyFrameRequest, "keyFrameRequest");
+		}
+
+		/**
+		 * @return the handle of the encoded frames, with a reference to drop.
+		 */
+		public long frames() {
+			return frames;
+		}
+
+		/**
+		 * Asks for a key frame. Does nothing once the sender or receiver is
+		 * gone.
+		 */
+		public void requestKeyFrame() {
+			keyFrameRequest.run();
+		}
+	}
+
+	private static volatile Access access;
+
+
+	private EncodedFrames() {
+	}
+
+	/**
+	 * Registers the client API's side. The first registration stays.
+	 *
+	 * @param side the client API's side.
+	 */
+	public static synchronized void register(Access side) {
+		if (access == null) {
+			access = requireNonNull(side, "side");
+		}
+	}
+
+	/**
+	 * @param sender a sender of the client API.
+	 *
+	 * @return a future with the source of the sender's encoded frames.
+	 */
+	public static CompletableFuture of(Sender sender) {
+		return access().of(requireNonNull(sender, "sender"));
+	}
+
+	/**
+	 * @param track a remote track of the client API.
+	 *
+	 * @return a future with the source of the track's encoded frames.
+	 */
+	public static CompletableFuture of(RemoteTrack track) {
+		return access().of(requireNonNull(track, "track"));
+	}
+
+	private static Access access() {
+		Access side = access;
+
+		if (side == null) {
+			// A sender or track exists, so its peer's class registered.
+			throw new IllegalStateException("No peer of the client API was created");
+		}
+
+		return side;
+	}
+}
diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java
index ec5e78d8..2b71a71e 100644
--- a/webrtc/src/main/java/module-info.java
+++ b/webrtc/src/main/java/module-info.java
@@ -21,4 +21,8 @@
 	// media source directly instead of carrying every frame through Java.
 	exports dev.onvoid.webrtc.internal to webrtc.java.media;
 
+	// Not API for applications either. The media module's recorder needs the
+	// encoded frames of client API senders and remote tracks.
+	exports dev.onvoid.webrtc.client.internal to webrtc.java.media;
+
 }
\ No newline at end of file

From a9b437c02fbc1bd86c3b7418e4ee818c520a64ef Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Tue, 20 Oct 2026 00:36:50 +0200
Subject: [PATCH 24/25] List devices on machines without an audio system

Without PulseAudio, reading the audio devices and watching for changes threw,
so devices() failed, and with it camera(). A list the system cannot read is
empty now, and without change reports the lists are read on each call.
---
 .../dev/onvoid/webrtc/client/Devices.java     |  5 ++-
 .../dev/onvoid/webrtc/client/DevicesImpl.java | 33 +++++++++++++++----
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java
index b215666e..36806f88 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/Devices.java
@@ -26,7 +26,10 @@
 
 /**
  * The media devices of the system. The lists are snapshots, kept current in
- * the background, so reading them is cheap on any thread.
+ * the background, so reading them is cheap on any thread. Where the system
+ * reports no device changes, as a Linux machine without PulseAudio, they are
+ * read on each call. A kind of device the system cannot list, such as audio
+ * devices without an audio system, comes back as an empty list.
  */
 public interface Devices {
 
diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java
index 308e32d0..3059c796 100644
--- a/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java
+++ b/webrtc/src/main/java/dev/onvoid/webrtc/client/DevicesImpl.java
@@ -38,7 +38,10 @@
 
 /**
  * The {@link Devices} of a runtime. The device lists are read once, then again
- * on the media executor whenever the system reports a change.
+ * on the media executor whenever the system reports a change. Where the system
+ * reports no changes, as a Linux machine without PulseAudio, the lists are
+ * read on each call instead. A list that cannot be read is empty: a machine
+ * without an audio system has no microphones to offer.
  */
 final class DevicesImpl implements Devices {
 
@@ -50,6 +53,9 @@ final class DevicesImpl implements Devices {
 
 	private final DeviceChangeListener listener;
 
+	/** True while the system reports changes, which keep the snapshots current. */
+	private final boolean watching;
+
 	private volatile List cameras = Collections.emptyList();
 
 	private volatile List microphones = Collections.emptyList();
@@ -77,22 +83,22 @@ public void deviceDisconnected(Device device) {
 			}
 		};
 
-		MediaDevices.addDeviceChangeListener(listener);
+		this.watching = watch(listener);
 	}
 
 	@Override
 	public List cameras() {
-		return cameras;
+		return watching ? cameras : read(MediaDevices::getVideoCaptureDevices);
 	}
 
 	@Override
 	public List microphones() {
-		return microphones;
+		return watching ? microphones : read(MediaDevices::getAudioCaptureDevices);
 	}
 
 	@Override
 	public List speakers() {
-		return speakers;
+		return watching ? speakers : read(MediaDevices::getAudioRenderDevices);
 	}
 
 	@Override
@@ -158,7 +164,9 @@ public Subscription onChange(Consumer handler) {
 	}
 
 	void dispose() {
-		MediaDevices.removeDeviceChangeListener(listener);
+		if (watching) {
+			MediaDevices.removeDeviceChangeListener(listener);
+		}
 		handlers.clear();
 	}
 
@@ -185,11 +193,22 @@ private  List read(Supplier> query) {
 					: Collections.unmodifiableList(new ArrayList<>(devices));
 		}
 		catch (Throwable e) {
-			media.rtc.reportError(new MediaException("Reading the devices failed", e));
+			// No device of the kind, such as without an audio system.
 			return Collections.emptyList();
 		}
 	}
 
+	private static boolean watch(DeviceChangeListener listener) {
+		try {
+			MediaDevices.addDeviceChangeListener(listener);
+			return true;
+		}
+		catch (Throwable e) {
+			// The system reports no changes, as Linux without PulseAudio.
+			return false;
+		}
+	}
+
 	private static List desktopSources(DesktopCapturer capturer) {
 		try {
 			List sources = capturer.getDesktopSources();

From 1da3b10cc8d614f4ce609c700da7c4001d046245 Mon Sep 17 00:00:00 2001
From: Alex Andres 
Date: Tue, 20 Oct 2026 00:56:07 +0200
Subject: [PATCH 25/25] Accept the negotiated video codec in the client
 recorder test

macOS negotiates H.264 first, which the test did not expect, and which WebM
cannot hold. The test records into Matroska and checks for video of the
sent size, whatever its codec.
---
 .../webrtc/media/recorder/ClientApiRecorderTest.java   | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
index 8cd3ac07..6061fafd 100644
--- a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
+++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/recorder/ClientApiRecorderTest.java
@@ -147,7 +147,7 @@ void recordsRemoteTracks() throws Exception {
 
 		MediaInfo info = readInfo(file);
 
-		assertEquals("vp8", info.getVideoCodec());
+		assertTrue(info.hasVideo());
 		assertEquals(WIDTH, info.getVideoWidth());
 		assertEquals(HEIGHT, info.getVideoHeight());
 		assertEquals("opus", info.getAudioCodec());
@@ -155,7 +155,9 @@ void recordsRemoteTracks() throws Exception {
 
 	@Test
 	void recordsSenders() throws Exception {
-		Path file = tempDir.resolve("sent.webm");
+		// Matroska, since the codec follows the machine: VP8 here, H.264 where
+		// a hardware encoder comes first, as on macOS.
+		Path file = tempDir.resolve("sent.mkv");
 
 		record(file, recorder -> {
 			recorder.addTrack(videoSender);
@@ -164,7 +166,7 @@ void recordsSenders() throws Exception {
 
 		MediaInfo info = readInfo(file);
 
-		assertEquals("vp8", info.getVideoCodec());
+		assertTrue(info.hasVideo());
 		assertEquals("opus", info.getAudioCodec());
 	}
 
@@ -196,7 +198,7 @@ public void onStarted() {
 			assertTrue(recorder.stop());
 		}
 
-		assertEquals("vp8", readInfo(file).getVideoCodec());
+		assertTrue(readInfo(file).hasVideo());
 	}
 
 	@Test