From 91087635429a1268db331278474ba0fb5dc0af64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 14:23:12 -0700 Subject: [PATCH 01/11] Add native macOS screen sharing MVP checkpoint --- .agents/skills/run-dev/SKILL.md | 2 + .github/workflows/webrtc-artifact.yml | 42 + .gitmodules | 3 + .repos/WebRTC | 1 + .../xcshareddata/swiftpm/Package.resolved | 11 +- .../Features/Home/HomeSidebarSection.swift | 1 + .../Features/Home/HomeView+Sidebar.swift | 4 +- .../Home/Views/HomeSidebarTabIcon.swift | 3 + .../Views/WorkspacePaneContentView.swift | 4 + .../WorkspaceScreen+PaneMutations.swift | 2 +- .../Features/Workspace/WorkspaceScreen.swift | 2 +- .../xcshareddata/swiftpm/Package.resolved | 11 +- apps/macos/Codevisor/CodevisorApp.swift | 45 +- .../Features/Panes/MarkdownDocumentPane.swift | 17 +- .../Features/Panes/NewTabPageView.swift | 15 +- .../macos/Codevisor/Features/Panes/Pane.swift | 16 +- .../Panes/PaneGroupModel+Operations.swift | 7 +- .../Features/Panes/PaneGroupModel.swift | 66 +- .../Codevisor/Features/Panes/PluginPane.swift | 5 +- .../Features/Panes/ScreenSharingPane.swift | 119 ++ .../Features/Panes/ScreenSharingToolbar.swift | 132 ++ .../Features/Panes/TerminalPane.swift | 3 +- .../Panes/WorkspaceSplitDragCoordinator.swift | 1 + .../Features/Panes/WorkspaceSplitView.swift | 3 +- .../SessionContainerView+Documents.swift | 4 +- .../Session/SessionContainerView+Panes.swift | 30 +- .../SessionContainerView+ReopenPane.swift | 18 +- .../SessionContainerView+SplitCommands.swift | 16 +- .../SessionContainerView+Toolbar.swift | 10 +- .../SessionContainerView+WorkspaceTabs.swift | 12 +- .../Session/SessionContainerView.swift | 77 +- .../Features/Session/SessionScreen.swift | 13 +- .../Session/SessionStore+PaneGroups.swift | 217 +++- .../Session/Views/ChatPaneContentView.swift | 19 +- .../Features/Sidebar/SidebarView+Tabs.swift | 38 +- .../Sidebar/SidebarView+Workspaces.swift | 16 +- .../Features/Sidebar/SidebarView.swift | 2 +- .../Views/SidebarWorkspaceTabRow.swift | 1 + .../Features/Terminal/TerminalSurface.swift | 16 +- apps/server/src/routes/screen-sharing.ts | 85 ++ apps/server/src/screen-sharing.test.ts | 232 ++++ apps/server/src/serve-boot.ts | 7 +- apps/server/src/serve.ts | 6 +- apps/server/src/server-context-types.ts | 2 + apps/server/src/server-router.ts | 4 + apps/server/src/server.ts | 1 + docs/plans/native-screen-sharing-mvp.md | 32 + docs/plans/native-screen-sharing.md | 168 +++ package.json | 1 + packages/api/src/index.ts | 2 + packages/api/src/screen-sharing.ts | 41 + .../src/computer-use-bridge.test.ts | 71 +- .../automation/src/computer-use-provider.ts | 23 +- .../CodevisorClient/CodevisorMachine.swift | 14 + .../CodevisorServerClienting.swift | 1 + .../ServerScreenSharingRequest.swift | 78 ++ .../UnresolvedMachineTests.swift | 30 + .../CodevisorCore/Panes/PaneGroupState.swift | 30 +- .../Panes/PaneWorkingDirectory.swift | 29 + .../Panes/ScreenSharingPanePreferences.swift | 14 + .../Panes/Workspace+Navigation.swift | 43 + .../Persistence/PaneGroupRepository.swift | 20 +- .../Persistence/WorkspaceRepository.swift | 7 +- .../WorkspaceSyncModel+Layout.swift | 2 +- .../WorkspaceSyncModel+ServerRecords.swift | 15 + .../ChatlessWorkspacePaneTests.swift | 193 +++ .../ConvertedPaneTitleTests.swift | 71 ++ .../ScreenSharingPaneMappingTests.swift | 44 + .../WorkspaceSelectionRouteTests.swift | 96 ++ .../NativeScreenSharingBridge.swift | 79 ++ .../ScreenSharingHostConnectivity.swift | 50 + .../ScreenSharingHostControl.swift | 118 ++ .../ScreenSharingHostIndicator.swift | 29 + .../ScreenSharingHostLease.swift | 44 + .../ScreenSharingHostService.swift | 401 ++++++ .../ScreenSharingInputInjector.swift | 87 ++ .../ScreenSharingInputSurface.swift | 137 +++ .../ScreenSharingPasteboard.swift | 33 + .../ScreenSharingVideoSurface.swift | 247 ++++ .../ScreenSharingViewerClipboard.swift | 56 + .../ScreenSharingViewerControl.swift | 79 ++ .../ScreenSharingViewerDiagnostics.swift | 44 + .../ScreenSharingViewerModel.swift | 267 ++++ .../Server/ComputerUseBridge+Dispatch.swift | 10 + .../Server/ComputerUseBridge.swift | 3 + .../ScreenSharingConnectivityTests.swift | 44 + .../ScreenSharingControlTests.swift | 184 +++ .../ScreenSharingDiagnosticsTests.swift | 52 + .../ScreenSharingHostLeaseTests.swift | 63 + .../ScreenSharingHostRequestTests.swift | 157 +++ .../ScreenSharingPasteboardTests.swift | 30 + .../ScreenSharingViewerTests.swift | 273 +++++ .../Probe/ProbeCapabilities.swift | 37 + .../Probe/ProbeCapturePicker.swift | 79 ++ .../Probe/ProbeClipboardCheck.swift | 71 ++ .../Probe/ProbeClockSync.swift | 31 + .../Probe/ProbeCodecCheck.swift | 324 +++++ .../Probe/ProbeCodecPattern.swift | 102 ++ .../Probe/ProbeControlCheck.swift | 88 ++ .../Probe/ProbeDesktopPainter.swift | 69 ++ .../Probe/ProbeDesktopWorkload.swift | 94 ++ .../Probe/ProbeHostRecovery.swift | 125 ++ .../Probe/ProbeMetalDisplayLink.swift | 57 + .../Probe/ProbeOptions.swift | 800 ++++++++++++ .../Probe/ProbeOwnedWorkloadWindow.swift | 279 +++++ .../Probe/ProbeTimeline.swift | 57 + .../Probe/ProbeTransportTimeline.swift | 23 + .../Probe/ProbeWindowObservation.swift | 271 +++++ .../Probe/ProbeWorkloadView.swift | 123 ++ .../Probe/ScreenSharingProbe.swift | 1083 +++++++++++++++++ .../Probe/SyntheticSource.swift | 181 +++ .../swift/CodevisorScreenSharing/README.md | 294 +++++ .../NALUnitBitstream.swift | 73 ++ .../Resources/WebRTC-LICENSE.txt | 29 + .../ScreenSharingAdaptiveQuality.swift | 41 + .../ScreenSharingCapture.swift | 327 +++++ .../ScreenSharingCaptureBufferCopy.swift | 50 + .../ScreenSharingCaptureIntervalRequest.swift | 31 + .../ScreenSharingCaptureRequestState.swift | 34 + .../ScreenSharingClipboardMessage.swift | 41 + .../ScreenSharingClipboardTransfer.swift | 126 ++ .../ScreenSharingCodecFactory.swift | 296 +++++ .../ScreenSharingControlMessage.swift | 78 ++ .../ScreenSharingDataChannel.swift | 117 ++ .../ScreenSharingDecoder.swift | 260 ++++ .../ScreenSharingDecoderRecoveryCheck.swift | 49 + .../ScreenSharingDiagnosticProfile.swift | 76 ++ .../ScreenSharingEncoder.swift | 376 ++++++ .../ScreenSharingEncoderDropCheck.swift | 17 + .../ScreenSharingEncoderDropLog.swift | 45 + .../ScreenSharingFieldTrials.swift | 162 +++ .../ScreenSharingFrameDeliveryAudit.swift | 354 ++++++ .../ScreenSharingFrameIdentity.swift | 34 + .../ScreenSharingFrameSender.swift | 108 ++ .../ScreenSharingICEConfiguration.swift | 46 + .../ScreenSharingKeyframeRequest.swift | 34 + .../ScreenSharingMetalView.swift | 478 ++++++++ .../ScreenSharingMetrics+Presentation.swift | 23 + .../ScreenSharingMetrics.swift | 100 ++ .../ScreenSharingOwnedWindowCapture.swift | 222 ++++ .../ScreenSharingOwnedWindowObservation.swift | 160 +++ .../ScreenSharingOwnedWindowSession.swift | 192 +++ .../ScreenSharingPeer.swift | 586 +++++++++ .../ScreenSharingRTCIntervalMetrics.swift | 41 + .../ScreenSharingRenderCoordinator.swift | 363 ++++++ .../ScreenSharingRtcEventLogDiagnostic.swift | 333 +++++ .../ScreenSharingScrollAccumulator.swift | 16 + .../ScreenSharingSourceIdle.swift | 428 +++++++ .../ScreenSharingSourceMarker.swift | 72 ++ .../ScreenSharingVideoCodec.swift | 62 + .../ScreenSharingVideoConfiguration.swift | 128 ++ .../ScreenSharingVideoRefresh.swift | 227 ++++ .../NALUnitBitstreamTests.swift | 37 + .../ScreenSharingAdaptiveQualityTests.swift | 38 + .../ScreenSharingBufferTests.swift | 80 ++ .../ScreenSharingCaptureBufferCopyTests.swift | 51 + ...enSharingCaptureIntervalRequestTests.swift | 48 + ...SharingCaptureUpdateTransactionTests.swift | 119 ++ .../ScreenSharingClipboardTests.swift | 144 +++ .../ScreenSharingControlMessageTests.swift | 81 ++ ...reenSharingDecoderRecoveryCheckTests.swift | 49 + .../ScreenSharingDiagnosticProfileTests.swift | 400 ++++++ ...creenSharingDrawTimestampRecordTests.swift | 40 + .../ScreenSharingEncoderDropLogTests.swift | 40 + ...ScreenSharingFrameDeliveryAuditTests.swift | 225 ++++ .../ScreenSharingHEVCFormatTests.swift | 65 + .../ScreenSharingICETests.swift | 32 + .../ScreenSharingKeyframeRequestTests.swift | 93 ++ ...ScreenSharingOwnedWindowCaptureTests.swift | 150 +++ ...enSharingOwnedWindowObservationTests.swift | 101 ++ ...ScreenSharingOwnedWindowSessionTests.swift | 268 ++++ .../ScreenSharingOwnershipTests.swift | 231 ++++ ...creenSharingPresentationMetricsTests.swift | 17 + ...ScreenSharingRTCIntervalMetricsTests.swift | 36 + .../ScreenSharingRenderCoordinatorTests.swift | 864 +++++++++++++ ...eenSharingRtcEventLogDiagnosticTests.swift | 351 ++++++ .../ScreenSharingSourceIdleTests.swift | 479 ++++++++ .../ScreenSharingSourceMarkerTests.swift | 102 ++ .../ScreenSharingVideoRefreshTests.swift | 177 +++ .../ScreenSharingWorkloadSequenceTests.swift | 44 + .../CodevisorUI/MarkdownDocumentFailure.swift | 8 + .../CodevisorUI/MarkdownDocumentModel.swift | 8 + packages/swift/Package.resolved | 11 +- packages/swift/Package.swift | 33 +- packages/swift/TestSupport/TestClock.swift | 3 + scripts/build-webrtc.mjs | 297 +++++ scripts/dev-instance.mjs | 2 + scripts/dev-instance.test.mjs | 40 + scripts/dev-macos-reuse.mjs | 34 + scripts/dev-macos-reuse.test.mjs | 85 ++ scripts/dev.mjs | 76 +- scripts/screen-sharing-probe.mjs | 112 ++ scripts/webrtc-build.lock.json | 23 + 193 files changed, 20146 insertions(+), 224 deletions(-) create mode 100644 .github/workflows/webrtc-artifact.yml create mode 160000 .repos/WebRTC create mode 100644 apps/macos/Codevisor/Features/Panes/ScreenSharingPane.swift create mode 100644 apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift create mode 100644 apps/server/src/routes/screen-sharing.ts create mode 100644 apps/server/src/screen-sharing.test.ts create mode 100644 docs/plans/native-screen-sharing-mvp.md create mode 100644 docs/plans/native-screen-sharing.md create mode 100644 packages/api/src/screen-sharing.ts create mode 100644 packages/swift/CodevisorClient/Sources/CodevisorClient/ServerScreenSharingRequest.swift create mode 100644 packages/swift/CodevisorClient/Tests/CodevisorClientTests/UnresolvedMachineTests.swift create mode 100644 packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneWorkingDirectory.swift create mode 100644 packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/ScreenSharingPanePreferences.swift create mode 100644 packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ChatlessWorkspacePaneTests.swift create mode 100644 packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ConvertedPaneTitleTests.swift create mode 100644 packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ScreenSharingPaneMappingTests.swift create mode 100644 packages/swift/CodevisorCore/Tests/CodevisorCoreTests/WorkspaceSelectionRouteTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/NativeScreenSharingBridge.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostConnectivity.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostControl.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostIndicator.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostLease.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostService.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputInjector.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputSurface.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingPasteboard.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingVideoSurface.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerClipboard.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerDiagnostics.swift create mode 100644 packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingConnectivityTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingControlTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingDiagnosticsTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostLeaseTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostRequestTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingPasteboardTests.swift create mode 100644 packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeCapabilities.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeCapturePicker.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeClipboardCheck.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeClockSync.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeCodecCheck.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeCodecPattern.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeControlCheck.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopPainter.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopWorkload.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeHostRecovery.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeMetalDisplayLink.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeOptions.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeOwnedWorkloadWindow.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeTimeline.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeTransportTimeline.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeWindowObservation.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ProbeWorkloadView.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/ScreenSharingProbe.swift create mode 100644 packages/swift/CodevisorScreenSharing/Probe/SyntheticSource.swift create mode 100644 packages/swift/CodevisorScreenSharing/README.md create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/NALUnitBitstream.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-LICENSE.txt create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingAdaptiveQuality.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCapture.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureBufferCopy.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureIntervalRequest.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureRequestState.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardMessage.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardTransfer.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCodecFactory.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingControlMessage.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDataChannel.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoder.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoderRecoveryCheck.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDiagnosticProfile.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoder.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropCheck.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropLog.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFieldTrials.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameDeliveryAudit.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameIdentity.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameSender.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingICEConfiguration.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingKeyframeRequest.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetalView.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics+Presentation.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowCapture.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowObservation.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowSession.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingPeer.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRTCIntervalMetrics.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRenderCoordinator.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRtcEventLogDiagnostic.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingScrollAccumulator.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceIdle.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceMarker.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoCodec.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoConfiguration.swift create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoRefresh.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/NALUnitBitstreamTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingAdaptiveQualityTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingBufferTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureBufferCopyTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureIntervalRequestTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureUpdateTransactionTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingClipboardTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingControlMessageTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDecoderRecoveryCheckTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDiagnosticProfileTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDrawTimestampRecordTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingEncoderDropLogTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingFrameDeliveryAuditTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingHEVCFormatTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingICETests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingKeyframeRequestTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowCaptureTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowObservationTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowSessionTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnershipTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingPresentationMetricsTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRTCIntervalMetricsTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRenderCoordinatorTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRtcEventLogDiagnosticTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceIdleTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceMarkerTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingVideoRefreshTests.swift create mode 100644 packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingWorkloadSequenceTests.swift create mode 100644 scripts/build-webrtc.mjs create mode 100644 scripts/dev-instance.test.mjs create mode 100644 scripts/dev-macos-reuse.mjs create mode 100644 scripts/dev-macos-reuse.test.mjs create mode 100644 scripts/screen-sharing-probe.mjs create mode 100644 scripts/webrtc-build.lock.json diff --git a/.agents/skills/run-dev/SKILL.md b/.agents/skills/run-dev/SKILL.md index 18b9fbb9c..4412708c6 100644 --- a/.agents/skills/run-dev/SKILL.md +++ b/.agents/skills/run-dev/SKILL.md @@ -21,4 +21,6 @@ The runners install locked dependencies, resolve GhosttyKit when needed, and kee Keep at most one runner active per worktree. Track and stop the parent process you started before rebuilding. Never stop or modify another worktree's instance; if this worktree already has an instance you do not own, reuse it or report that it is running. +For a permission-grant restart that must preserve the existing signed macOS binary, use `bun run dev:macos --reuse-macos-build` (container flags still apply). This verifies the worktree app's identity and strict signature, skips native building/signing, and starts the usual managed services and app. It deliberately does not incorporate native source edits. Missing, mismatched, or invalid artifacts fail without falling back to a rebuild. Normal launches continue to build. This option is macOS-only. + For website-only development, use `bun run dev:web`. diff --git a/.github/workflows/webrtc-artifact.yml b/.github/workflows/webrtc-artifact.yml new file mode 100644 index 000000000..3f2f525c1 --- /dev/null +++ b/.github/workflows/webrtc-artifact.yml @@ -0,0 +1,42 @@ +name: Native WebRTC Artifact + +# Manual until the source build and generated notices have been reviewed. +# This job uploads a candidate; it does not publish or change SwiftPM pins. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-webrtc-artifact + cancel-in-progress: false + +jobs: + build: + runs-on: [self-hosted, macOS, ARM64, codevisor-macos-arm64] + timeout-minutes: 240 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Read pinned tools + id: tools + run: | + node -e 'const p=JSON.parse(require("fs").readFileSync("scripts/webrtc-build.lock.json")); console.log("python="+p.pythonVersion); console.log("xcode="+p.xcodeVersion)' >> "$GITHUB_OUTPUT" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ steps.tools.outputs.python }} + - name: Build frameworks and dependency notices + env: + DEVELOPER_DIR: /Applications/Xcode_${{ steps.tools.outputs.xcode }}.app/Contents/Developer + run: node scripts/build-webrtc.mjs + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: native-webrtc-candidate + retention-days: 14 + if-no-files-found: error + path: | + tmp/webrtc-source/*/artifacts/WebRTC.xcframework.zip + tmp/webrtc-source/*/artifacts/manifest.json + tmp/webrtc-source/*/artifacts/revisions.txt + tmp/webrtc-source/*/artifacts/*-gn-args.txt + tmp/webrtc-source/*/artifacts/notices-*/LICENSE.md diff --git a/.gitmodules b/.gitmodules index 7ece614a0..e6e1e7cab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -128,3 +128,6 @@ [submodule ".repos/cef"] path = .repos/cef url = https://github.com/chromiumembedded/cef.git +[submodule ".repos/WebRTC"] + path = .repos/WebRTC + url = https://github.com/stasel/WebRTC.git diff --git a/.repos/WebRTC b/.repos/WebRTC new file mode 160000 index 000000000..1d0469269 --- /dev/null +++ b/.repos/WebRTC @@ -0,0 +1 @@ +Subproject commit 1d04692697cb642bfebf6ad2dd99fe52649c3d6d diff --git a/apps/ios/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/apps/ios/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index ffeb62cc7..258f42f08 100644 --- a/apps/ios/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/apps/ios/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "35d49b86089100e9627cb12a984950b3d7590c5a60fbf1fd56250b4a429d9187", + "originHash" : "e4c1ace4bc5f71677b3ab179e798262a24616c20b9d3b22d343c1084ecb77130", "pins" : [ { "identity" : "posthog-ios", @@ -36,6 +36,15 @@ "revision" : "dd2fb8ac5b861e7bf617c872895e338f38165648", "version" : "1.15.0" } + }, + { + "identity" : "webrtc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/stasel/WebRTC.git", + "state" : { + "revision" : "1d04692697cb642bfebf6ad2dd99fe52649c3d6d", + "version" : "152.0.0" + } } ], "version" : 3 diff --git a/apps/ios/Codevisor/Features/Home/HomeSidebarSection.swift b/apps/ios/Codevisor/Features/Home/HomeSidebarSection.swift index 329a9adb3..1d5737eab 100644 --- a/apps/ios/Codevisor/Features/Home/HomeSidebarSection.swift +++ b/apps/ios/Codevisor/Features/Home/HomeSidebarSection.swift @@ -31,6 +31,7 @@ struct HomeSidebarTabRow: Identifiable, Equatable { case browser(favicon: UIImage?) case plugin(pluginId: String, paneType: String?) case document + case screenSharing case newTab } diff --git a/apps/ios/Codevisor/Features/Home/HomeView+Sidebar.swift b/apps/ios/Codevisor/Features/Home/HomeView+Sidebar.swift index 47a5162dc..c3d1c61f7 100644 --- a/apps/ios/Codevisor/Features/Home/HomeView+Sidebar.swift +++ b/apps/ios/Codevisor/Features/Home/HomeView+Sidebar.swift @@ -122,7 +122,7 @@ extension HomeView { return "New Tab" case .browser: return BrowserPaneCache.shared.localTitle(paneId: pane.id) ?? pane.name - case .terminal, .plugin, .document: + case .terminal, .plugin, .document, .screenSharing: return pane.name } } @@ -142,6 +142,8 @@ extension HomeView { .plugin(pluginId: pane.pluginId ?? "", paneType: pane.pluginPaneType) case .document: .document + case .screenSharing: + .screenSharing case .newTab: .newTab } diff --git a/apps/ios/Codevisor/Features/Home/Views/HomeSidebarTabIcon.swift b/apps/ios/Codevisor/Features/Home/Views/HomeSidebarTabIcon.swift index 7ec65d0d6..6344d1856 100644 --- a/apps/ios/Codevisor/Features/Home/Views/HomeSidebarTabIcon.swift +++ b/apps/ios/Codevisor/Features/Home/Views/HomeSidebarTabIcon.swift @@ -48,6 +48,8 @@ struct HomeSidebarTabIcon: View { } else { symbol("puzzlepiece.extension") } + case .screenSharing: + symbol("display") case .document: symbol("doc.richtext") case .newTab: @@ -71,6 +73,7 @@ extension HomeSidebarTabRow.Icon { case let .terminal(isAgentOwned): isAgentOwned ? "Agent terminal" : "Terminal" case .browser: "Browser" case .plugin: "Plugin" + case .screenSharing: "Screen Sharing" case .document: "Document" case .newTab: "New Tab" } diff --git a/apps/ios/Codevisor/Features/Workspace/Views/WorkspacePaneContentView.swift b/apps/ios/Codevisor/Features/Workspace/Views/WorkspacePaneContentView.swift index 46c2e6d9e..e70367fdd 100644 --- a/apps/ios/Codevisor/Features/Workspace/Views/WorkspacePaneContentView.swift +++ b/apps/ios/Codevisor/Features/Workspace/Views/WorkspacePaneContentView.swift @@ -110,6 +110,10 @@ struct WorkspacePaneContentView: View { .task(id: environment.machines.httpConnectionState(forMachineId: machineId)) { await browserPaneModel(pane).connectionDidChange() } + case .screenSharing: + ContentUnavailableView( + "Screen Sharing", systemImage: "display", + description: Text("Screen Sharing currently requires Codevisor on a Mac.")) case .document: if let path = pane.documentPath { MarkdownDocumentView( diff --git a/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen+PaneMutations.swift b/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen+PaneMutations.swift index 7d0867e1a..5217293b2 100644 --- a/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen+PaneMutations.swift +++ b/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen+PaneMutations.swift @@ -154,7 +154,7 @@ extension WorkspaceScreen { return lhs.chatSessionId != nil && lhs.chatSessionId == rhs.chatSessionId case .terminal: return lhs.terminalKey.caseInsensitiveCompare(rhs.terminalKey) == .orderedSame - case .newTab, .plugin, .document, .browser: + case .newTab, .plugin, .document, .browser, .screenSharing: return false } } diff --git a/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen.swift b/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen.swift index 2cacdaa41..7130f4001 100644 --- a/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen.swift +++ b/apps/ios/Codevisor/Features/Workspace/WorkspaceScreen.swift @@ -222,7 +222,7 @@ struct WorkspaceScreen: View { return "New Tab" case .browser: return BrowserPaneCache.shared.localTitle(paneId: pane.id) ?? pane.name - case .terminal, .plugin, .document: + case .terminal, .plugin, .document, .screenSharing: return pane.name } } diff --git a/apps/macos/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/apps/macos/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 8f1b14799..397168eff 100644 --- a/apps/macos/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/apps/macos/Codevisor.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "aaaf46b8bd150932159e0baafbbf82405ece06f941aa3fd05f0f57722ca439a1", + "originHash" : "da75afb3d4ca32b49af7986c40633508d582c53102496cedab3c2bd742bef7d8", "pins" : [ { "identity" : "posthog-ios", @@ -27,6 +27,15 @@ "revision" : "d46d456107feacc80711b21847b82b07bd9fb46e", "version" : "2.9.3" } + }, + { + "identity" : "webrtc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/stasel/WebRTC.git", + "state" : { + "revision" : "1d04692697cb642bfebf6ad2dd99fe52649c3d6d", + "version" : "152.0.0" + } } ], "version" : 3 diff --git a/apps/macos/Codevisor/CodevisorApp.swift b/apps/macos/Codevisor/CodevisorApp.swift index 06415895c..6dda6a048 100644 --- a/apps/macos/Codevisor/CodevisorApp.swift +++ b/apps/macos/Codevisor/CodevisorApp.swift @@ -476,6 +476,8 @@ struct RootView: View { switch selection { case let .session(serverId, sessionId): sessionDetail(store, serverId: serverId, sessionId: sessionId) + case let .workspace(serverId, workspaceId): + workspaceDetail(store, serverId: serverId, workspaceId: workspaceId) case let .newChat(target): newChat(store, target: target) case .none: @@ -498,10 +500,9 @@ struct RootView: View { { let controller = store.controller(for: session, project: project) SessionContainerView( - session: session, + mount: .chat(session, controller), project: project, store: store, - controller: controller, onFocusedChatChanged: { chatId in self.selection = .session(serverId: serverId, id: chatId) } @@ -524,6 +525,42 @@ struct RootView: View { } } + /// A workspace shown without a chat: the same container, mounted on the + /// workspace itself. Its panes, splits, toolbar and New Tab page are the + /// shared ones; nothing here creates a session, a worktree or an agent. + @ViewBuilder + private func workspaceDetail( + _ store: SessionStore, + serverId: String, + workspaceId: UUID + ) -> some View { + if let workspace = environment.workspaces.workspace(id: workspaceId), + workspace.serverId == serverId, + let project = environment.projectList.projects.first(where: { + $0.serverId == serverId && $0.id == workspace.projectId + }) + { + SessionContainerView( + mount: .workspace(workspace), + project: project, + store: store, + // The moment a chat exists in this workspace (New Tab → New Chat), the + // selection moves to it: the container remounts as `.chat`, which is + // what upgrades the cached leaf group and restores chat affordances. + onFocusedChatChanged: { chatId in + self.selection = .session(serverId: serverId, id: chatId) + } + ) + .id("\(serverId):\(workspaceId.uuidString)") + } else { + ContentUnavailableView( + "Workspace Unavailable", + systemImage: "rectangle.on.rectangle.slash", + description: Text("This workspace is no longer available on its machine.") + ) + } + } + /// The standalone new-chat page. Creates NOTHING until the first message /// is sent — sending resolves the picked directory (project folder or a /// fresh worktree) and materializes the workspace around the started @@ -544,6 +581,10 @@ struct RootView: View { /// Identifies the current sidebar selection. enum SidebarSelection: Hashable { case session(serverId: String, id: UUID) + /// A workspace shown on its own. Workspaces own their layout and server + /// identity independently of any chat, so one that has never hosted a chat + /// is still somewhere the user can be. + case workspace(serverId: String, id: UUID) case newChat(NewChatTarget?) } diff --git a/apps/macos/Codevisor/Features/Panes/MarkdownDocumentPane.swift b/apps/macos/Codevisor/Features/Panes/MarkdownDocumentPane.swift index 3f1ba56f8..433da6bef 100644 --- a/apps/macos/Codevisor/Features/Panes/MarkdownDocumentPane.swift +++ b/apps/macos/Codevisor/Features/Panes/MarkdownDocumentPane.swift @@ -14,11 +14,18 @@ final class MarkdownDocumentPane: Pane { init(context: PaneContext, descriptor: PaneDescriptorState) { self.id = descriptor.id - self.document = MarkdownDocumentModel( - path: descriptor.documentPath ?? "", - sessionId: context.sessionId, - client: context.client ?? CodevisorServerClient(config: context.machine.serverConfig) - ) + let path = descriptor.documentPath ?? "" + // Reading a file goes through a chat session's file API. A pane group + // without one reports the document as unavailable rather than fetching + // under some other session's identity. + self.document = + context.sessionId.map { sessionId in + MarkdownDocumentModel( + path: path, + sessionId: sessionId, + client: context.client ?? CodevisorServerClient(config: context.machine.serverConfig) + ) + } ?? MarkdownDocumentModel(unavailablePath: path) } func makeView() -> AnyView { diff --git a/apps/macos/Codevisor/Features/Panes/NewTabPageView.swift b/apps/macos/Codevisor/Features/Panes/NewTabPageView.swift index 222a6e21a..b4a0194fb 100644 --- a/apps/macos/Codevisor/Features/Panes/NewTabPageView.swift +++ b/apps/macos/Codevisor/Features/Panes/NewTabPageView.swift @@ -18,6 +18,7 @@ private struct NewTabOption: Identifiable, Equatable { case chat case terminal case browser + case screenSharing case plugin(pluginId: String, paneType: String, iconPath: String?) } @@ -42,6 +43,7 @@ struct NewTabPageView: View { var client: (any CodevisorServerClienting)? = nil var iconCacheNamespace = "preview" + @State private var supportsScreenSharing = false @State private var pluginOptions: [NewTabOption] = [] @State private var query = "" /// Focusing this pane focuses the picker's input — never on appearance, @@ -55,7 +57,9 @@ struct NewTabPageView: View { NewTabOption(id: "chat", title: "New Chat", kind: .chat), NewTabOption(id: "browser", title: "New Browser", kind: .browser), NewTabOption(id: "terminal", title: "New Terminal", kind: .terminal), - ] + pluginOptions + ] + + (supportsScreenSharing + ? [NewTabOption(id: "screen-sharing", title: "Screen Sharing", kind: .screenSharing)] : []) + pluginOptions } var body: some View { @@ -102,6 +106,8 @@ struct NewTabPageView: View { Autocomplete.Action(option.title, id: option.id, systemImage: "text.bubble") { open(option) } case .browser: Autocomplete.Action(option.title, id: option.id, systemImage: "globe") { open(option) } + case .screenSharing: + Autocomplete.Action(option.title, id: option.id, systemImage: "display") { open(option) } case .terminal: Autocomplete.Action(option.title, id: option.id, systemImage: "terminal") { open(option) } case let .plugin(pluginId, paneType, iconPath): @@ -134,6 +140,8 @@ struct NewTabPageView: View { } else { group?.convertNewTabPane(id: paneId, to: .chat) } + case .screenSharing: + group?.convertNewTabPane(id: paneId, to: .screenSharing) case .browser: group?.convertNewTabPane(id: paneId, to: .browser) case .terminal: @@ -158,7 +166,10 @@ struct NewTabPageView: View { /// page's built-in options never depend on the request. private func loadPluginOptions() async { guard let client else { return } - guard let plugins = try? await client.listPlugins() else { return } + async let info = try? client.info() + async let availablePlugins = try? client.listPlugins() + supportsScreenSharing = await info?.features?.contains("screen-sharing-v1") == true + guard let plugins = await availablePlugins else { return } pluginOptions = plugins.flatMap { plugin in plugin.panes.map { pane in NewTabOption( diff --git a/apps/macos/Codevisor/Features/Panes/Pane.swift b/apps/macos/Codevisor/Features/Panes/Pane.swift index 79a88853f..17a75d13f 100644 --- a/apps/macos/Codevisor/Features/Panes/Pane.swift +++ b/apps/macos/Codevisor/Features/Panes/Pane.swift @@ -15,8 +15,11 @@ import CodevisorCore struct PaneContext { /// This pane's stable identity (persisted; survives app restarts). let paneId: UUID - /// The chat session this pane group belongs to. - let sessionId: UUID + /// The chat session this pane group belongs to, or nil when the group's + /// identity comes from its workspace instead (a workspace that has never + /// hosted a chat). Panes that genuinely need a session decline rather than + /// borrowing another identity. + let sessionId: UUID? /// The key the server's PTY manager stores this pane's shell under. let terminalKey: String /// Agent-owned background terminal: attach to the registered terminal @@ -25,9 +28,14 @@ struct PaneContext { /// The machine (server URL + auth) the pane's backing resources live on. let machine: CodevisorMachine /// Source data for working-directory resolution: the anchor session's cwd - /// (the workspace's one working directory), else the project folder. - let session: ChatSession + /// (the workspace's one working directory), else the project folder. Nil + /// when the workspace has no chat yet — the project folder then applies, + /// exactly as it does for a session without a cwd. + let session: ChatSession? let project: Project + /// The workspace's own working directory, used when there is no chat to + /// anchor on. Nil falls back to the project folder. + var workspaceRootDirectory: String? = nil /// The owning workspace's identity, for panes whose server resources are /// workspace-scoped (plugin pane tokens). Nil in previews. var workspaceId: UUID? = nil diff --git a/apps/macos/Codevisor/Features/Panes/PaneGroupModel+Operations.swift b/apps/macos/Codevisor/Features/Panes/PaneGroupModel+Operations.swift index a33041abf..42fb4dc83 100644 --- a/apps/macos/Codevisor/Features/Panes/PaneGroupModel+Operations.swift +++ b/apps/macos/Codevisor/Features/Panes/PaneGroupModel+Operations.swift @@ -9,8 +9,12 @@ extension PaneGroupModel { /// Adds a terminal tab, selects it, opens the group, and returns the live /// pane (so callers can focus it). The shell opens in the workspace's /// working directory. + /// Returns nil for a group with no session identity: the terminal key + /// namespaces the server's live PTY per session. Only bottom panels create + /// terminals directly, and those are always session-scoped. @discardableResult - func addTerminalPane() -> any Pane { + func addTerminalPane() -> (any Pane)? { + guard let sessionId else { return nil } let previouslySelected = selectedPane let descriptor = state.addTerminalPane(sessionId: sessionId) persist() @@ -280,6 +284,7 @@ extension PaneGroupModel { wireChatHost(chat, paneId: descriptor.id) } if let browser = livePane as? BrowserPane { wireBrowser(browser) } + if let sharing = livePane as? ScreenSharingPane { wireScreenSharing(sharing) } live[descriptor.id] = livePane } if let previous, previous.id != descriptor.id { diff --git a/apps/macos/Codevisor/Features/Panes/PaneGroupModel.swift b/apps/macos/Codevisor/Features/Panes/PaneGroupModel.swift index 9d9c1d070..2c20641b2 100644 --- a/apps/macos/Codevisor/Features/Panes/PaneGroupModel.swift +++ b/apps/macos/Codevisor/Features/Panes/PaneGroupModel.swift @@ -7,12 +7,19 @@ import Foundation import Observation import SwiftUI import CodevisorCore +import CodevisorCoreMac import CodevisorUI @MainActor @Observable final class PaneGroupModel: Identifiable { - let sessionId: UUID + /// The chat session this group belongs to, or nil when the group's identity + /// comes from its workspace instead (a workspace that has never hosted a + /// chat). Nothing substitutes another id here: a nil session means features + /// that need a real session identity are unavailable, not renamed. It becomes + /// non-nil exactly once, through `adoptSession`, when a real chat appears in + /// this leaf. + private(set) var sessionId: UUID? /// A center leaf or the persisted group of background-task terminals. let placement: PaneGroupPlacement var state: PaneGroupState @@ -25,7 +32,9 @@ final class PaneGroupModel: Identifiable { @ObservationIgnored var live: [UUID: any Pane] = [:] @ObservationIgnored private let repository: any PaneGroupRepository - @ObservationIgnored private let makeContext: (PaneDescriptorState) -> PaneContext + /// Rebuilt by `adoptSession` so panes created after a chat appears get the + /// chat-anchored context instead of the workspace-only one. + @ObservationIgnored private var makeContext: (PaneDescriptorState) -> PaneContext @ObservationIgnored let pluginIconClient: (any CodevisorServerClienting)? @ObservationIgnored let pluginIconCacheNamespace: String /// Set by the workspace container: moves keyboard focus to the composer (used @@ -89,7 +98,7 @@ final class PaneGroupModel: Identifiable { @ObservationIgnored var workspaceCommandHandler: ((PaneGroupCommand) -> Bool)? init( - sessionId: UUID, + sessionId: UUID?, placement: PaneGroupPlacement = .bottom, repository: any PaneGroupRepository, pluginIconClient: (any CodevisorServerClienting)? = nil, @@ -110,7 +119,9 @@ final class PaneGroupModel: Identifiable { let initial: PaneGroupState = switch placement { case .bottom: PaneGroupState() - case .center: .centerInitial(sessionId: sessionId) + case .center: + sessionId.map { PaneGroupState.centerInitial(sessionId: $0) } + ?? .centerInitialWithoutChat() } self.state = initial repository.save(initial, sessionId: sessionId, placement: placement) @@ -118,11 +129,33 @@ final class PaneGroupModel: Identifiable { ChromiumAutomationBridge.shared.addGroup(self) } + /// Binds this group to the chat that now exists in its workspace. Groups are + /// cached per workspace+leaf, so the model the user created a chat in is the + /// same object that later hosts it: without this it would keep a nil identity + /// and leave terminals, browser automation and file-backed panes unavailable + /// forever. Live panes are preserved; only those whose content depended on + /// the missing session are dropped so they rebuild against the real one. + /// Adoption happens once — an existing identity is never re-pointed. + func adoptSession( + _ sessionId: UUID, + makeContext: @escaping (PaneDescriptorState) -> PaneContext + ) { + guard self.sessionId == nil else { return } + self.sessionId = sessionId + self.makeContext = makeContext + for pane in state.panes where pane.kind == .document { + discardLivePane(id: pane.id) + } + } + func canHostBrowserAutomation(sessionId requested: String) -> Bool { guard placement == .center, createBrowserTab != nil, let descriptor = state.panes.first, makeContext(descriptor).machine.isLocal else { return false } - return sessionId.uuidString.lowercased() == requested.lowercased() + // Both sides compare ACTUAL session identities: a group without one simply + // never matches on its own id, and can still host automation for a real + // chat pane living inside it. + return sessionId?.uuidString.lowercased() == requested.lowercased() || state.panes.contains { $0.chatSessionId?.uuidString.lowercased() == requested.lowercased() } } @@ -143,6 +176,10 @@ final class PaneGroupModel: Identifiable { let browser = BrowserPane(context: makeContext(descriptor), descriptor: descriptor) wireBrowser(browser) pane = browser + case .screenSharing: + let sharing = ScreenSharingPane(context: makeContext(descriptor), descriptor: descriptor) + wireScreenSharing(sharing) + pane = sharing case .document: let document = MarkdownDocumentPane(context: makeContext(descriptor), descriptor: descriptor) document.onFocus = { [weak self] in self?.requestBackgroundFocus?() } @@ -175,6 +212,19 @@ final class PaneGroupModel: Identifiable { return pane } + func wireScreenSharing(_ sharing: ScreenSharingPane) { + sharing.onFocus = { [weak self] in self?.requestBackgroundFocus?() } + sharing.onPreferencesChanged = { [weak self, weak sharing] preferences in + guard let self, let sharing, + let index = self.state.panes.firstIndex(where: { $0.id == sharing.id }), + self.state.panes[index].screenSharing != preferences + else { return } + self.state.panes[index].screenSharing = preferences + self.persist() + self.onPaneChanged?(self.state.panes[index]) + } + } + func wireBrowser(_ browser: BrowserPane) { browser.model.onOpenLink = { [weak self, weak browser] url, destination, popup in guard let self, let browser else { return false } @@ -219,7 +269,7 @@ final class PaneGroupModel: Identifiable { self.pendingNewTabFocus = paneId self.requestBackgroundFocus?() } - case .terminal, .plugin, .document, .browser: + case .terminal, .plugin, .document, .browser, .screenSharing: break } } @@ -313,6 +363,8 @@ final class PaneGroupModel: Identifiable { if Self.requiresNewLivePane(previous: previous, next: next) { discardLivePane(id: id) invalidatedLiveIds.insert(id) + } else if let sharing = live[id] as? ScreenSharingPane { + sharing.model?.applyPreferences(next.screenSharing ?? .init()) } } @@ -403,6 +455,8 @@ final class PaneGroupModel: Identifiable { // re-pointed at another plugin/pane type needs a fresh webview. return previous.pluginId != next.pluginId || previous.pluginPaneType != next.pluginPaneType + case (.screenSharing, .screenSharing): + return false case (.browser, .browser): return false case (.document, .document): diff --git a/apps/macos/Codevisor/Features/Panes/PluginPane.swift b/apps/macos/Codevisor/Features/Panes/PluginPane.swift index 2b0fc6ca9..e0af0ee53 100644 --- a/apps/macos/Codevisor/Features/Panes/PluginPane.swift +++ b/apps/macos/Codevisor/Features/Panes/PluginPane.swift @@ -56,7 +56,10 @@ final class PluginPane: Pane, Identifiable { /// The workspace's one working directory (the anchor session's cwd, else /// the project folder) — same resolution the terminal pane uses. private var workingDirectory: String { - context.session.cwd ?? context.project.folderURL.path + PaneWorkingDirectory.resolve( + anchor: context.session.map { .session(cwd: $0.cwd) } ?? .workspace, + workspaceRootDirectory: context.workspaceRootDirectory, + projectFolderPath: context.project.folderURL.path) } private var client: any CodevisorServerClienting { diff --git a/apps/macos/Codevisor/Features/Panes/ScreenSharingPane.swift b/apps/macos/Codevisor/Features/Panes/ScreenSharingPane.swift new file mode 100644 index 000000000..dc5b2e528 --- /dev/null +++ b/apps/macos/Codevisor/Features/Panes/ScreenSharingPane.swift @@ -0,0 +1,119 @@ +import AppKit +import CodevisorCore +import CodevisorCoreMac +import CodevisorUI +import SwiftUI + +@MainActor +final class ScreenSharingPane: Pane { + let id: UUID + let kind: PaneKind = .screenSharing + let model: ScreenSharingViewerModel? + let machineName: String + let isLocal: Bool + var onGroupCommand: ((PaneGroupCommand) -> Void)? + var onFocusChanged: ((Bool) -> Void)? { didSet { model?.onFocusChanged = onFocusChanged } } + var onFocus: (() -> Void)? + var onPreferencesChanged: ((ScreenSharingPanePreferences) -> Void)? { + didSet { model?.onPreferencesChanged = onPreferencesChanged } + } + private var mounts = Set() + + init(context: PaneContext, descriptor: PaneDescriptorState) { + id = descriptor.id + machineName = context.machine.name + isLocal = context.machine.isLocal + model = context.workspaceId.map { + ScreenSharingViewerModel( + client: context.client ?? CodevisorServerClient(config: context.machine.serverConfig), + workspaceId: $0, paneId: descriptor.id, preferences: descriptor.screenSharing ?? .init()) + } + } + func makeView() -> AnyView { AnyView(ScreenSharingPaneView(pane: self)) } + func focus() { if model?.control?.state != .controlling { onFocus?() } } + func visibilityChanged(_ visible: Bool) { model?.setVisible(visible) } + func willDelete() async { mounts = []; await model?.close() } + func detach() { mounts = []; model?.setVisible(false) } + func mounted(_ token: UUID) { mounts.insert(token); model?.setVisible(true) } + func unmounted(_ token: UUID) { + mounts.remove(token) + // SwiftUI reparents carried panes within the same presentation update. + // Coalesce that handoff; a true navigation-away has no replacement mount. + DispatchQueue.main.async { [weak self] in + guard let self, self.mounts.isEmpty else { return } + self.model?.setVisible(false) + } + } +} + +private struct ScreenSharingPaneView: View { + let pane: ScreenSharingPane + @Environment(\.theme) private var theme + @State private var mount = UUID() + + var body: some View { + Group { + if let model = pane.model { + VStack(spacing: 0) { + ScreenSharingToolbar(model: model, machineName: pane.machineName) + if let message = model.control?.message { + Text(message).font(.caption).foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading).padding(.horizontal, 12).padding(.bottom, 8) + } + if let message = model.clipboard?.message { + Text(message).font(.caption).foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading).padding(.horizontal, 12).padding(.bottom, 8) + } + Divider() + ZStack { + if let videoView = model.videoView { ScreenSharingNativeView(view: videoView) } + if model.phase != .viewing { + VStack(spacing: 12) { + if model.phase == .loading || model.phase == .connecting || model.phase == .reconnecting { + ProgressView().controlSize(.small) + Text( + model.phase == .loading + ? "Finding displays…" + : model.phase == .reconnecting + ? "Reconnecting to \(pane.machineName)…" : "Connecting to \(pane.machineName)…") + } else { + Image(systemName: "display").font(.system(size: 32)).foregroundStyle(.secondary) + Text("Screen Sharing").font(.headline) + Text(model.message ?? "Choose a display to view on \(pane.machineName).") + .foregroundStyle(.secondary).multilineTextAlignment(.center).frame(maxWidth: 380) + if model.phase == .failed { + Button("Retry") { model.refresh() } + if pane.isLocal { + Button("Screen Recording Settings") { + if let url = URL( + string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") + { + NSWorkspace.shared.open(url) + } + } + } + } + } + } + .padding(24) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } else { + ContentUnavailableView( + "Screen Sharing unavailable", systemImage: "display", + description: Text("Open this pane in a workspace connected to a Mac.")) + } + } + .background(theme.paneBackground) + .onAppear { pane.mounted(mount) } + .onDisappear { pane.unmounted(mount) } + } +} + +private struct ScreenSharingNativeView: NSViewRepresentable { + let view: NSView + func makeNSView(context: Context) -> NSView { view } + func updateNSView(_ nsView: NSView, context: Context) {} +} diff --git a/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift b/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift new file mode 100644 index 000000000..6f285ebba --- /dev/null +++ b/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift @@ -0,0 +1,132 @@ +import CodevisorClient +import CodevisorCore +import CodevisorCoreMac +import SwiftUI + +struct ScreenSharingToolbar: View { + let model: ScreenSharingViewerModel + let machineName: String + @State private var showDiagnostics = false + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + ViewThatFits(in: .horizontal) { + HStack(spacing: 12) { + display; Spacer(minLength: 8); controls + } + VStack(alignment: .leading, spacing: 8) { + display + controls + } + } + if model.control?.state == .controlling { + Text("⌃⌥Esc releases control").font(.caption).foregroundStyle(.secondary) + } + } + .controlSize(.small).padding(.horizontal, 12).padding(.vertical, 8) + } + + private var display: some View { + HStack(spacing: 8) { + Image(systemName: "display") + if !model.displays.isEmpty { + Picker("Display", selection: Binding(get: { model.selectedDisplayId ?? "" }, set: { model.selectDisplay($0) })) + { + if model.selectedDisplayId == nil { Text("Choose a display").tag("") } + ForEach(model.displays) { display in + Text("\(display.name) · \(display.width) × \(display.height)").tag(display.id) + } + } + .labelsHidden().frame(idealWidth: 300, maxWidth: 300) + } else { + Text(machineName).lineLimit(1) + } + } + } + + private var controls: some View { + ViewThatFits(in: .horizontal) { + HStack(spacing: 12) { + controlActions; videoActions + } + VStack(alignment: .leading, spacing: 8) { + HStack { + controlActions; Spacer(minLength: 0) + } + HStack { + videoActions; Spacer(minLength: 0) + } + } + } + } + + @ViewBuilder private var controlActions: some View { + if model.phase == .viewing, let control = model.control { + if control.state == .controlling { + Button("Stop Control") { control.release() } + } else if control.state == .requesting { + Button("Cancel Control") { control.release() } + } else { + Text("View Only").foregroundStyle(.secondary) + Button("Control") { control.request() }.disabled(!control.available) + .help("Send mouse, keyboard and app shortcuts to this Mac. Control–Option–Escape returns to viewing.") + } + } else { + Text("View Only").foregroundStyle(.secondary) + } + } + + @ViewBuilder private var videoActions: some View { + if model.phase == .viewing || model.phase == .connecting || model.phase == .reconnecting { + Picker("Size", selection: Binding(get: { model.preferences.fitToWindow }, set: { model.setFitToWindow($0) })) { + Text("Fit").tag(true) + Text("Actual Size").tag(false) + } + .labelsHidden().frame(width: 110) + if let clipboard = model.clipboard { + Menu { + Button("Send Clipboard to Mac") { clipboard.sendLocalText() } + Button("Get Clipboard from Mac") { clipboard.getRemoteText() } + } label: { + Image(systemName: "doc.on.clipboard") + } + .accessibilityLabel("Clipboard") + .help("Transfer plain text between clipboards") + .disabled(!clipboard.available || clipboard.busy) + } + Button { + showDiagnostics.toggle() + } label: { + Image(systemName: "info.circle") + } + .accessibilityLabel("Connection Details") + .popover(isPresented: $showDiagnostics) { details.padding(16).frame(width: 280) } + Button("Disconnect") { model.disconnect() } + } else { + Button("Connect") { model.connect() } + .disabled(model.selectedDisplayId == nil || model.phase == .loading) + } + } + + @ViewBuilder private var details: some View { + if let diagnostics = model.diagnostics { + VStack(alignment: .leading, spacing: 10) { + Text("Connection Details").font(.headline) + LabeledContent("Route", value: diagnostics.route) + LabeledContent("Video", value: diagnostics.resolution) + if let fps = diagnostics.framesPerSecond { LabeledContent("Presented", value: String(format: "%.1f fps", fps)) } + if let rate = diagnostics.megabitsPerSecond { + LabeledContent("Receiving", value: String(format: "%.2f Mbps", rate)) + } + if let rtt = diagnostics.roundTripMilliseconds { + LabeledContent("Round trip", value: String(format: "%.1f ms", rtt)) + } + if let decode = diagnostics.decodeMilliseconds { + LabeledContent("Decode p95", value: String(format: "%.2f ms", decode)) + } + Text(diagnostics.decoder).font(.caption).foregroundStyle(.secondary) + } + .font(.callout) + } + } +} diff --git a/apps/macos/Codevisor/Features/Panes/TerminalPane.swift b/apps/macos/Codevisor/Features/Panes/TerminalPane.swift index 35359f5a2..1e63478b1 100644 --- a/apps/macos/Codevisor/Features/Panes/TerminalPane.swift +++ b/apps/macos/Codevisor/Features/Panes/TerminalPane.swift @@ -38,7 +38,8 @@ final class TerminalPane: Pane, Identifiable { project: context.project, machine: context.machine, terminalKey: context.terminalKey, - attachOnly: context.attachOnly + attachOnly: context.attachOnly, + workspaceRootDirectory: context.workspaceRootDirectory ) } diff --git a/apps/macos/Codevisor/Features/Panes/WorkspaceSplitDragCoordinator.swift b/apps/macos/Codevisor/Features/Panes/WorkspaceSplitDragCoordinator.swift index 89d6ba3dc..9ee0a3f23 100644 --- a/apps/macos/Codevisor/Features/Panes/WorkspaceSplitDragCoordinator.swift +++ b/apps/macos/Codevisor/Features/Panes/WorkspaceSplitDragCoordinator.swift @@ -252,6 +252,7 @@ struct WorkspaceSplitDragGhost: View { case .plugin: "puzzlepiece.extension" case .document: "doc.richtext" case .browser: "globe" + case .screenSharing: "display" } } diff --git a/apps/macos/Codevisor/Features/Panes/WorkspaceSplitView.swift b/apps/macos/Codevisor/Features/Panes/WorkspaceSplitView.swift index 2a0b036d5..5b4919de9 100644 --- a/apps/macos/Codevisor/Features/Panes/WorkspaceSplitView.swift +++ b/apps/macos/Codevisor/Features/Panes/WorkspaceSplitView.swift @@ -379,6 +379,7 @@ private struct SplitLeafHeader: View { case .plugin: "puzzlepiece.extension" case .document: "doc.richtext" case .browser: "globe" + case .screenSharing: "display" case .newTab, .none: "square.dashed" } } @@ -444,7 +445,7 @@ private struct SplitLeafHeader: View { switch pane?.kind { case .chat: "Rename Chat" case .terminal: "Rename Terminal" - case .newTab, .plugin, .document, .browser, .none: "Rename Pane" + case .newTab, .plugin, .document, .browser, .screenSharing, .none: "Rename Pane" } } diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+Documents.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+Documents.swift index 4f1a721bf..d73acbfe8 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+Documents.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+Documents.swift @@ -9,10 +9,10 @@ extension SessionContainerView { /// Document identity belongs to the workspace's machine and canonical /// path. Clicking the same document again selects its existing tab/split. func openMarkdownDocument(_ target: String) -> Bool { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace guard let path = MarkdownDocumentPath.resolve( - target, relativeTo: workspace.rootDirectory ?? session.cwd ?? project.folderURL.path), + target, relativeTo: workspace.rootDirectory ?? session?.cwd ?? project.folderURL.path), MarkdownDocumentPath.isMarkdown(path) else { return false } diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+Panes.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+Panes.swift index b879b1378..c8cf4190b 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+Panes.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+Panes.swift @@ -10,7 +10,7 @@ extension SessionContainerView { func configuredCenterModel(leafId: UUID) -> PaneGroupModel { let model = store.centerGroup( leafId: leafId, - workspace: store.workspace(for: session, project: project), + workspace: selectedWorkspace, session: session, project: project ) @@ -56,7 +56,7 @@ extension SessionContainerView { // session (recoverable from the archived list); the // session itself always survives. if let closed = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == closedSessionId + $0.serverId == selectedWorkspace.serverId && $0.id == closedSessionId }) { environment.archiveSession(closed) } @@ -88,8 +88,8 @@ extension SessionContainerView { onNewChat: { [weak model] in createChat(convertingPlaceholder: descriptor.id, in: model) }, - client: environment.machines.client(for: session.serverId), - iconCacheNamespace: session.serverId + client: environment.machines.client(for: selectedWorkspace.serverId), + iconCacheNamespace: selectedWorkspace.serverId )) } return AnyView( @@ -98,6 +98,7 @@ extension SessionContainerView { group: model, focus: sessionFocus, session: session, + hostWorkspace: selectedWorkspace, project: project, store: store, environment: environment @@ -110,7 +111,7 @@ extension SessionContainerView { /// leaf, when it is a chat. Reads the live group model so pane selection /// changes re-evaluate the publisher above. var focusedChatCandidate: UUID? { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let leafId = workspace.selectedCenterTab?.resolvedActiveLeafId(preferred: activeLeafId) else { return nil } @@ -131,7 +132,7 @@ extension SessionContainerView { ) { guard let model else { return } guard model.state.panes.contains(where: { $0.id == paneId }) else { return } - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let created = NewChatPanePromoter.promote( paneId: paneId, @@ -149,7 +150,7 @@ extension SessionContainerView { /// Removes an emptied split leaf. A layout may need an empty shell, but /// that shell is not a shared pane and is never uploaded as New Tab. func dissolveIfEmpty(leafId: UUID) { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace let model = store.centerGroup( leafId: leafId, workspace: workspace, session: session, project: project ) @@ -174,14 +175,14 @@ extension SessionContainerView { environment.workspaceSync.publishPane( pane, workspaceId: workspaceId, - client: environment.machines.client(for: session.serverId) + client: environment.machines.client(for: selectedWorkspace.serverId) ) } /// Makes a leaf the active group (keyboard routing + hints). func activateLeaf(_ leafId: UUID?) { guard let leafId else { return } - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace store.selectDestination(.leaf(leafId), in: workspace.id) } @@ -204,7 +205,7 @@ extension SessionContainerView { func rememberWorkspaceDefaults(from chatId: UUID) { guard let chat = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == chatId + $0.serverId == selectedWorkspace.serverId && $0.id == chatId }) else { return } if let live = store.activeController(for: chat) { @@ -230,7 +231,7 @@ extension SessionContainerView { func chatPaneTitle(_ descriptor: PaneDescriptorState) -> String { guard let id = descriptor.chatSessionId else { return descriptor.name } return environment.projectList.sessions.first { - $0.serverId == session.serverId && $0.id == id + $0.serverId == selectedWorkspace.serverId && $0.id == id }?.title ?? descriptor.name } @@ -239,11 +240,11 @@ extension SessionContainerView { /// a chat whose controller isn't cached contributes nothing, and its /// persisted tabs survive untouched until it reconnects. var workspaceChatControllers: [(chatId: UUID, controller: SessionController)] { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace return workspace.chatSessionIds.compactMap { chatId in guard let chat = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == chatId + $0.serverId == selectedWorkspace.serverId && $0.id == chatId }), let controller = store.activeController(for: chat) else { return nil } return (chatId, controller) @@ -262,6 +263,9 @@ extension SessionContainerView { } func syncWorkspaceBackgroundTerminals() { + // Agent-owned background terminals belong to chats. A workspace with no + // chat has no controllers to sync and no bottom panel keyed to a session. + guard let session else { return } let panel = store.paneGroup(for: session, project: project) for (chatId, controller) in workspaceChatControllers { panel.syncAgentTerminals( diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+ReopenPane.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+ReopenPane.swift index 497c1fb0b..492622d87 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+ReopenPane.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+ReopenPane.swift @@ -13,7 +13,7 @@ extension SessionContainerView { if descriptor.kind == .chat, descriptor.chatSessionId == nil { return } // The hook fires before the tab is pruned, so its position is still // readable here. - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let tabIndex = workspace.centerTabs.firstIndex(where: { $0.root.group(id: leafId) != nil }) else { return } @@ -34,7 +34,7 @@ extension SessionContainerView { /// server shell); a plugin pane reloads. Entries whose chat no longer /// exists are skipped in favor of the next one. func reopenClosedPane() { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace while let record = store.popClosedPane(workspaceId: workspace.id) { // A chat reopened meanwhile (from the archive) already has a pane: // go to it rather than open a duplicate. @@ -43,7 +43,7 @@ extension SessionContainerView { workspace.tabId(containingChat: chatId) != nil { store.selectDestination(.chat(chatId), in: workspace.id) - if chatId != session.id { onFocusedChatChanged?(chatId) } + if chatId != session?.id { onFocusedChatChanged?(chatId) } return } guard let pane = restoredDescriptor(for: record.descriptor) else { continue } @@ -57,7 +57,7 @@ extension SessionContainerView { environment.workspaces.save(workspace) store.selectDestination(.tab(tab.id), in: workspace.id) publishPane(pane, workspaceId: workspace.id) - if pane.kind == .chat, let chatId = pane.chatSessionId, chatId != session.id { + if pane.kind == .chat, let chatId = pane.chatSessionId, chatId != session?.id { // The sidebar follows the already selected chat tab. onFocusedChatChanged?(chatId) } @@ -75,7 +75,7 @@ extension SessionContainerView { case .chat: guard let chatId = closed.chatSessionId, let chat = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == chatId + $0.serverId == selectedWorkspace.serverId && $0.id == chatId }) else { return nil } if chat.isArchived { @@ -86,6 +86,10 @@ extension SessionContainerView { terminalKey: paneId.uuidString, chatSessionId: chatId ) case .terminal: + // The terminal key namespaces the server's PTY per session; a workspace + // with no chat cannot name one, so its terminal panes stay closed rather + // than reopening under a substituted namespace. + guard let session else { return nil } return PaneDescriptorState( id: paneId, kind: .terminal, name: closed.name, terminalKey: "\(session.id.uuidString):\(paneId.uuidString)" @@ -100,6 +104,10 @@ extension SessionContainerView { return PaneDescriptorState( id: paneId, kind: .browser, name: closed.name, terminalKey: paneId.uuidString, browserURL: closed.browserURL) + case .screenSharing: + return PaneDescriptorState( + id: paneId, kind: .screenSharing, name: closed.name, + terminalKey: paneId.uuidString, screenSharing: closed.screenSharing) case .document: return PaneDescriptorState( id: paneId, kind: .document, name: closed.name, diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+SplitCommands.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+SplitCommands.swift index ab45d7a76..d65205473 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+SplitCommands.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+SplitCommands.swift @@ -9,7 +9,7 @@ extension SessionContainerView { /// workspace. Resolve navigation from the committed window destination. var navigationWorkspace: Workspace { store.navigationWorkspaceId.flatMap { environment.workspaces.workspace(id: $0) } - ?? store.workspace(for: session, project: project) + ?? selectedWorkspace } func saveSelectedTree(_ tree: SplitNode, workspaceId: UUID) { @@ -72,7 +72,7 @@ extension SessionContainerView { } func splitActiveLeaf(edge: SplitEdge) { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let tab = workspace.selectedCenterTab else { return } splitLeaf(activeLeafId ?? tab.activeLeafId, edge: edge) } @@ -81,7 +81,7 @@ extension SessionContainerView { /// their owning leaf; keyboard/menu commands pass the active leaf. func splitLeaf(_ leafId: UUID, edge: SplitEdge) { guard openingSplit == nil else { return } - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace rememberWorkspaceDefaults(fromLeaf: leafId, in: workspace) guard let tabIndex = workspace.centerTabs.firstIndex(where: { @@ -125,7 +125,7 @@ extension SessionContainerView { /// group id survives, so its cached model and any live terminal surface /// move with the layout instead of being torn down and recreated. func moveSplitLeaf(_ sourceLeafId: UUID, relativeTo targetLeafId: UUID, edge: SplitEdge) { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace guard let tabIndex = workspace.selectedCenterTabIndex else { return } let current = workspace.centerTabs[tabIndex].root guard current.group(id: sourceLeafId) != nil, @@ -158,7 +158,7 @@ extension SessionContainerView { edge: SplitEdge, canvasSize: CGSize ) -> Bool { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let current = workspace.selectedCenterTab?.root, canvasSize.width > 0, canvasSize.height > 0 @@ -187,13 +187,13 @@ extension SessionContainerView { } func closeActiveLeaf() { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let tab = workspace.selectedCenterTab else { return } closeLeaf(activeLeafId ?? tab.activeLeafId) } func closeLeaf(_ leafId: UUID) { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace let closesActiveLeaf = leafId == (activeLeafId ?? workspace.selectedCenterTab?.activeLeafId) let model = configuredCenterModel(leafId: leafId) guard let paneId = model.state.selectedPaneId else { return } @@ -209,7 +209,7 @@ extension SessionContainerView { if descriptor.kind == .chat, let chatId = descriptor.chatSessionId, let chat = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == chatId + $0.serverId == selectedWorkspace.serverId && $0.id == chatId }) { environment.projectList.renameSession(chat, to: trimmed) diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+Toolbar.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+Toolbar.swift index b8743adb6..ff47babc3 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+Toolbar.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+Toolbar.swift @@ -6,7 +6,7 @@ extension SessionContainerView { /// change and its focus callback. A stale split must never own browser commands. private var activeToolbarGroup: PaneGroupModel? { let _ = (workspaceRevision, store.workspaceLayoutRevision, environment.workspaceSync.revision) - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace guard let leafId = workspace.selectedCenterTab?.resolvedActiveLeafId(preferred: activeLeafId) else { return nil } return configuredCenterModel(leafId: leafId) } @@ -28,24 +28,24 @@ extension SessionContainerView { get: { guard let descriptor = activePaneDescriptor else { return "New Tab" } if descriptor.kind == .browser { return "" } - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace return workspace.selectedCenterTab?.customTitle ?? paneTitle(descriptor) }, set: { title in guard activePaneDescriptor?.kind != .browser else { return } - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace renameCenterTab(workspace.selectedCenterTabId, to: title) } ) } var activePaneSubtitle: String { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace let candidates: [String?] = [ workspace.name, project.name, workspace.worktreeName, - environment.machines.fleetMachineName(for: session.serverId), + environment.machines.fleetMachineName(for: workspace.serverId), ] var parts: [String] = [] for candidate in candidates { diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView+WorkspaceTabs.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView+WorkspaceTabs.swift index f65eb31a0..cff5ec871 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView+WorkspaceTabs.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView+WorkspaceTabs.swift @@ -10,7 +10,7 @@ extension SessionContainerView { /// that content identity instead of leaving their creation-time snapshot /// in front of repository truth. func synchronizeMountedPaneGroups() { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace if store.reconcileMountedPaneGroups(in: workspace) { workspaceRevision += 1 } @@ -33,7 +33,7 @@ extension SessionContainerView { } func selectCenterTab(_ tabId: UUID) { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace store.selectDestination(.tab(tabId), in: workspace.id) } @@ -47,7 +47,7 @@ extension SessionContainerView { } func addCenterTab() { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace if let current = workspace.selectedCenterTab { rememberWorkspaceDefaults( fromLeaf: activeLeafId ?? current.activeLeafId, @@ -67,7 +67,7 @@ extension SessionContainerView { } func renameCenterTab(_ tabId: UUID, to customTitle: String?) { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace guard let index = workspace.centerTabs.firstIndex(where: { $0.id == tabId }) else { return } let trimmed = customTitle?.trimmingCharacters(in: .whitespacesAndNewlines) let normalized = trimmed.flatMap { $0.isEmpty ? nil : $0 } @@ -78,7 +78,7 @@ extension SessionContainerView { } func closeCenterTab(_ tabId: UUID) { - var workspace = store.workspace(for: session, project: project) + var workspace = selectedWorkspace guard let index = workspace.centerTabs.firstIndex(where: { $0.id == tabId }) else { return } let closing = workspace.centerTabs[index] let closesSelectedTab = workspace.selectedCenterTabId == tabId @@ -95,7 +95,7 @@ extension SessionContainerView { // this tab would close the workspace's final pane, that pane has been // converted in place and the tab remains. Otherwise empty leaves and // the now-empty layout tab are purely local cleanup. - workspace = store.workspace(for: session, project: project) + workspace = selectedWorkspace workspace.pruneClosedCenterTab(tabId) for leaf in closing.root.allGroups where workspace.centerTabs.allSatisfy({ $0.root.group(id: leaf.id) == nil }) { diff --git a/apps/macos/Codevisor/Features/Session/SessionContainerView.swift b/apps/macos/Codevisor/Features/Session/SessionContainerView.swift index 4513b1554..0aedeaf5f 100644 --- a/apps/macos/Codevisor/Features/Session/SessionContainerView.swift +++ b/apps/macos/Codevisor/Features/Session/SessionContainerView.swift @@ -2,15 +2,36 @@ import SwiftUI import CodevisorCore import CodevisorUI -/// Hosts an already-resolved session controller below the native toolbar -/// (which follows the active pane). +/// Hosts one workspace below the native toolbar (which follows the active +/// pane): either a resolved chat session and its controller, or the workspace +/// alone when it has never hosted a chat. struct SessionContainerView: View { - let session: ChatSession + /// What this container is mounted on. A workspace owns its layout, server + /// identity and pane persistence with or without a chat; the chat case adds + /// the anchor session and its controller. There is no third state, so no + /// call site has to invent a session to show a workspace. + enum Mount { + /// Resolved synchronously with the navigation selection so the destination + /// shell never waits for this view's asynchronous setup task to run. + case chat(ChatSession, SessionController) + /// The mount-time snapshot; the live record is re-read from the repository. + case workspace(Workspace) + } + + let mount: Mount let project: Project let store: SessionStore - /// Resolved synchronously with the navigation selection so the destination - /// shell never waits for this view's asynchronous setup task to run. - let controller: SessionController + + /// The anchor chat, when there is one. Chat focus, read and open reporting, + /// and anything keyed by a session identity, all go through this. + var session: ChatSession? { + if case let .chat(session, _) = mount { return session } + return nil + } + var controller: SessionController? { + if case let .chat(_, controller) = mount { return controller } + return nil + } /// Fired when the user's focus lands in a DIFFERENT chat of this /// workspace (composer/transcript click, chat tab) — the sidebar /// selection follows, keeping its tab rows in sync with focus. @@ -40,9 +61,24 @@ struct SessionContainerView: View { } } + /// Re-runs the container's setup task when the mounted thing changes. + var mountIdentity: UUID { + switch mount { + case let .chat(session, _): return session.id + case let .workspace(snapshot): return snapshot.id + } + } + var selectedWorkspace: Workspace { let _ = (workspaceRevision, store.workspaceLayoutRevision, environment.workspaceSync.revision) - return store.workspace(for: session, project: project) + switch mount { + case let .chat(session, _): + return store.workspace(for: session, project: project) + case let .workspace(snapshot): + // The repository owns the live record; the snapshot covers the window + // between a remote deletion and the selection moving away. + return environment.workspaces.workspace(id: snapshot.id) ?? snapshot + } } var activeLeafId: UUID? { @@ -80,11 +116,11 @@ struct SessionContainerView: View { .focusedSceneValue( \.workspaceLayoutActions, WorkspaceLayoutActions( - workspaceId: store.workspace(for: session, project: project).id, + workspaceId: selectedWorkspace.id, newTab: addCenterTab, closeSplit: closeActiveLeaf, closeTab: { - let workspace = store.workspace(for: session, project: project) + let workspace = selectedWorkspace closeCenterTab(workspace.selectedCenterTabId) }, reopenClosedPane: reopenClosedPane, @@ -114,7 +150,7 @@ struct SessionContainerView: View { // Navigation itself has already committed before view construction. .onChange(of: store.centerTabRequest, initial: true) { _, request in guard let request, store.centerTabRequest == request, - request.workspaceId == store.workspace(for: session, project: project).id + request.workspaceId == selectedWorkspace.id else { return } store.centerTabRequest = nil performCenterTabRequest(request) @@ -144,7 +180,10 @@ struct SessionContainerView: View { // attention coordinator, which marks the focused chat read. .onChange(of: focusedChatCandidate, initial: true) { _, candidate in publishedFocusCandidate = candidate - store.setFocusedChat(candidate, serverId: session.serverId) + // Focus/read reporting is per server; the candidate is nil unless an + // actual chat pane faces the user, so a chatless workspace publishes + // "no chat focused" rather than a fabricated one. + store.setFocusedChat(candidate, serverId: selectedWorkspace.serverId) } // Release only the focus this container published. Navigating to // another workspace mounts the new container (which publishes its @@ -157,7 +196,7 @@ struct SessionContainerView: View { store.clearFocusedChat(ifCurrent: candidate) } } - .task(id: session.id) { + .task(id: mountIdentity) { splitDragCoordinator.canResolve = { sourceLeafId, resolution, canvasSize in canMoveSplitLeaf( sourceLeafId, @@ -180,20 +219,22 @@ struct SessionContainerView: View { sessionFocus.onChatComposerFocused = { chatId in guard isVisible, store.navigationWorkspaceId == selectedWorkspace.id, - store.workspace(for: session, project: project).centerTree.groupId(containingChat: chatId) != nil + selectedWorkspace.centerTree.groupId(containingChat: chatId) != nil else { return } - if let leaf = store.workspace(for: session, project: project) - .centerTree.groupId(containingChat: chatId), + if let leaf = selectedWorkspace.centerTree.groupId(containingChat: chatId), leaf != activeLeafId { activateLeaf(leaf) } rememberWorkspaceDefaults(from: chatId) - if chatId != session.id { + if chatId != session?.id { onFocusedChatChanged?(chatId) } } - store.markOpened(session.id, serverId: session.serverId) + // Opening is a chat event: a workspace mount has nothing to mark read. + if let session { + store.markOpened(session.id, serverId: session.serverId) + } } } @@ -212,7 +253,7 @@ struct SessionContainerView: View { focus: sessionFocus, onWorkspaceCommand: handleWorkspaceCommand, centerTree: liveCenterTree ?? workspace.centerTree, - primaryLeafId: workspace.centerTree.groupId(containingChat: session.id), + primaryLeafId: session.flatMap { workspace.centerTree.groupId(containingChat: $0.id) }, activeLeafId: activeLeafId, centerLeafModel: { leafId in configuredCenterModel(leafId: leafId) }, centerPaneTitle: paneTitle, diff --git a/apps/macos/Codevisor/Features/Session/SessionScreen.swift b/apps/macos/Codevisor/Features/Session/SessionScreen.swift index 15f1fc7e9..4a540c721 100644 --- a/apps/macos/Codevisor/Features/Session/SessionScreen.swift +++ b/apps/macos/Codevisor/Features/Session/SessionScreen.swift @@ -5,7 +5,9 @@ import CodevisorUI /// The selected workspace tab, with focus routing and attachment loading. struct SessionScreen: View { - @Bindable var controller: SessionController + /// The chat controller, or nil when this screen hosts a workspace that has + /// no chat. Only attachment previews read it; panes, splits and focus do not. + var controller: SessionController? /// The active split's pane group; keyboard commands route through it. var centerGroup: PaneGroupModel /// The session's focus coordinator. Owned by the container (which also @@ -50,7 +52,7 @@ struct SessionScreen: View { focus.startTypeToFocus() installAttachmentImageStoreIfNeeded() } - .onChange(of: controller.previewCacheNamespace) { + .onChange(of: controller?.previewCacheNamespace) { installAttachmentImageStoreIfNeeded() } .onDisappear { @@ -61,6 +63,13 @@ struct SessionScreen: View { } private func installAttachmentImageStoreIfNeeded() { + // No chat controller, no attachment previews: drop any store a previous + // controller installed rather than leaving this screen serving a cache + // whose owner is gone. + guard let controller else { + attachmentImages = nil + return + } let namespace = controller.previewCacheNamespace guard attachmentImages?.namespace != namespace else { return } attachmentImages = AttachmentImageStore( diff --git a/apps/macos/Codevisor/Features/Session/SessionStore+PaneGroups.swift b/apps/macos/Codevisor/Features/Session/SessionStore+PaneGroups.swift index 77d9e0711..4e97cdaf2 100644 --- a/apps/macos/Codevisor/Features/Session/SessionStore+PaneGroups.swift +++ b/apps/macos/Codevisor/Features/Session/SessionStore+PaneGroups.swift @@ -88,15 +88,24 @@ extension SessionStore { /// A specific center-tree LEAF's group model (split groups beyond the /// primary). Cached per (workspace, leaf) so panes survive navigation. + /// `session` is nil for a workspace that has never hosted a chat: the leaf + /// still has its own persisted layout, keyed by workspace and leaf. func centerGroup( leafId: UUID, workspace: Workspace, - session: ChatSession, + session: ChatSession?, project: Project ) -> PaneGroupModel { let key = CenterLeafKey(workspaceId: workspace.id, groupId: leafId) - if let existing = centerLeafGroups[key] { return existing } - let group = makePaneGroup(for: session, project: project, placement: .center, leafId: leafId) + if let existing = centerLeafGroups[key] { + // The same leaf can be entered first without a chat and later with one. + if let session, existing.sessionId == nil { + adoptSession(session, project: project, workspace: workspace, in: existing) + } + return existing + } + let group = makePaneGroup( + workspace: workspace, session: session, project: project, placement: .center, leafId: leafId) centerLeafGroups[key] = group return group } @@ -132,84 +141,92 @@ extension SessionStore { return changed } + /// Unchanged entry point for every session-rooted call site: it resolves the + /// session's workspace (backfilling pre-workspace state on first access) and + /// then runs the same shared implementation a chatless workspace uses. func makePaneGroup( for session: ChatSession, project: Project, placement: PaneGroupPlacement, leafId: UUID? = nil ) -> PaneGroupModel { - let machine = environment.machines.machine(for: session.serverId) ?? CodevisorMachine.local - // Pane layout persists in the session's workspace (the pre-workspace - // per-session states migrate in on first access). Center groups pin - // to a specific tree leaf: the given one, else the leaf hosting this - // session's chat. - let workspace = workspace(for: session, project: project) + makePaneGroup( + workspace: workspace(for: session, project: project), session: session, project: project, + placement: placement, leafId: leafId) + } + + /// The shared implementation. The WORKSPACE supplies identity (server, + /// persistence, publication); the session, when there is one, supplies chat + /// affordances: the leaf that hosts its chat, the pane context's session and + /// the browser link/automation hooks. + func makePaneGroup( + workspace: Workspace, + session: ChatSession?, + project: Project, + placement: PaneGroupPlacement, + leafId: UUID? = nil + ) -> PaneGroupModel { + let serverId = workspace.serverId + // A session mount keeps its historical fallback. A workspace mount must not: + // substituting `.local` for an unresolved REMOTE machine would quietly move + // that workspace's terminals and screen sharing onto this Mac. Unavailable + // is the correct answer, under the workspace's own server id. + let machine = + environment.machines.machine(for: serverId) + ?? (session != nil || serverId == CodevisorMachine.local.id + ? CodevisorMachine.local : CodevisorMachine.unresolved(id: serverId)) + // Center groups pin to a specific tree leaf: the given one, else the leaf + // hosting this session's chat (a workspace without a chat has only the + // given leaf). let resolvedLeafId = placement == .center ? (leafId - ?? workspace.centerTabs.lazy.compactMap { - $0.root.groupId(containingChat: session.id) - }.first) + ?? session.flatMap { session in + workspace.centerTabs.lazy.compactMap { + $0.root.groupId(containingChat: session.id) + }.first + }) : nil let repository = WorkspacePaneGroupRepository( workspaceId: workspace.id, groupId: resolvedLeafId, repository: environment.workspaces ) - let client = environment.machines.client(for: session.serverId) + let client = environment.machines.client(for: serverId) let workspaceIdForPanes = workspace.id + // The workspace's own working directory anchors panes that have no chat. + let workspaceRootDirectory = workspace.rootDirectory let model = PaneGroupModel( - sessionId: session.id, + sessionId: session?.id, placement: placement, repository: repository, pluginIconClient: client, - pluginIconCacheNamespace: session.serverId, - makeContext: { - [ - weak projectList = environment.projectList, - weak machines = environment.machines - ] descriptor in - // Panes are built lazily, so this cached closure can outlive - // the snapshot passed in above: a fresh worktree session may - // not have synced its cwd yet. Resolve the live session at - // pane-creation time so terminals open in the worktree, not - // the project folder. - let liveSession = - projectList?.sessions.first { - $0.serverId == session.serverId && $0.id == session.id - } ?? session - return PaneContext( - paneId: descriptor.id, - sessionId: session.id, - terminalKey: descriptor.terminalKey, - attachOnly: descriptor.attachOnly, - machine: machine, - session: liveSession, - project: project, - workspaceId: workspaceIdForPanes, - client: client, - resolveHTTPBaseURL: { - await machines?.effectiveHTTPBaseURL(forMachineId: session.serverId) - } - ) - } + pluginIconCacheNamespace: serverId, + makeContext: paneContextFactory( + session: session, project: project, machine: machine, client: client, serverId: serverId, + workspaceId: workspaceIdForPanes, workspaceRootDirectory: workspaceRootDirectory) ) - if placement == .center { - model.openBrowserLink = { [weak self] source, url, destination, popup in - self?.openBrowserLink( - for: session, project: project, sourcePaneId: source, url: url, destination: destination, popup: popup - ) ?? false - } - model.createBrowserTab = { [weak self] url in - self?.createBrowserTab(for: session, project: project, url: url) - } + // Browser links and automation are chat-rooted: they open next to a real + // session. A workspace without one leaves both hooks nil, which is also + // what makes `canHostBrowserAutomation` decline for its panes. + if placement == .center, let session { + installBrowserHooks(on: model, session: session, project: project) } - model.onPaneChanged = { [weak environment] pane in + model.onPaneChanged = { [weak self, weak environment] pane in + // A pane changing IN PLACE — a New Tab becoming Screen Sharing, a rename, + // a draft binding its chat — is a local layout write just like adding or + // closing a tab, and the descriptor is already persisted by the time this + // runs. Bump the same token those structural writes use so the sidebar + // re-reads the repository now; otherwise its row keeps the previous name + // until an unrelated sync revision happens to arrive, because + // `centerLeafGroups` is deliberately not observable and a row rendered + // before this leaf's model existed holds no dependency on it. + self?.workspaceLayoutRevision += 1 guard let environment else { return } environment.workspaceSync.publishPane( pane, - workspaceId: workspace.id, - client: environment.machines.client(for: session.serverId) + workspaceId: workspaceIdForPanes, + client: environment.machines.client(for: serverId) ) } let workspaceId = workspace.id @@ -229,12 +246,98 @@ extension SessionStore { id: pane.id, workspaceId: workspaceId, optimisticReplacement: replacement, - client: environment.machines.client(for: session.serverId) + client: environment.machines.client(for: serverId) ) } return model } + /// One definition of a pane's context, used when a group is created and again + /// when a chatless group adopts the chat that later appears in its leaf. + private func paneContextFactory( + session: ChatSession?, + project: Project, + machine: CodevisorMachine, + client: (any CodevisorServerClienting)?, + serverId: String, + workspaceId: UUID, + workspaceRootDirectory: String? + ) -> (PaneDescriptorState) -> PaneContext { + { + [ + weak projectList = environment.projectList, + weak machines = environment.machines + ] descriptor in + // Panes are built lazily, so this cached closure can outlive the snapshot + // passed in above: a fresh worktree session may not have synced its cwd + // yet. Resolve the live session at pane-creation time so terminals open in + // the worktree, not the project folder. + let liveSession = session.map { session in + projectList?.sessions.first { + $0.serverId == session.serverId && $0.id == session.id + } ?? session + } + return PaneContext( + paneId: descriptor.id, + sessionId: session?.id, + terminalKey: descriptor.terminalKey, + attachOnly: descriptor.attachOnly, + machine: machine, + session: liveSession, + project: project, + workspaceRootDirectory: workspaceRootDirectory, + workspaceId: workspaceId, + client: client, + resolveHTTPBaseURL: { + await machines?.effectiveHTTPBaseURL(forMachineId: serverId) + } + ) + } + } + + /// Upgrades a cached group that was built for a workspace with no chat once a + /// real chat exists in its leaf. The model, its live panes and its persisted + /// state are kept; only the identity, the context factory and the chat-rooted + /// browser hooks are (re)established. Without this the cached group would keep + /// a nil identity forever and leave terminals, browser automation and + /// file-backed panes unavailable in a workspace that now has a chat. + private func adoptSession( + _ session: ChatSession, + project: Project, + workspace: Workspace, + in model: PaneGroupModel + ) { + let serverId = workspace.serverId + let machine = + environment.machines.machine(for: serverId) + ?? (serverId == CodevisorMachine.local.id + ? CodevisorMachine.local : CodevisorMachine.unresolved(id: serverId)) + let client = environment.machines.client(for: serverId) + model.adoptSession( + session.id, + makeContext: paneContextFactory( + session: session, project: project, machine: machine, client: client, serverId: serverId, + workspaceId: workspace.id, workspaceRootDirectory: workspace.rootDirectory)) + installBrowserHooks(on: model, session: session, project: project) + } + + /// The chat-rooted browser hooks, shared by creation and adoption. + private func installBrowserHooks( + on model: PaneGroupModel, + session: ChatSession, + project: Project + ) { + model.openBrowserLink = { [weak self] source, url, destination, popup in + self?.openBrowserLink( + for: session, project: project, sourcePaneId: source, url: url, destination: destination, + popup: popup + ) ?? false + } + model.createBrowserTab = { [weak self] url in + self?.createBrowserTab(for: session, project: project, url: url) + } + } + /// Browser Use appends a workspace tab in the background, like a pane /// arriving from another device. The user's selected tab and focus stay put. private func createBrowserTab( diff --git a/apps/macos/Codevisor/Features/Session/Views/ChatPaneContentView.swift b/apps/macos/Codevisor/Features/Session/Views/ChatPaneContentView.swift index 2ae37b922..ad5d24c4a 100644 --- a/apps/macos/Codevisor/Features/Session/Views/ChatPaneContentView.swift +++ b/apps/macos/Codevisor/Features/Session/Views/ChatPaneContentView.swift @@ -9,7 +9,12 @@ struct ChatPaneContentView: View { let descriptor: PaneDescriptorState let group: PaneGroupModel? let focus: TerminalFocusController - let session: ChatSession + /// The container's anchor chat, when it has one. Nil for a workspace that + /// has never hosted a chat: identity below comes from `hostWorkspace`. + let session: ChatSession? + /// The workspace hosting this pane — its server identity and id, which exist + /// with or without a chat. + let hostWorkspace: Workspace let project: Project let store: SessionStore let environment: AppEnvironment @@ -17,10 +22,10 @@ struct ChatPaneContentView: View { var body: some View { if let chatSessionId = descriptor.chatSessionId { if let chatSession = environment.projectList.sessions.first(where: { - $0.serverId == session.serverId && $0.id == chatSessionId + $0.serverId == hostWorkspace.serverId && $0.id == chatSessionId }), let chatProject = environment.projectList.projects.first(where: { - $0.serverId == session.serverId && $0.id == chatSession.projectId + $0.serverId == hostWorkspace.serverId && $0.id == chatSession.projectId }) { if isUnstarted(chatSession) { @@ -34,7 +39,7 @@ struct ChatPaneContentView: View { initialProjectTarget: NewChatTarget(chatProject), paneDraftId: descriptor.id, onCreatedInPane: { created in - (group ?? store.centerPaneGroup(for: session, project: project)) + (group ?? session.map { store.centerPaneGroup(for: $0, project: project) })? .assignChatSession( paneId: descriptor.id, sessionId: created.id, @@ -43,7 +48,7 @@ struct ChatPaneContentView: View { }, preCreatedSession: chatSession, paneFocus: focus, - hostWorkspaceId: store.workspace(for: session, project: project).id + hostWorkspaceId: hostWorkspace.id ) } else { let controller = store.controller(for: chatSession, project: chatProject) @@ -90,14 +95,14 @@ struct ChatPaneContentView: View { onCreatedInPane: { created in // Bind through the pane's OWNING group (the draft may // live in any split leaf, not just the primary). - (group ?? store.centerPaneGroup(for: session, project: project)) + (group ?? session.map { store.centerPaneGroup(for: $0, project: project) })? .assignChatSession( paneId: descriptor.id, sessionId: created.id, name: created.title ) }, - hostWorkspaceId: store.workspace(for: session, project: project).id + hostWorkspaceId: hostWorkspace.id ) } } diff --git a/apps/macos/Codevisor/Features/Sidebar/SidebarView+Tabs.swift b/apps/macos/Codevisor/Features/Sidebar/SidebarView+Tabs.swift index 79aef75ec..c7a020d48 100644 --- a/apps/macos/Codevisor/Features/Sidebar/SidebarView+Tabs.swift +++ b/apps/macos/Codevisor/Features/Sidebar/SidebarView+Tabs.swift @@ -169,11 +169,12 @@ extension SidebarView { func activateTab(_ tab: WorkspaceTab, in item: SidebarWorkspaceListItem) { let workspace = item.workspace guard store?.selectDestination(.tab(tab.id), in: workspace.id) == true else { return } - if let chat = routableChat(in: tab, serverId: workspace.serverId) { - selection = .session(serverId: chat.serverId, id: chat.id) - } else if !routesSelectedSession(workspace), let routing = item.routingSession { - selection = .session(serverId: routing.serverId, id: routing.id) - } + apply( + workspace.selectionRoute( + activatedChatSessionId: routableChat(in: tab, serverId: workspace.serverId)?.id, + routingSessionId: item.routingSession?.id, + selectionAlreadyRoutesWorkspace: routesSelectedSession(workspace) + )) } /// A pane row: name the leaf, and route through its own chat when it @@ -181,12 +182,27 @@ extension SidebarView { func activateLeaf(_ leafId: UUID, state: PaneGroupState, in item: SidebarWorkspaceListItem) { let workspace = item.workspace guard store?.selectDestination(.leaf(leafId), in: workspace.id) == true else { return } - if let pane = state.selectedPane, let chat = sessionForPane(pane, serverId: workspace.serverId), - !chat.isArchived - { - selection = .session(serverId: chat.serverId, id: chat.id) - } else if !routesSelectedSession(workspace), let routing = item.routingSession { - selection = .session(serverId: routing.serverId, id: routing.id) + let paneChat = state.selectedPane + .flatMap { sessionForPane($0, serverId: workspace.serverId) } + .flatMap { $0.isArchived ? nil : $0 } + apply( + workspace.selectionRoute( + activatedChatSessionId: paneChat?.id, + routingSessionId: item.routingSession?.id, + selectionAlreadyRoutesWorkspace: routesSelectedSession(workspace) + )) + } + + /// Applies the resolved route. A nil route means the current selection + /// already shows this workspace and must not be disturbed. + private func apply(_ route: WorkspaceSelectionRoute?) { + switch route { + case let .session(serverId, id): + selection = .session(serverId: serverId, id: id) + case let .workspace(serverId, id): + selection = .workspace(serverId: serverId, id: id) + case nil: + break } } diff --git a/apps/macos/Codevisor/Features/Sidebar/SidebarView+Workspaces.swift b/apps/macos/Codevisor/Features/Sidebar/SidebarView+Workspaces.swift index dead7b801..db4cd3f78 100644 --- a/apps/macos/Codevisor/Features/Sidebar/SidebarView+Workspaces.swift +++ b/apps/macos/Codevisor/Features/Sidebar/SidebarView+Workspaces.swift @@ -63,12 +63,18 @@ extension SidebarView { ) } - /// Whether the sidebar's selected chat lives in this workspace. + /// Whether the sidebar's selection is showing this workspace: its selected + /// chat lives here, or the workspace itself is selected (it has no chat). func routesSelectedSession(_ workspace: Workspace) -> Bool { - guard case let .session(serverId, sessionId) = selection, - serverId == workspace.serverId - else { return false } - return environment.workspaces.workspaceId(forSession: sessionId) == workspace.id + switch selection { + case let .session(serverId, sessionId): + guard serverId == workspace.serverId else { return false } + return environment.workspaces.workspaceId(forSession: sessionId) == workspace.id + case let .workspace(serverId, id): + return serverId == workspace.serverId && id == workspace.id + case .newChat, .none: + return false + } } /// Archives the WORKSPACE (not just a chat): the record is flagged, its diff --git a/apps/macos/Codevisor/Features/Sidebar/SidebarView.swift b/apps/macos/Codevisor/Features/Sidebar/SidebarView.swift index e155c3de5..d734071bf 100644 --- a/apps/macos/Codevisor/Features/Sidebar/SidebarView.swift +++ b/apps/macos/Codevisor/Features/Sidebar/SidebarView.swift @@ -49,7 +49,7 @@ struct SidebarView: View { var isNewChatSelected: Bool { switch selection { case .newChat, .none: true - case .session: false + case .session, .workspace: false } } diff --git a/apps/macos/Codevisor/Features/Sidebar/Views/SidebarWorkspaceTabRow.swift b/apps/macos/Codevisor/Features/Sidebar/Views/SidebarWorkspaceTabRow.swift index 3376a5b51..a4d6fc751 100644 --- a/apps/macos/Codevisor/Features/Sidebar/Views/SidebarWorkspaceTabRow.swift +++ b/apps/macos/Codevisor/Features/Sidebar/Views/SidebarWorkspaceTabRow.swift @@ -125,6 +125,7 @@ struct SidebarWorkspaceTabRow: View { case .plugin: "puzzlepiece.extension" case .document: "doc.richtext" case .browser: "globe" + case .screenSharing: "display" } } diff --git a/apps/macos/Codevisor/Features/Terminal/TerminalSurface.swift b/apps/macos/Codevisor/Features/Terminal/TerminalSurface.swift index 4630a4688..6a205fda3 100644 --- a/apps/macos/Codevisor/Features/Terminal/TerminalSurface.swift +++ b/apps/macos/Codevisor/Features/Terminal/TerminalSurface.swift @@ -20,18 +20,22 @@ struct TerminalLaunchDescriptor: Equatable { let command: String static func make( - session: ChatSession, + session: ChatSession?, project: Project, machine: CodevisorMachine, terminalKey: String, - attachOnly: Bool = false + attachOnly: Bool = false, + workspaceRootDirectory: String? = nil ) -> TerminalLaunchDescriptor { // The session's cwd IS the workspace's one working directory // (worktree sessions open in the worktree), else the project - // folder. The proxy passes the folder along via --cwd. - let sessionFolder = - session.cwd.map(URL.init(fileURLWithPath:)) - ?? project.folderURL + // folder. The proxy passes the folder along via --cwd. Without a session + // the workspace's own directory takes that place. + let sessionFolder = URL( + fileURLWithPath: PaneWorkingDirectory.resolve( + anchor: session.map { .session(cwd: $0.cwd) } ?? .workspace, + workspaceRootDirectory: workspaceRootDirectory, + projectFolderPath: project.folderURL.path)) return TerminalLaunchDescriptor( terminalKey: terminalKey, attachOnly: attachOnly, diff --git a/apps/server/src/routes/screen-sharing.ts b/apps/server/src/routes/screen-sharing.ts new file mode 100644 index 000000000..2aed5a4a0 --- /dev/null +++ b/apps/server/src/routes/screen-sharing.ts @@ -0,0 +1,85 @@ +import { ScreenSharingReply, ScreenSharingRequest } from "@codevisor/api" +import { Schema } from "effect" +import type { IncomingMessage, ServerResponse } from "node:http" +import { + HttpFailure, + run, + writeJson, + type CodevisorServerConfig, + type CodevisorServerServices +} from "../server-context.js" + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +export const routeScreenSharing = async ( + services: CodevisorServerServices, + config: CodevisorServerConfig, + request: IncomingMessage, + response: ServerResponse, + url: URL +): Promise => { + if (url.pathname !== "/v1/screen-sharing") return false + // Machine authentication has already run. Websites cannot use loopback trust. + if (request.headers.origin !== undefined || request.headers["sec-fetch-site"] !== undefined) + throw new HttpFailure(403, "Native clients only") + if (request.method !== "POST") throw new HttpFailure(405, "Use POST for Screen Sharing") + response.setHeader("Cache-Control", "no-store") + const chunks: Buffer[] = [] + let size = 0 + for await (const chunk of request) { + const bytes = Buffer.from(chunk) + size += bytes.length + if (size > 300 * 1024) throw new HttpFailure(413, "Screen Sharing request is too large") + chunks.push(bytes) + } + let payload: ScreenSharingRequest + try { + payload = Schema.decodeUnknownSync(ScreenSharingRequest)( + JSON.parse(Buffer.concat(chunks).toString("utf8")) + ) + } catch { + throw new HttpFailure(400, "Invalid Screen Sharing request") + } + if (![payload.workspaceId, payload.paneId, payload.viewerId].every((id) => uuid.test(id))) + throw new HttpFailure(400, "Invalid Screen Sharing identity") + if ( + (payload.operation === "start" || payload.operation === "restart") && + (payload.offer === undefined || + Buffer.byteLength(payload.offer) > 256 * 1024 || + !payload.offer.includes("a=fingerprint:sha-256 ") || + payload.displayId === undefined || + !uuid.test(payload.displayId)) + ) + throw new HttpFailure(400, "Invalid Screen Sharing offer or display") + if (["start", "restart", "heartbeat"].includes(payload.operation)) { + const workspace = (await run(services.db.listWorkspaces)).find( + (item) => item.id.toLowerCase() === payload.workspaceId.toLowerCase() + ) + const pane = (await run(services.db.listWorkspacePanes)).find( + (item) => item.id.toLowerCase() === payload.paneId.toLowerCase() + ) + if ( + workspace === undefined || + workspace.isArchived || + pane === undefined || + pane.workspaceId.toLowerCase() !== workspace.id.toLowerCase() || + pane.providerId !== "codevisor" || + pane.paneType !== "screen-sharing" + ) + throw new HttpFailure(404, "Screen Sharing pane is no longer available") + } + if (config.screenSharing === undefined) + throw new HttpFailure(501, "Screen Sharing requires the native Codevisor app on a Mac") + let result: ScreenSharingReply + try { + result = Schema.decodeUnknownSync(ScreenSharingReply)(await config.screenSharing(payload)) + } catch { + // Never expose helper paths, tokens, or SDP through a transport error. + throw new HttpFailure( + 503, + "Open or update Codevisor on the host Mac, then retry Screen Sharing" + ) + } + writeJson(response, 200, result) + return true +} diff --git a/apps/server/src/screen-sharing.test.ts b/apps/server/src/screen-sharing.test.ts new file mode 100644 index 000000000..1f2057c45 --- /dev/null +++ b/apps/server/src/screen-sharing.test.ts @@ -0,0 +1,232 @@ +import type { ScreenSharingRequest } from "@codevisor/api" +import { randomUUID } from "node:crypto" +import { describe, expect, it, vi } from "vitest" +import { jsonRequest, makeServices, run, runningServers, startWithApp } from "./test-support.js" + +const capabilities = (): ScreenSharingRequest => ({ + version: 1, + operation: "capabilities", + workspaceId: randomUUID(), + paneId: randomUUID(), + viewerId: randomUUID() +}) +const offer = "v=0\r\na=fingerprint:sha-256 fixture\r\n" +const reply = { version: 1, status: "available", displays: [] } + +const fixture = async ( + screenSharing: (request: ScreenSharingRequest) => Promise = async () => reply +) => { + const { services } = await makeServices() + const server = await startWithApp(services, undefined, { screenSharing }) + runningServers.push(server) + const project = await run(services.db.createProject({ folderPath: "/fixture/screen-sharing" })) + const workspace = await run( + services.db.upsertWorkspace({ projectId: project.id, name: "Sharing", hasCustomName: false }) + ) + const pane = await run( + services.db.upsertWorkspacePane(workspace.id, { + id: randomUUID(), + providerId: "codevisor", + paneType: "screen-sharing", + title: "Screen Sharing" + }) + ) + const request: ScreenSharingRequest = { + ...capabilities(), + operation: "start", + workspaceId: workspace.id, + paneId: pane.id, + displayId: randomUUID(), + offer + } + const post = (body: unknown = request, headers: Record = {}) => + jsonRequest(server, "/v1/screen-sharing", { + method: "POST", + body: JSON.stringify(body), + headers + }) + return { server, services, project, workspace, pane, request, post } +} + +describe("native Screen Sharing signaling", () => { + it("requires machine authentication and rejects browser-originated requests", async () => { + const helper = vi.fn(async () => reply) + const { services } = await makeServices() + const token = await run(services.db.issuePairingToken) + const server = await startWithApp(services, undefined, { + screenSharing: helper, + auth: { allowLocalhostWithoutAuth: false, requireBearerToken: true } + }) + runningServers.push(server) + const request = { method: "POST", body: JSON.stringify(capabilities()) } + expect((await jsonRequest(server, "/v1/screen-sharing", request)).status).toBe(401) + for (const browserHeader of [ + { Origin: "https://example.com" }, + { "Sec-Fetch-Site": "same-origin" } + ]) { + expect( + ( + await jsonRequest(server, "/v1/screen-sharing", { + ...request, + headers: { Authorization: `Bearer ${token}`, ...browserHeader } + }) + ).status + ).toBe(403) + } + expect(helper).not.toHaveBeenCalled() + expect( + ( + await jsonRequest(server, "/v1/screen-sharing", { + ...request, + headers: { Authorization: `Bearer ${token}` } + }) + ).body + ).toEqual(reply) + expect(helper).toHaveBeenCalledOnce() + }) + + it("advertises availability, passes a valid offer transiently, and permits stop after pane deletion", async () => { + const helper = vi.fn(async (request: ScreenSharingRequest) => ({ + ...reply, + status: request.operation === "start" ? "connecting" : "stopped", + answer: "fixture answer" + })) + const { server, services, workspace, pane, request, post } = await fixture(helper) + expect((await jsonRequest(server, "/v1/info")).body).toMatchObject({ + features: expect.arrayContaining(["screen-sharing-v1"]) + }) + const response = await fetch(`${server.url}/v1/screen-sharing`, { + method: "POST", + body: JSON.stringify(request) + }) + expect(response.status).toBe(200) + expect(response.headers.get("cache-control")).toBe("no-store") + expect(await response.json()).toMatchObject({ status: "connecting", answer: "fixture answer" }) + expect(helper).toHaveBeenLastCalledWith(request) + expect((await post({ ...request, operation: "restart" })).status).toBe(200) + expect((await post({ ...request, operation: "heartbeat" })).status).toBe(200) + expect((await run(services.db.listWorkspacePanes)).find((item) => item.id === pane.id)).toEqual( + pane + ) + await run(services.db.deleteWorkspacePane(workspace.id, pane.id)) + expect((await post({ ...request, operation: "heartbeat" })).status).toBe(404) + expect((await post({ ...request, operation: "restart" })).status).toBe(404) + expect((await post({ ...request, operation: "stop" })).status).toBe(200) + }) + + it("rejects malformed, oversized, or incompatible requests before invoking the helper", async () => { + const helper = vi.fn(async () => reply) + const { server, request, post } = await fixture(helper) + expect((await jsonRequest(server, "/v1/screen-sharing")).status).toBe(405) + expect( + (await jsonRequest(server, "/v1/screen-sharing", { method: "POST", body: "{" })).status + ).toBe(400) + expect((await post({ ...request, version: 2 })).status).toBe(400) + for (const field of ["workspaceId", "paneId", "viewerId"]) { + expect((await post({ ...request, [field]: "invalid" })).status).toBe(400) + } + for (const change of [ + { offer: undefined }, + { offer: "x".repeat(256 * 1024 + 1) }, + { offer: "v=0" }, + { displayId: undefined }, + { displayId: "invalid" } + ]) { + expect((await post({ ...request, ...change })).status).toBe(400) + expect((await post({ ...request, operation: "restart", ...change })).status).toBe(400) + } + expect((await post({ ...request, offer: "x".repeat(301 * 1024) })).status).toBe(413) + expect(helper).not.toHaveBeenCalled() + }) + + it("returns short-lived connectivity credentials without persisting them or allowing caching", async () => { + const connectivity = { + servers: [ + { + urls: ["turn:relay.example.test:3478"], + username: "fixture-viewer", + credential: "ephemeral-password" + } + ], + relayOnly: true, + expiresAt: 1700000300 + } + const { server, request, services, pane } = await fixture(async () => ({ + ...reply, + connectivity + })) + const response = await fetch(`${server.url}/v1/screen-sharing`, { + method: "POST", + body: JSON.stringify({ ...request, operation: "capabilities" }) + }) + expect(response.headers.get("cache-control")).toBe("no-store") + expect(await response.json()).toEqual({ ...reply, connectivity }) + expect((await run(services.db.listWorkspacePanes)).find((item) => item.id === pane.id)).toEqual( + pane + ) + }) + + it("validates workspace, pane, provider and archive state for start and heartbeat", async () => { + const helper = vi.fn(async () => reply) + const { services, project, workspace, pane, request, post } = await fixture(helper) + expect((await post({ ...request, workspaceId: randomUUID() })).status).toBe(404) + expect((await post({ ...request, paneId: randomUUID() })).status).toBe(404) + const second = await run( + services.db.upsertWorkspace({ projectId: project.id, name: "Other", hasCustomName: false }) + ) + expect((await post({ ...request, workspaceId: second.id })).status).toBe(404) + await run( + services.db.upsertWorkspacePane(workspace.id, { + id: pane.id, + title: "Screen Sharing", + providerId: "plugin", + paneType: "screen-sharing" + }) + ) + expect((await post()).status).toBe(404) + await run( + services.db.upsertWorkspacePane(workspace.id, { + id: pane.id, + title: "Screen Sharing", + providerId: "codevisor", + paneType: "browser" + }) + ) + expect((await post()).status).toBe(404) + await run( + services.db.upsertWorkspacePane(workspace.id, { + id: pane.id, + title: "Screen Sharing", + providerId: "codevisor", + paneType: "screen-sharing" + }) + ) + await run(services.db.updateWorkspace(workspace.id, { isArchived: true })) + expect((await post()).status).toBe(404) + expect(helper).not.toHaveBeenCalled() + }) + + it("returns actionable errors for an absent, incompatible or failed native helper", async () => { + const { services } = await makeServices() + const server = await startWithApp(services) + runningServers.push(server) + expect((await jsonRequest(server, "/v1/info")).body).toMatchObject({ + features: expect.not.arrayContaining(["screen-sharing-v1"]) + }) + expect( + ( + await jsonRequest(server, "/v1/screen-sharing", { + method: "POST", + body: JSON.stringify(capabilities()) + }) + ).status + ).toBe(501) + const helper = vi.fn(async (): Promise => ({ version: 2 })) + const { post } = await fixture(helper) + expect((await post(capabilities())).status).toBe(503) + helper.mockRejectedValueOnce(new Error("secret helper path and token")) + const failed = await post(capabilities()) + expect(failed.status).toBe(503) + expect(JSON.stringify(failed.body)).not.toContain("secret") + }) +}) diff --git a/apps/server/src/serve-boot.ts b/apps/server/src/serve-boot.ts index 523c3e0ac..8858dda56 100644 --- a/apps/server/src/serve-boot.ts +++ b/apps/server/src/serve-boot.ts @@ -1,7 +1,8 @@ import { makeTerminalPersistence } from "./infra/terminal-persistence.js" import type { StartupReporter } from "./startup-progress.js" import type { BackgroundTerminalIntegration } from "@codevisor/agent-runtime" -import type { DataUpgradeProgress } from "@codevisor/api" +import type { DataUpgradeProgress, ScreenSharingRequest } from "@codevisor/api" +import { requestMacScreenSharing } from "@codevisor/automation" import type { TerminalManagerService } from "@codevisor/terminal" import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" @@ -18,6 +19,10 @@ import type { ServerLease } from "./infra/server-lease.js" /// initialization, and the background terminal integration. export const SERVER_PROCESS_TITLE = "codevisor-server" +export const nativeScreenSharing = (dataDir: string) => + process.platform === "darwin" + ? (request: ScreenSharingRequest) => requestMacScreenSharing(dataDir, request) + : undefined /// Background cache only: clients checking on the user's behalf pass /// `force` (GET /v1/update?refresh=1) and bypass this entirely. Six hours /// here made remote machines deny fresh releases for most of a day. diff --git a/apps/server/src/serve.ts b/apps/server/src/serve.ts index c880d2199..b07fd7fec 100644 --- a/apps/server/src/serve.ts +++ b/apps/server/src/serve.ts @@ -34,10 +34,9 @@ import { makeHarnessLifecycleManager } from "@codevisor/harness-manager" import { defaultServerConfig, startCodevisorServer } from "./server.js" import { acquireServerLease } from "./infra/server-lease.js" import type { ServerLease } from "./infra/server-lease.js" -import { restoreTerminalPersistence } from "./serve-boot.js" +import { restoreTerminalPersistence, nativeScreenSharing } from "./serve-boot.js" import { makeHarnessAuthManager } from "@codevisor/harness-manager" -import { makeMcpManager } from "@codevisor/mcp" -import { makeNativeMcpManager } from "@codevisor/mcp" +import { makeMcpManager, makeNativeMcpManager } from "@codevisor/mcp" import { makePluginRegistryClient, makePluginsManager, @@ -468,6 +467,7 @@ export const runServe = ( }, 250) }, sessionActivity, + screenSharing: nativeScreenSharing(dirname(databasePath)), updater }) ) diff --git a/apps/server/src/server-context-types.ts b/apps/server/src/server-context-types.ts index 31fe35e14..5185b71c5 100644 --- a/apps/server/src/server-context-types.ts +++ b/apps/server/src/server-context-types.ts @@ -1,3 +1,4 @@ +import type { ScreenSharingRequest } from "@codevisor/api" import type { BrowserProxy } from "./infra/browser-proxy.js" import type { CredentialSource } from "@codevisor/harness-manager" import type { AgentRuntimeService } from "@codevisor/agent-runtime" @@ -76,6 +77,7 @@ export interface CodevisorServerConfig { /// Host power policy for active locally hosted turns. The production macOS /// server supplies a scoped idle-sleep assertion; other platforms/tests /// omit it. + readonly screenSharing?: ((request: ScreenSharingRequest) => Promise) | undefined readonly sessionActivity?: SessionActivityController | undefined /// This machine's Codevisor Cloud device id (from `codevisor auth login`), /// advertised via /v1/info so clients can match this machine to its cloud diff --git a/apps/server/src/server-router.ts b/apps/server/src/server-router.ts index ec09c1ad8..fff5678ad 100644 --- a/apps/server/src/server-router.ts +++ b/apps/server/src/server-router.ts @@ -1,3 +1,4 @@ +import { routeScreenSharing } from "./routes/screen-sharing.js" import { routeBrowserState } from "./routes/browser-state.js" import { routeTranscriptStress } from "./routes/transcript-stress.js" import { applyAfterDrain } from "./apply-after-drain.js" @@ -153,6 +154,8 @@ export const handleRequest = async ( if (await routeTranscriptStress(services, fanout, routeState, request, response, url)) return + if (await routeScreenSharing(services, config, request, response, url)) return + if (await routeBrowserState(services, request, response, url)) return if (request.method === "POST" && url.pathname === "/v1/browser/proxy-session") { @@ -200,6 +203,7 @@ export const handleRequest = async ( "browser-proxy-v1", "browser-http-proxy-v1", "browser-state-v1", + ...(config.screenSharing === undefined ? [] : ["screen-sharing-v1"]), "canonical-chat-v1", "session-event-stream-v1", "transcript-pagination-v1", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 09d794abc..4ab19c521 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -73,6 +73,7 @@ export const defaultServerConfig = ( restartSnapshotPath: overrides.restartSnapshotPath, restartDrainTimeoutMs: overrides.restartDrainTimeoutMs, sessionActivity: overrides.sessionActivity, + screenSharing: overrides.screenSharing, cloudDeviceId: overrides.cloudDeviceId, cloud: overrides.cloud }) diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md new file mode 100644 index 000000000..7107db381 --- /dev/null +++ b/docs/plans/native-screen-sharing-mvp.md @@ -0,0 +1,32 @@ +# Native Screen Sharing MVP checkpoint + +This is an implementation review checkpoint for a macOS LAN beta. It is not a performance-parity or release-acceptance report. + +The intended MVP is one Codevisor Mac viewing and explicitly controlling an existing display on another Codevisor Mac. The host runs the native app in its logged-in session with Screen Recording permission; control additionally requires Accessibility permission. One viewer owns the host at a time. + +## Implemented + +- ScreenCaptureKit capture, VideoToolbox H.264 encode/decode, encrypted WebRTC media and Metal rendering. +- Authenticated workspace/pane signaling, display selection, Fit/Actual Size, explicit control, bounded plain-text clipboard transfer and connection diagnostics. +- Chatless workspace navigation through the shared pane container, including conversion from New Tab and sidebar title updates. +- Bounded media ownership, terminal renderer stop and owner-scoped cancellation of pending host starts. +- A default-off `CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE=paced15-worker` profile: 120 fps capture request at adaptive level zero, 1–15 ms receiver playout bounds, synchronized arrival rendering with two drawables and off-main preparation. A capture request is not an achieved frame rate. Process-wide WebRTC configuration requires an app restart to change. +- A standalone probe and a pinned WebRTC artifact-build recipe. The recipe's complete source build and generated dependency notices remain unverified. + +## Existing verification + +Earlier actual-app checks established local viewing/control and remote video between two Macs. The latest two-Mac check did not establish remote control: the host's native Accessibility check denied it, and external desktop activity changed the source setup. That run is not a performance benchmark. + +The final host stop-ordering source passed 167 CoreMac tests and serial macOS/iOS native builds on September 13, 2026. The sidebar and host-ordering fixes were compiled in isolated build outputs but have not yet been exercised in a newly launched two-Mac session. These are historical results for the current implementation, not a claim that the complete branch has passed fresh CI or integration with current main. + +The local measurement archives contain raw session and machine-specific evidence and are intentionally excluded from this public checkpoint. Historical measurement links in the implementation notes refer to those local archives. No credentials, raw signaling, desktop captures or runner logs are required to build the feature. + +## Before MVP acceptance + +1. Verify the final source in two actual apps: permissions, viewing, typing, clicking, dragging, scrolling, clipboard, scaling and display selection. +2. Verify input release and capture cleanup on focus loss, pane hide/close, disconnect, host stop and reconnect; verify no stale start revives a stopped session. +3. Complete a visible sustained-use check for responsiveness, resources and bounded queues using the configuration intended for users. +4. Review the pinned dependency, distributed notices, signing/packaging and clean CI results. Any revision to the source-build prerequisite in the original plan must be explicit. +5. Resolve integration with current main and review the shared workspace-navigation changes for regressions. + +Apple parity, a production-default tuning change, iOS viewing, audio, virtual displays, multiple viewers and internet/forced-relay guarantees are outside this MVP checkpoint. Existing connectivity code does not establish those guarantees. diff --git a/docs/plans/native-screen-sharing.md b/docs/plans/native-screen-sharing.md new file mode 100644 index 000000000..23ec71e50 --- /dev/null +++ b/docs/plans/native-screen-sharing.md @@ -0,0 +1,168 @@ +# Native Screen Sharing implementation plan + +Status: Native macOS viewing and interactive control are implemented, including bounded explicit text clipboard transfer, captured-cursor fallback, adaptive capture quality, connection diagnostics, optional host-issued STUN/TURN credentials and lease-bound media replacement after network loss. Local native media, UI and authenticated signaling checks pass. Exploratory two-Mac media runs and hardware HEVC 4:4:4 probes now pass on tuftlord (M1 Pro) and the local M4 Max. Transport-loss recovery before an idle desktop (in-band frame identity, host idle notice, viewer delivery audit with a 100 ms threshold, 100 ms grace and up to four progress extensions) is implemented and measured locally through an encrypted UDP relay. The controlled LAN/Apple latency comparison, forced-TURN/network-restriction checks and a completed source artifact/notice audit remain open. The iOS viewer and audio/HEVC/fidelity expansions remain later milestones. See [the implementation and measured results](../../packages/swift/CodevisorScreenSharing/README.md). + +Build Screen Sharing as a native Codevisor feature alongside Terminal: a persistent workspace pane with a native video surface, native input, and a session controller. The first-party designation does not constrain this feature to the HTTP plugin protocol or a webview. + +**Scope and sequence** + +The first usable milestone is one Codevisor Mac viewing and controlling an existing display on another Codevisor Mac over the LAN. Use Apple silicon Macs as the initial performance reference. The host must have the native app running in the logged-in session and the necessary macOS permissions. The workspace's machine is the host; this version does not introduce an arbitrary-host connection manager. + +The initial beta supports one active media viewer per host. A second viewer gets an explicit in-use state and cannot displace the first. Persisting the same pane on several devices does not automatically attach every device. Multiple simultaneous viewers can follow once resource sharing and control handoff are implemented. + +Next come internet connectivity and native iOS viewing/control, followed by audio and additional fidelity work. Virtual displays, independent login sessions, access before login, Apple VNC interoperability, file transfer, and non-macOS hosts are separate follow-up scope. HEVC and 4:4:4 are capability investigations; they are not prerequisites for the first H.264-based milestone. + +**Architecture** + +```mermaid +flowchart LR + HC[Host ScreenCaptureKit] --> HE[VideoToolbox encoder] + HE --> HT[Native WebRTC sender] + HT -->|Encrypted media| VR[Native WebRTC receiver] + VR --> VD[VideoToolbox decoder] + VD --> VM[Metal surface] + VI[Native viewer input] -->|Data channels| HI[Host input service] + VA[Viewer session controller] <-->|Authenticated signaling| CS[Codevisor server] + CS <-->|Local authenticated IPC| HS[Native host service] + HS --> HC + HS --> HI +``` + +The encoder and decoder integrate through WebRTC's native codec interfaces; the diagram describes ownership, not a second encoding pass. Frames stay in native memory. SwiftUI observes connection state and renders controls, while the video surface consumes decoded frames directly. + +| Owner | Responsibility | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| New `CodevisorScreenSharing` Swift target | Versioned native messages, viewer/session state, media interfaces, VideoToolbox adapters, Metal surface, metrics. No dependency on application pane classes. | +| `CodevisorCoreMac` | Host capture, permissions, display enumeration, input injection, native host service and its lifecycle. | +| `CodevisorCore` and `CodevisorClient` | Pane identity, workspace synchronization, authenticated capability/session requests, signaling integration. | +| macOS app | `ScreenSharingPane`, `NSView` responder behavior, commands, focus and pane lifecycle. | +| iOS app, in a later milestone | Native `UIView` surface, touch mapping, hardware/software keyboard handling and background lifecycle. | +| TypeScript server | Capability discovery, session authorization and signaling proxy to the native host. Existing workspace persistence. Video does not traverse its JSON tool bridge. | + +Use native WebRTC as the proposed media transport, with encrypted RTP media and data channels. Existing Codevisor HTTP/WebSocket connections carry signaling, including when signaling is relayed. Bind the negotiated peer fingerprint and session generation to that authenticated exchange. The existing cloud relay and direct LAN connection are WebSocket-based; neither is a TURN server or a UDP media path. + +Milestone 0 must establish the native WebRTC dependency: pinned upstream revision, macOS/device/simulator slices required by our builds, repeatable artifact build, checksums, notices, signing integration, and a supported codec adapter path. Reuse existing dependency-artifact conventions. Do not select an unversioned binary or design around an untested codec hook. A material transport change after this investigation requires an updated design record, not an implicit substitution with TCP video. + +**What the repository already provides** + +| Existing code | Reuse and required change | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [PaneGroupState.swift](../../packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneGroupState.swift), [Pane.swift](../../apps/macos/Codevisor/Features/Panes/Pane.swift), [TerminalPane.swift](../../apps/macos/Codevisor/Features/Panes/TerminalPane.swift) | Add `.screenSharing` and use the existing creation, focus, visibility, detach and delete hooks. Preserve expensive session state across view reconstruction and tab moves. | +| [WorkspaceSyncModel+ServerRecords.swift](../../packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+ServerRecords.swift), [workspaces.ts](../../packages/api/src/workspaces.ts) | Add native pane serialization and versioned metadata. The registry already accepts provider/type strings and opaque metadata; a database schema change is not expected for pane identity. | +| [ComputerUseRecording.swift](../../packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseRecording.swift), [ComputerUseVideoWriter.swift](../../packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseVideoWriter.swift) | Reuse capture/filter/permission knowledge. Add a live encoder; the current MP4 writer and recording duration limits are not a live-stream interface. | +| [ComputerUseBridge+Keyboard.swift](../../packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Keyboard.swift), [ComputerUseBridge+Mouse.swift](../../packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Mouse.swift) | Reuse applicable key/coordinate primitives. Add continuous input state and delivery; current automation actions contain waits and target individual apps/windows. | +| [LocalCodevisorServer.swift](../../packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/LocalCodevisorServer.swift) | Own host-service startup and shutdown alongside the native bridge. A running standalone server must report unavailable when the native host is absent. | +| [CloudDirectConnection.swift](../../packages/swift/CodevisorCloud/Sources/CodevisorCloud/CloudDirectConnection.swift), [CloudChannelTransport.swift](../../packages/swift/CodevisorCloud/Sources/CodevisorCloud/CloudChannelTransport.swift) | Reuse authenticated machine routing and signaling infrastructure. Add media connectivity separately. | + +**Pane and session contract** + +- Publish `providerId: "codevisor"`, `paneType: "screen-sharing"`. Store metadata with `schemaVersion: 1`, an optional preferred display identity, and fit/actual-size preference. The workspace supplies machine identity. Resolve a persistent display identity against current displays; never silently redirect input to a different display when it disappears. +- Keep the shared pane descriptor separate from device-local visibility/focus and ephemeral viewer/session identifiers. Never persist SDP, credentials, control leases, or live frames in the workspace registry. Disable durable screen thumbnails for this pane initially. +- Add capability discovery and a typed, versioned session-signaling channel under the existing authenticated machine API. Requests identify the workspace, pane and connection-local viewer. The native host validates display selection, codec limits and view/control entitlement before starting capture or accepting input. +- States: idle, connecting, viewing, controlling, suspended, reconnecting, unavailable and failed. Only a live, focused viewer with an active control lease can inject input. Use connection generations to reject late callbacks and messages from previous sessions. +- Creating/restoring an inactive pane does not capture. Selecting it can establish viewing. Control is an explicit mode. Hiding the pane releases held input and the control lease immediately, pauses media and retains session identity. Showing it resumes viewing with a fresh decodable frame; control is reacquired explicitly. +- Moving a tab or rebuilding its view does not destroy the session. Closing/detaching cancels pending work and stops its stream. Host capture stops when there is no active viewer. The host exposes a native stop action; a stop or permission revocation releases all injected keys/buttons. +- Network loss, screen lock, display removal, app shutdown and permission loss suspend input immediately. On reconnection, reacquire capability/display state and negotiate a fresh generation before resuming viewing. No queued input is replayed. +- Newer clients must round-trip the descriptor; older clients must ignore the unsupported pane without deleting its server record or losing sibling tabs. Compile both apps when adding the enum case, with an explicit unavailable presentation on iOS until its renderer lands. + +**Input and media behavior** + +Use reliable ordered delivery for key/button transitions, control ownership and release-all messages. Coalesce expendable pointer-motion updates separately. Button events carry their own coordinates and display generation so an unordered motion update cannot change a click's target. Scroll events carry sequence/cumulative information so coalescing does not lose movement. Track every held key/button on the host and release it on disconnect or lease expiration. + +Map video content coordinates through letterboxing, source pixel dimensions, display scale, rotation and global display origin, including negative origins. Separate physical key events from committed Unicode text. Define a visible local/remote shortcut mode and an always-local escape command before wiring application command handling. OS-reserved shortcuts require explicit testing; native input does not imply every shortcut is interceptable. + +Use bounded capture/encode/render queues and low-delay codec settings. Drop stale raw frames before encoding, and recover decoder reference state through the transport/codec layer rather than dropping arbitrary compressed packets. Apply encoder bitrate changes from congestion feedback and reduce resolution/frame rate when necessary. Keep color-space/range metadata end to end; render at backing-pixel resolution for text clarity. + +Investigate cursor shape, hotspot and position updates during Milestone 0. Use a local cursor overlay only when synchronization is correct. Until that is proven, capture the host cursor and suppress duplicate overlays; this is an explicit prototype limitation, not a claim of final native fidelity. + +**Implementation milestones** + +| Milestone | Concrete deliverable | Exit criterion | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **0. Prove the native media path** | Add the isolated media target and reproducible WebRTC dependency. Feed synthetic and live display frames through capture → encoder → transport → decoder → Metal. Probe H.264/HEVC hardware use, profile/pixel-format support, cursor access and native input feasibility. | Repeatable two-Mac LAN demonstration with codec, format, hardware/software path and latency results recorded. H.264 provides a working baseline. Resolve the transport adapter and cursor approach before the interactive beta. | +| **1. Native viewing tab** | Add `.screenSharing`, metadata and sync mappings, New Tab entry, native host capabilities/signaling, viewer state machine, display selection, fit/actual-size, connection status and native surface. Keep feature exposure gated until the slice works. | Open a tab on Mac A in Mac B's workspace and see B's selected display. Switching/splitting/moving/restoring tabs preserves identity; hidden tabs stop capture work; closing cleans up. Older servers and absent native hosts show an actionable unavailable state. | +| **2. Interactive LAN beta** | Add control mode, continuous input, shortcut routing, Retina/multiple-display coordinate handling, cursor behavior, basic text clipboard, lease recovery and host stop control. | Complete real editing, terminal and browser interactions. No stuck keys/buttons after loss of focus, disconnect, lock, host stop or display change. Meet the measured LAN targets below or document a revised target with the observed bottleneck. | +| **3. Internet connectivity** | Add STUN/TURN configuration and short-lived relay credentials, ICE restart, authenticated peer binding, adaptive quality and diagnostic connection stats. Media remains encrypted through TURN. | Connect across separate networks, including a forced TURN route and UDP-restricted network. Recover after a route change without stale frames/input. Record relay bandwidth and cost assumptions; never put fixed TURN secrets in the client. | +| **4. Native iOS viewer** | Reuse media/session interfaces; add native rendering, explicit pointer/scroll gestures, hardware keyboard and text entry, rotation and background handling. | Physical-device tests against a Mac host verify rendering, input mapping and reconnection. Backgrounding releases control and suspends media. Existing iOS workspace behavior remains intact. | +| **5. Fidelity and expansion** | Evaluate HEVC preference, 4:4:4 alternatives, HDR, synchronized audio, multiple simultaneous viewers/displays and optional virtual-display work. | Promote each capability only after hardware/OS negotiation, text/color fidelity, latency and resource measurements pass. Separate login sessions remain their own architectural investigation. | + +Milestones are sequential at their interfaces. Milestone 0 can be split into dependency/protocol and codec/rendering changes; Milestones 1 and 2 should land as small reviewable changes around a working vertical slice. No staffing or multi-agent parallelism is assumed. + +**Implementation checklist** + +- [x] Native ScreenCaptureKit → VideoToolbox H.264 → WebRTC → hardware decoder → Metal path. +- [x] Native workspace pane, persisted display/size preferences and authenticated signaling. +- [x] Explicit control, bounded input queues, held-input release and always-local release shortcut. +- [x] Explicit plain-text clipboard transfer over an independent bounded channel. +- [x] Supported captured-cursor fallback with duplicate viewer cursor suppression. +- [x] Connection details and text-first adaptive resolution/frame-rate policy. +- [x] Optional STUN/TURN configuration and short-lived host-issued relay credentials. +- [x] Fresh media/ICE/DTLS generation for interrupted sessions, bound to the existing host lease. +- [x] Thirty-minute local media endurance run, live quality transitions and authenticated host recovery checks recorded in the [measurement report](../measurements/native-screen-sharing-2026-09-10/README.md). +- [x] Correct physical-presentation accounting; per-second WebRTC delay metrics and bounded renderer scheduling experiment. +- [x] Public-API hardware HEVC 4:4:4 codec probes on M1 Pro and M4 Max, plus bidirectional native media experiments; see the [tuftlord report](../measurements/native-screen-sharing-tuftlord-2026-09-10/README.md). +- [x] Tuftlord Screen Recording grant verified; real-desktop 1080p/4K capture runs and a shared Apple/native workload recorded, with explicit network and presentation limits. +- [x] Exploratory matched-scale 4K Apple High Performance / Codevisor comparison: two recordings per viewer, a local recording control, frame-code analysis and stage diagnostics; see the [September 11 report](../measurements/native-screen-sharing-apple-2026-09-11/README.md). +- [x] Bounded capture/encoder/decoder cadence diagnostics and a probe-only VT rate-control comparison; see the [timing investigation](../measurements/native-screen-sharing-timing-2026-09-11/README.md). Product rate-control and renderer defaults remain unchanged. +- [x] Correct the synthetic input-format description and compare explicit BGRA/NV12 sources across both rate controllers; see the [pixel-format investigation](../measurements/native-screen-sharing-pixel-format-2026-09-11/README.md). Isolated colored-text and full media results favor different modes, so desktop validation remains required. +- [x] Explicit native HEVC/Main444 transport and NV24 rendering in the probe, Release builds, scoped capture picker and bounded pipeline experiments; see the [pipeline report](../measurements/native-screen-sharing-pipeline-2026-09-11/README.md). Performance parity and product codec promotion remain open. +- [x] Keep requested recovery keyframes pending across encoder admission drops; explicit decoder-state-loss checks pass for H.264 and Main444, including 4K encoder backpressure. This is local fault injection, not loss acceptance on a real network. +- [x] Pinned source-build recipe and manual artifact workflow prepared; published dependency checksum/slices verified. +- [ ] Complete the pinned source build with Xcode 26.5, audit generated third-party notices and validate the candidate binary before promotion. +- [ ] Run the specified two-Mac LAN editing/terminal/browser and input-to-photon checks, including rotated/external displays and non-US input methods. +- [ ] Validate separate networks, forced TURN, UDP restrictions and route changes against a deployed relay. +- [ ] Complete isolated wired-LAN and 30-minute resource/latency comparisons, matched 1080p Apple runs, quantitative text fidelity and external input-to-photon measurements. +- [ ] Native iOS viewer/control and physical-device validation (Milestone 4). +- [ ] HEVC, audio, HDR and other fidelity expansions (Milestone 5). + +The recovery implementation replaces the whole native peer while retaining host authorization, instead of invoking ICE restart inside an existing peer. This deliberately discards queued media/input and renegotiates fresh codec state. Host stop and the original lease deadline apply across asynchronous teardown and display enumeration. + +**Measurement and validation** + +Initial engineering targets for Milestone 2, to validate in Milestone 0: two specified Apple silicon Macs on wired gigabit LAN, SDR 1920×1080 at a requested 60 fps, one viewer, continuously changing content, warmed hardware codecs. Aim for at least 55 presented frames/second and p95 input-to-visible-response at or below 100 ms. These are proposed acceptance targets, not current measurements or a promise of Apple parity. Record a separate 4K/text workload without claiming the same throughput. + +Instrument capture, encode, send, receive, decode and presentation timestamps, queue depth, dropped frames, codec/profile/pixel format, hardware use, bitrate, CPU/GPU utilization and memory. Keep timestamps monotonic within each process. Do not subtract unrelated host/viewer clocks: use an external input-to-photon recording or a calibrated method with stated clock error. Run the identical workload in Apple's High Performance mode for comparison. A 30-minute run must show bounded queues, stable resource usage and no accumulating latency. + +Correctness tests use the existing `TestClock`/`TestSignal` fixtures and injected media/input transports. Cover pane metadata round-trips and mixed versions, authorization and session generations, out-of-order callbacks, bounded queues, coordinate transforms, held-input release, visibility, cancellation, reconnects and idempotent teardown. Hardware capture/codec and real-socket tests stay in focused integration suites. Permission prompts and performance comparisons do not run in ordinary unit tests. Follow [deterministic-tests](../../.agents/skills/deterministic-tests/SKILL.md). + +Each implementation change runs affected Swift and TypeScript tests/typechecks and relevant formatting/lint checks. Build both native apps when shared pane or media interfaces change using `bun run build:macos` and `bun run build:ios`, following [build-codevisor](../../.agents/skills/build-codevisor/SKILL.md). Use the run-dev and ios-development workflows for interactive validation when those milestones are implemented. The initial implementation has passed both native app builds and the focused media tests; see the implementation README for current validation and limits. + +**Technical decisions to resolve in Milestone 0** + +1. Can the pinned native WebRTC build expose the required pixel buffers and low-delay VideoToolbox controls without a large fork? Record the chosen adapter and its maintenance/build footprint. +2. Which codec/profile/format combinations actually use hardware on the supported machines? Establish H.264 first; native access alone does not establish HEVC 4:4:4 support. +3. How are cursor shape/position and remote keyboard shortcuts obtained reliably with the chosen APIs? Record limitations and the prototype fallback. +4. What dominates observed latency: capture, codec buffering, transport, decode or presentation? Optimize the measured stage and confirm text quality before adding resolution or features. + +The next Milestone 0 work is capture/encoder pacing and receive-buffer profiling, the remaining controlled comparisons, and dependency-build checks. Tuftlord is available over authorized SSH, and exploratory native media runs in both directions are recorded. For the original recording comparison, Apple Screen Sharing's connection settings confirmed High Performance with one virtual display at 1920×1080 points / 4K backing pixels and 60 Hz. Local `replayd` descriptor exhaustion was diagnosed and capture recovered. The [September 11 comparison](../measurements/native-screen-sharing-apple-2026-09-11/README.md) now records approximately 53.9 sampled updates/s for Apple versus 38.9–39.7 for Codevisor, using matching 4K source and viewer content size. A local recording control observed 57.4 updates/s at a requested 60 fps. These are recording observations, not physical presentation or input-to-photon measurements. Current runs used a Wi-Fi/Ethernet environment with Tailscale active, and Apple continued streaming during Codevisor trials to supply the virtual display; the specified isolated wired-LAN setup remains open. + +The [timing follow-up](../measurements/native-screen-sharing-timing-2026-09-11/README.md) adds stage cadence, VT submit-call timing and encoder-drop diagnostics. A standard-rate-control experiment improved local synthetic 4K presentation from 43.7–44.6 to 48.1–48.3 fps, without a consistent receive-buffer reduction. It remains probe-only. Tuftlord's attempted follow-up was invalidated by a protected keychain dialog and severe competing load; the dialog later cleared but the load remained high. Resume valid desktop/bitrate trials on a quiet host before promoting a rate-control or transport change. + +That initial local source was BGRA, incorrectly described as NV12 in the first timing report. The [pixel-format follow-up](../measurements/native-screen-sharing-pixel-format-2026-09-11/README.md) corrects the report and adds measured preconversion before WebRTC. In repeated local 4K media runs, NV12/standard reaches 48.7–52.4 fps versus 44.1–44.9 for NV12/low-latency. The isolated colored-text codec test instead favors NV12/low-latency, encoding all 180 opportunities versus 122–135 for NV12/standard. Neither workload reproduces the earlier desktop receive buffering. Production already captures NV12 and retains low-latency rate control. Tuftlord remained heavily loaded at the final check, so the instrumented real-desktop comparison is still open. + +The measured architecture decision is to retain WebRTC as the current baseline, investigate real-capture pacing/receive buffering alongside HEVC 4:4:4 integration and presentation scheduling, and only pursue a replacement transport when measurements establish a material limit. Frame arrival improved the revised desktop workload to 56.58 presented fps with 29.57 ms renderer p95, versus 52.14 fps / 35.24 ms for display-link scheduling. Earlier synthetic experiments worsened p95 and the network conditions varied, so it remains probe-only. The initial real-desktop 4K run reached 37.03 fps, with a 93.66 ms average of sampled receive-buffer interval means. These are not input-to-photon measurements or isolated Apple comparisons. The later probe now negotiates HEVC/Main444 through the pinned transport and validates full-chroma decode, but 4K60 product performance and Apple parity remain unproven. The Apple session subsequently disconnected, then reconnected through the verified local hostname without entering credentials. High Performance with one virtual display was selected; the restored display was verified as 4K/60 Hz. Current follow-up work uses a shared synthetic text source to separate capture from encoding, transport and presentation; see the [pipeline report](../measurements/native-screen-sharing-pipeline-2026-09-11/README.md). + +The next isolation test reproduced the desktop Main444 slowdown with a synthetic source while Apple streamed a separate visible workload: 20.45 fps and 124.16 ms median VT completion. After closing Apple, the first interval reached 56.95 fps and 39.20 ms median completion, but later packet loss and ICE route changes invalidated a clean full-run comparison. Restricting exchanged candidates to verified LAN interfaces is now a labeled diagnostic option. Apple was deliberately disconnected for this control; its virtual display disappeared. Match source/display geometry again before a fresh Apple comparison. Current product codec, keyframe interval and renderer defaults remain unchanged. + +A subsequent [separate-stream comparison](../measurements/native-screen-sharing-independent-2026-09-11/README.md) removes concurrent Apple encoding during Codevisor capture. The Main444 candidate recorded 50.23/47.30 distinct updates per second; fresh Apple High Performance samples recorded 49.89/49.84. Both used the same 4K workload and local viewer content size, but Codevisor captured a window while Apple streamed a virtual display. Codevisor had longer worst-case update gaps (70/158 ms versus 47/42 ms). Average update rate is close in this setup; smoothness, interaction latency, text fidelity and sustained behavior still need validation. A subsequent 72 Hz display-link experiment reduced presentation cadence and remains diagnostic. A two-drawable trial encountered a seven-second media interruption with a selected ICE-pair change, so it does not isolate renderer behavior. The subsequent calibrated series joins visible marker codes to exact source draw timestamps using bracketing clock calibrations, with a local capture control and explicit clock uncertainty. + +The first [calibrated image-age samples](../measurements/native-screen-sharing-independent-2026-09-11/image-age/README.md) now measure source AppKit draw start to a local capture of the viewer: Codevisor median/p95 is 135.46/166.98 ms, versus Apple High Performance at 130.79/141.42 ms, with clock bounds of approximately ±1.90/±2.22 ms. Both used 960×540-pixel content on the same secondary 60 Hz display. Codevisor's longer tail and update gaps remain the measured gap; this is not physical input-to-photon or controlled sustained parity. Exact source timestamps, numeric window observation, clock responders and explicit probe monitor placement are implemented and validated without changing product defaults. + +The following zero-delay WebRTC playout trial reduced median image age to 109.41 ms, but p95 remained 165.04 ms and sampled updates 52.05/s, versus Apple's earlier 141.42 ms and 57.84/s. Selecting frames after drawable acquisition lowered median further while worsening cadence and long gaps; the original selection order was restored. A probe-only CAMetalDisplayLink driver passed a local recovery control and both native builds. The next comparison uses a composited-display crop to avoid treating independent window updates as proof of visible scanout; physical input-to-photon, fidelity and sustained parity remain open. + +**Primary technical references** + +- [Apple: ScreenCaptureKit](https://developer.apple.com/documentation/screencapturekit) — capture framework and content selection. +- [Apple: low-latency VideoToolbox encoding](https://developer.apple.com/videos/play/wwdc2021/10158/) — hardware H.264 low-delay configuration; do not assume the same settings apply to every HEVC encoder. +- [Apple: Core Video and Metal textures](https://developer.apple.com/documentation/CoreVideo/cvmetaltexturecache-q3j) — GPU access to decoded image buffers. +- [WebRTC: native development](https://webrtc.github.io/webrtc-org/native-code/development/) and [native iOS builds](https://webrtc.googlesource.com/src/+/main/docs/native-code/ios/) — native integration and framework build starting points; verify against the pinned revision. +- [WebRTC: TURN](https://webrtc.org/getting-started/turn-server) — relayed media connectivity. +- [Apple: High Performance mode](https://support.apple.com/en-ie/guide/mac-help/mchl1883115d/mac) — comparison feature set; reproducing its proprietary protocol is outside this plan. + +The [display-crop controls](../measurements/native-screen-sharing-independent-2026-09-11/display-age/README.md) now validate source-draw timestamps and nonzero viewer crop coordinates on the laptop display. Remote comparisons were correctly excluded when the viewer moved offscreen during warmup; they establish no parity result. Background work hardened keyframe requests through VT output failure and added a headless recovery check. The 49 media tests, both native builds, local H.264 recovery and local 4K Main444 recovery pass; see the [recovery record](../measurements/native-screen-sharing-independent-2026-09-11/recovery/README.md). + +Recovery is now verified with one admitted forced output deliberately discarded after decoder reset: local H.264, local 4K Main444 and tuftlord 4K Main444 all recovered with two transport requests and three forced submissions. Recovery times were 32.39, 61.94 and 53.84 ms, respectively. This is an application callback fault, not induced network loss or an internal VT failure. The final 50-test media suite and both native builds pass. + +The [idle recovery investigation](../measurements/native-screen-sharing-independent-2026-09-11/idle-source/README.md) reproduced a static-screen freeze after decoder reset: no replacement frame arrived before the twelve-second run ended. A bounded, separately negotiated refresh data channel now requests a forced keyframe from a one-buffer capture cache. Retries run only while recovery is pending and stop after a valid keyframe decodes. Local H.264 and 4K Main444 idle checks recovered in 9.19/28.22 ms, and tuftlord's 4K Main444 check recovered in 33.14 ms. Discarding the first recovery output still recovered in 119.78 ms locally and 124.22 ms on tuftlord. All checks settled with unchanged encoded-frame and request counts. These are headless functional checks; sustained SCK buffer pressure, actual network loss and visible Apple parity remain open. The media suite passes 57 tests, including fake-clock retry/cancellation and stale-callback ordering. + +Both native app builds and the Release probe pass after the idle-refresh change. Next background gate: inject loss in the actual encrypted UDP media path, including a lost frame followed by source idle, and verify that transport-only feedback can cause a fresh frame. A decoder-reset check alone does not prove that case. diff --git a/package.json b/package.json index 53a546bae..8edea4575 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "release:macos": "scripts/release/build-macos-app.sh", "release:tap": "node scripts/release/update-homebrew-tap.mjs", "setup:github-oauth": "node scripts/setup-github-oauth.mjs", + "screen-sharing:probe": "bun scripts/screen-sharing-probe.mjs", "swift:test": "swift test --package-path packages/swift", "swift:test:macos-transcript": "bun scripts/test-macos-transcript.mjs", "swift:test:macos-composer": "bun scripts/test-macos-composer.mjs", diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 329768daa..6cb24a7dd 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -17,3 +17,5 @@ export * from "./terminals.js" export * from "./openapi.js" export * from "./browser-state.js" + +export * from "./screen-sharing.js" diff --git a/packages/api/src/screen-sharing.ts b/packages/api/src/screen-sharing.ts new file mode 100644 index 000000000..648ea8ec2 --- /dev/null +++ b/packages/api/src/screen-sharing.ts @@ -0,0 +1,41 @@ +import { Schema } from "effect" + +export const ScreenSharingRequest = Schema.Struct({ + version: Schema.Literal(1), + operation: Schema.Literals(["capabilities", "start", "restart", "heartbeat", "stop"]), + workspaceId: Schema.String, + paneId: Schema.String, + viewerId: Schema.String, + displayId: Schema.optional(Schema.String), + offer: Schema.optional(Schema.String) +}) +export type ScreenSharingRequest = typeof ScreenSharingRequest.Type + +export const ScreenSharingReply = Schema.Struct({ + version: Schema.Literal(1), + status: Schema.String, + message: Schema.optional(Schema.String), + displays: Schema.Array( + Schema.Struct({ + id: Schema.String, + name: Schema.String, + width: Schema.Number, + height: Schema.Number + }) + ), + answer: Schema.optional(Schema.String), + connectivity: Schema.optional( + Schema.Struct({ + servers: Schema.Array( + Schema.Struct({ + urls: Schema.Array(Schema.String), + username: Schema.String, + credential: Schema.String + }) + ), + relayOnly: Schema.Boolean, + expiresAt: Schema.Number + }) + ) +}) +export type ScreenSharingReply = typeof ScreenSharingReply.Type diff --git a/packages/automation/src/computer-use-bridge.test.ts b/packages/automation/src/computer-use-bridge.test.ts index 99e776f11..9013412ff 100644 --- a/packages/automation/src/computer-use-bridge.test.ts +++ b/packages/automation/src/computer-use-bridge.test.ts @@ -3,7 +3,11 @@ import { createServer } from "node:net" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, expect, it, vi } from "vitest" -import { macComputerUseSocketPath, makeComputerUseProvider } from "./computer-use-provider.js" +import { + macComputerUseSocketPath, + makeComputerUseProvider, + requestMacScreenSharing +} from "./computer-use-provider.js" describe("macOS Computer Use bridge discovery", () => { it("shares the native socket address across users and Unicode data paths", () => { @@ -62,3 +66,68 @@ describe("macOS Computer Use bridge discovery", () => { } ) }) + +describe("native Screen Sharing bridge", () => { + it.each(["success", "request failure", "authentication failure"])( + "authenticates and closes its one-shot socket on %s", + async (outcome) => { + const dataDir = mkdtempSync(join(tmpdir(), "codevisor-sharing-bridge-")) + const socketPath = macComputerUseSocketPath(dataDir) + const requests: Record[] = [] + const sockets = new Set() + let acknowledgeClose!: () => void + const closed = new Promise((resolve) => { + acknowledgeClose = resolve + }) + const server = createServer((socket) => { + sockets.add(socket) + socket.once("close", () => { + sockets.delete(socket) + acknowledgeClose() + }) + let pending = "" + socket.on("data", (data) => { + pending += data.toString() + let newline: number + while ((newline = pending.indexOf("\n")) >= 0) { + const request = JSON.parse(pending.slice(0, newline)) as Record + pending = pending.slice(newline + 1) + requests.push(request) + const failure = + (outcome === "authentication failure" && request.type === "authenticate") || + (outcome === "request failure" && request.type === "screenSharing") + socket.write( + `${JSON.stringify({ id: request.id, ...(failure ? { error: "fixture failure" } : { result: { version: 1, status: "available", displays: [] } }) })}\n` + ) + } + }) + }) + try { + vi.stubEnv("CODEVISOR_COMPUTER_USE_SOCKET", socketPath) + vi.stubEnv("CODEVISOR_COMPUTER_USE_TOKEN", "fixture-token") + await new Promise((resolve, reject) => { + server.once("error", reject) + server.listen(socketPath, resolve) + }) + const request = { version: 1, operation: "capabilities" } + const operation = requestMacScreenSharing(dataDir, request) + if (outcome === "success") + await expect(operation).resolves.toEqual({ + version: 1, + status: "available", + displays: [] + }) + else await expect(operation).rejects.toThrow("fixture failure") + await closed + expect(requests[0]).toMatchObject({ type: "authenticate", token: "fixture-token" }) + if (outcome === "authentication failure") expect(requests).toHaveLength(1) + else expect(requests[1]).toMatchObject({ type: "screenSharing", request }) + } finally { + for (const socket of sockets) socket.destroy() + await new Promise((resolve) => server.close(() => resolve())) + vi.unstubAllEnvs() + rmSync(dataDir, { recursive: true, force: true }) + } + } + ) +}) diff --git a/packages/automation/src/computer-use-provider.ts b/packages/automation/src/computer-use-provider.ts index 5831ba614..220cb64f8 100644 --- a/packages/automation/src/computer-use-provider.ts +++ b/packages/automation/src/computer-use-provider.ts @@ -146,8 +146,27 @@ const connectMacHelper = async (dataDir: string): Promise => { socket.once("error", onClose) } ) - await client.request({ type: "authenticate", token }) - return client + try { + await client.request({ type: "authenticate", token }) + return client + } catch (cause) { + await client.close() + throw cause + } +} + +// A dedicated native service message, not an agent tool invocation. The server +// supplies the validated request and authenticates this private socket. +export const requestMacScreenSharing = async ( + dataDir: string, + request: Readonly> +): Promise => { + const client = await connectMacHelper(dataDir) + try { + return await client.request({ type: "screenSharing", request }) + } finally { + await client.close() + } } export const linuxComputerUseHelperPath = ( diff --git a/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorMachine.swift b/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorMachine.swift index b833c0559..f1c35cfda 100644 --- a/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorMachine.swift +++ b/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorMachine.swift @@ -32,6 +32,20 @@ public struct CodevisorMachine: Identifiable, Sendable, Codable, Equatable { public var isLocal: Bool { id == Self.local.id } + /// A machine whose record cannot be resolved right now. The identity is the + /// real server id, so nothing is mistaken for another machine, and the + /// address is a reserved name that can never resolve: panes report + /// unavailable instead of silently addressing a different machine — in + /// particular this one, which `CodevisorMachine.local` would do. + public static func unresolved(id: String) -> CodevisorMachine { + CodevisorMachine( + id: id, name: id, baseURL: URL(string: "http://unresolved.invalid")!, kind: "unresolved") + } + + /// True for the placeholder above: its server is not reachable and its panes + /// must not act as though they own local resources. + public var isUnresolved: Bool { kind == "unresolved" } + /// True for machines reached through the Codevisor Cloud relay (their ids /// are `cloud:`; they are synthesized from cloud presence, not /// stored in the registry). diff --git a/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorServerClienting.swift b/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorServerClienting.swift index efdaa3c20..ef0d5e8e1 100644 --- a/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorServerClienting.swift +++ b/packages/swift/CodevisorClient/Sources/CodevisorClient/CodevisorServerClienting.swift @@ -3,6 +3,7 @@ import CodevisorProtocol import Foundation public protocol CodevisorServerClienting: BrowserStateClienting { + func screenSharing(_ request: ServerScreenSharingRequest) async throws -> ServerScreenSharingReply func exchangeBrowserCookies(_ mutations: [BrowserCookieMutation]) async throws -> BrowserCookieSnapshot func browserNavigation(paneId: UUID) async throws -> BrowserNavigation? func publishBrowserNavigation(paneId: UUID, navigation: BrowserNavigation) async throws diff --git a/packages/swift/CodevisorClient/Sources/CodevisorClient/ServerScreenSharingRequest.swift b/packages/swift/CodevisorClient/Sources/CodevisorClient/ServerScreenSharingRequest.swift new file mode 100644 index 000000000..214e2124d --- /dev/null +++ b/packages/swift/CodevisorClient/Sources/CodevisorClient/ServerScreenSharingRequest.swift @@ -0,0 +1,78 @@ +import Foundation + +public struct ServerScreenSharingDisplay: Codable, Sendable, Equatable, Identifiable { + public let id: String + public let name: String + public let width: Int + public let height: Int + public init(id: String, name: String, width: Int, height: Int) { + self.id = id; self.name = name; self.width = width; self.height = height + } +} + +/// Ephemeral signaling only. Never persist this request or SDP in pane metadata. +public struct ServerScreenSharingRequest: Codable, Sendable { + public enum Operation: String, Codable, Sendable { case capabilities, start, restart, heartbeat, stop } + public let version: Int + public let operation: Operation + public let workspaceId: UUID + public let paneId: UUID + public let viewerId: UUID + public var displayId: String? + public var offer: String? + + public init( + operation: Operation, workspaceId: UUID, paneId: UUID, viewerId: UUID, + displayId: String? = nil, offer: String? = nil + ) { + version = 1; self.operation = operation; self.workspaceId = workspaceId + self.paneId = paneId; self.viewerId = viewerId; self.displayId = displayId; self.offer = offer + } +} + +public struct ServerScreenSharingReply: Codable, Sendable { + public let version: Int + public var status: String + public var message: String? + public var displays: [ServerScreenSharingDisplay] + public var answer: String? + public var connectivity: ServerScreenSharingConnectivity? + + public init( + status: String, message: String? = nil, displays: [ServerScreenSharingDisplay] = [], answer: String? = nil, + connectivity: ServerScreenSharingConnectivity? = nil + ) { + version = 1; self.status = status; self.message = message; self.displays = displays; self.answer = answer + self.connectivity = connectivity + } +} + +public struct ServerScreenSharingConnectivity: Codable, Sendable { + public struct Server: Codable, Sendable { + public let urls: [String] + public let username: String + public let credential: String + public init(urls: [String], username: String = "", credential: String = "") { + self.urls = urls; self.username = username; self.credential = credential + } + } + public let servers: [Server] + public let relayOnly: Bool + public let expiresAt: Int + public init(servers: [Server], relayOnly: Bool, expiresAt: Int) { + self.servers = servers; self.relayOnly = relayOnly; self.expiresAt = expiresAt + } +} + +extension CodevisorServerClient { + public func screenSharing(_ request: ServerScreenSharingRequest) async throws -> ServerScreenSharingReply { + try await send("/v1/screen-sharing", method: "POST", body: request) + } +} + +public extension CodevisorServerClienting { + func screenSharing(_ request: ServerScreenSharingRequest) async throws -> ServerScreenSharingReply { + throw CodevisorServerClientError.httpStatus( + 501, "Screen Sharing requires an updated Codevisor app on the host Mac.") + } +} diff --git a/packages/swift/CodevisorClient/Tests/CodevisorClientTests/UnresolvedMachineTests.swift b/packages/swift/CodevisorClient/Tests/CodevisorClientTests/UnresolvedMachineTests.swift new file mode 100644 index 000000000..e1d96ba61 --- /dev/null +++ b/packages/swift/CodevisorClient/Tests/CodevisorClientTests/UnresolvedMachineTests.swift @@ -0,0 +1,30 @@ +import Foundation +import Testing + +@testable import CodevisorClient + +/// A workspace addressed without a chat still names its own machine. When that +/// machine cannot be resolved, its panes must report unavailable rather than +/// quietly addressing a different machine — above all this one, which is what +/// falling back to `CodevisorMachine.local` would do. +@Suite("Unresolved machine") +struct UnresolvedMachineTests { + @Test func anUnresolvedMachineKeepsItsIdentityAndIsNeverLocal() { + let machine = CodevisorMachine.unresolved(id: "stage3v") + + #expect(machine.id == "stage3v") // the real server id, not a substitute + #expect(!machine.isLocal) + #expect(machine.isUnresolved) + #expect(machine.id != CodevisorMachine.local.id) + #expect(machine.baseURL != CodevisorMachine.local.baseURL) + // A reserved name that cannot resolve, so a request fails instead of + // reaching some other server. + #expect(machine.baseURL.host() == "unresolved.invalid") + #expect(machine.token == nil) + } + + @Test func theLocalMachineIsNotMistakenForAnUnresolvedOne() { + #expect(CodevisorMachine.local.isLocal) + #expect(!CodevisorMachine.local.isUnresolved) + } +} diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneGroupState.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneGroupState.swift index f0beaf814..c87a3c809 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneGroupState.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneGroupState.swift @@ -26,6 +26,7 @@ public enum PaneKind: String, Codable, Sendable { /// A read-only Markdown document on the workspace's machine. case document case browser + case screenSharing } /// Which of a session's pane groups a state belongs to: the center group @@ -64,6 +65,7 @@ public struct PaneDescriptorState: Identifiable, Codable, Sendable, Equatable { public var pluginId: String? /// Plugin panes only: which of the plugin's pane types this renders. public var pluginPaneType: String? + public var screenSharing: ScreenSharingPanePreferences? public var browserURL: String? public var documentPath: String? /// Every pane moves between groups alike — tabs are tabs (the only @@ -83,7 +85,8 @@ public struct PaneDescriptorState: Identifiable, Codable, Sendable, Equatable { pluginId: String? = nil, pluginPaneType: String? = nil, documentPath: String? = nil, - browserURL: String? = nil + browserURL: String? = nil, + screenSharing: ScreenSharingPanePreferences? = nil ) { self.id = id self.kind = kind @@ -96,6 +99,7 @@ public struct PaneDescriptorState: Identifiable, Codable, Sendable, Equatable { self.pluginPaneType = pluginPaneType self.documentPath = documentPath self.browserURL = browserURL + self.screenSharing = screenSharing } public init(from decoder: Decoder) throws { @@ -118,7 +122,8 @@ public struct PaneDescriptorState: Identifiable, Codable, Sendable, Equatable { pluginId: try container.decodeIfPresent(String.self, forKey: .pluginId), pluginPaneType: try container.decodeIfPresent(String.self, forKey: .pluginPaneType), documentPath: try container.decodeIfPresent(String.self, forKey: .documentPath), - browserURL: try container.decodeIfPresent(String.self, forKey: .browserURL) + browserURL: try container.decodeIfPresent(String.self, forKey: .browserURL), + screenSharing: try container.decodeIfPresent(ScreenSharingPanePreferences.self, forKey: .screenSharing) ) } } @@ -184,6 +189,15 @@ public struct PaneGroupState: Codable, Sendable, Equatable { return PaneGroupState(panes: [pane], selectedPaneId: pane.id) } + /// The state a center group starts with when the workspace has no chat to + /// bind: the same New Tab placeholder the user gets from `addNewTabPane`, + /// so nothing here invents a chat or a session-scoped key. + public static func centerInitialWithoutChat() -> PaneGroupState { + var state = PaneGroupState() + state.addNewTabPane() + return state + } + /// The state a session's center group starts with: the chat pane, bound /// to its session (an unbound chat pane is a DRAFT — see addChatPane), /// always visible (the center group has no collapsed state — it IS the @@ -253,6 +267,10 @@ public struct PaneGroupState: Codable, Sendable, Equatable { /// Appends a new terminal pane named "Terminal N" (N = highest existing /// numeric suffix + 1), selects it, and opens the group. The shell spawns /// in the workspace's working directory (the anchor session's cwd). + /// Requires a session identity: the terminal key namespaces the server's + /// live PTY per session, so a substitute namespace would either collide with + /// a real session or orphan the shell. A group without a session never calls + /// this (see `convertNewTabPane`, which declines the terminal kind). @discardableResult public mutating func addTerminalPane(sessionId: UUID) -> PaneDescriptorState { let paneId = UUID() @@ -346,7 +364,7 @@ public struct PaneGroupState: Codable, Sendable, Equatable { public mutating func convertNewTabPane( id: UUID, to kind: PaneKind, - sessionId: UUID, + sessionId: UUID?, chatSessionId: UUID? = nil, name: String? = nil, pluginId: String? = nil, @@ -359,6 +377,8 @@ public struct PaneGroupState: Codable, Sendable, Equatable { let pane: PaneDescriptorState switch kind { case .terminal: + // Same rule as `addTerminalPane`: no session identity, no terminal. + guard let sessionId else { return nil } pane = PaneDescriptorState( id: paneId, kind: .terminal, @@ -385,6 +405,10 @@ public struct PaneGroupState: Codable, Sendable, Equatable { pane = PaneDescriptorState( id: paneId, kind: .browser, name: "Browser", terminalKey: paneId.uuidString, browserURL: "https://www.google.com/") + case .screenSharing: + pane = PaneDescriptorState( + id: paneId, kind: .screenSharing, name: "Screen Sharing", + terminalKey: paneId.uuidString, screenSharing: ScreenSharingPanePreferences()) case .newTab, .document: return nil } diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneWorkingDirectory.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneWorkingDirectory.swift new file mode 100644 index 000000000..960221359 --- /dev/null +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/PaneWorkingDirectory.swift @@ -0,0 +1,29 @@ +import Foundation + +/// Where a pane's work happens. A chat anchors on its own working directory, +/// exactly as it always has; a workspace with no chat anchors on the working +/// directory the workspace itself owns. Neither case silently reinterprets the +/// other's directory as the project root. +public enum PaneWorkingDirectory { + /// What the pane group is anchored on. + public enum Anchor: Equatable, Sendable { + /// A chat mount: the session's cwd (worktree sessions open in the worktree), + /// else the project folder — unchanged behaviour. + case session(cwd: String?) + /// No chat: the workspace's own working directory, else the project folder. + case workspace + } + + public static func resolve( + anchor: Anchor, + workspaceRootDirectory: String?, + projectFolderPath: String + ) -> String { + switch anchor { + case let .session(cwd): + return cwd ?? projectFolderPath + case .workspace: + return workspaceRootDirectory ?? projectFolderPath + } + } +} diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/ScreenSharingPanePreferences.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/ScreenSharingPanePreferences.swift new file mode 100644 index 000000000..2a9fc7295 --- /dev/null +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/ScreenSharingPanePreferences.swift @@ -0,0 +1,14 @@ +import Foundation + +/// Shared presentation preferences. Session credentials and SDP never belong here. +public struct ScreenSharingPanePreferences: Codable, Equatable, Sendable { + public var schemaVersion: Int + public var preferredDisplayId: String? + public var fitToWindow: Bool + + public init(preferredDisplayId: String? = nil, fitToWindow: Bool = true) { + schemaVersion = 1 + self.preferredDisplayId = preferredDisplayId + self.fitToWindow = fitToWindow + } +} diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/Workspace+Navigation.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/Workspace+Navigation.swift index 6cd088cf0..cd3d35e34 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/Workspace+Navigation.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Panes/Workspace+Navigation.swift @@ -9,6 +9,49 @@ public enum WorkspaceDestination: Hashable, Sendable { case pane(UUID) } +/// Where the app's sidebar selection should land after a workspace destination +/// is activated. `selectDestination` has already moved layout selection; this +/// decides only what the detail area shows. +public enum WorkspaceSelectionRoute: Hashable, Sendable { + /// A real chat session of this workspace. + case session(serverId: String, id: UUID) + /// The workspace itself. A workspace owns its server identity, project and + /// layout independently of any chat, so one that has never hosted a chat is + /// still a place the user can be. + case workspace(serverId: String, id: UUID) +} + +extension Workspace { + /// Resolves what the selection should become when a tab or leaf of this + /// workspace is activated. Returns nil to leave the current selection alone. + /// + /// - Parameters: + /// - activatedChatSessionId: the chat owned by the activated destination, + /// or nil when it hosts none (a New Tab, terminal or screen-sharing pane). + /// - routingSessionId: a session already routed to this workspace, used as + /// today's fallback when the activated destination has no chat of its own. + /// - selectionAlreadyRoutesWorkspace: the current selection is a chat that + /// lives in this workspace, so the detail area already shows it. + public func selectionRoute( + activatedChatSessionId: UUID?, + routingSessionId: UUID?, + selectionAlreadyRoutesWorkspace: Bool + ) -> WorkspaceSelectionRoute? { + // An actual chat always wins: unchanged behaviour for every workspace + // that has one. + if let activatedChatSessionId { + return .session(serverId: serverId, id: activatedChatSessionId) + } + if selectionAlreadyRoutesWorkspace { return nil } + if let routingSessionId { + return .session(serverId: serverId, id: routingSessionId) + } + // No chat anywhere in this workspace: address the workspace itself rather + // than leaving the selection pointing at some other destination. + return .workspace(serverId: serverId, id: id) + } +} + extension Workspace { /// Select all levels together so the first destination render, toolbar, /// and sidebar agree before any pane begins loading. Invalid intents leave diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/PaneGroupRepository.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/PaneGroupRepository.swift index 5c9e3321c..bf877496d 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/PaneGroupRepository.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/PaneGroupRepository.swift @@ -5,9 +5,13 @@ import Foundation /// identity MUST survive app restarts: the codevisor server keeps one live PTY /// per pane key with no reaping, so stable keys are what let terminals /// reattach instead of orphaning shells. +/// `sessionId` is the SESSION-SCOPED key. It is nil for a group whose identity +/// comes from its workspace instead of a chat (a workspace that has never +/// hosted one). Session-keyed stores have no key to use then and decline; +/// workspace-keyed stores ignore the parameter entirely. public protocol PaneGroupRepository: Sendable { - func load(sessionId: UUID, placement: PaneGroupPlacement) -> PaneGroupState? - func save(_ state: PaneGroupState, sessionId: UUID, placement: PaneGroupPlacement) + func load(sessionId: UUID?, placement: PaneGroupPlacement) -> PaneGroupState? + func save(_ state: PaneGroupState, sessionId: UUID?, placement: PaneGroupPlacement) func removeAll() } @@ -34,11 +38,17 @@ public final class DefaultPaneGroupRepository: PaneGroupRepository, @unchecked S self.store = store } - public func load(sessionId: UUID, placement: PaneGroupPlacement) -> PaneGroupState? { - loadAll()[Self.storageKey(sessionId: sessionId, placement: placement)] + public func load(sessionId: UUID?, placement: PaneGroupPlacement) -> PaneGroupState? { + // No session key, no legacy entry: this store only ever held per-session + // states, and inventing a key here would collide with a real session's. + guard let sessionId else { return nil } + return loadAll()[Self.storageKey(sessionId: sessionId, placement: placement)] } - public func save(_ state: PaneGroupState, sessionId: UUID, placement: PaneGroupPlacement) { + public func save(_ state: PaneGroupState, sessionId: UUID?, placement: PaneGroupPlacement) { + // Same reason as `load`: without a session key there is no entry this + // store owns, and a substitute key would masquerade as a session. + guard let sessionId else { return } var all = loadAll() all[Self.storageKey(sessionId: sessionId, placement: placement)] = state lock.withLock { cache = all } diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/WorkspaceRepository.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/WorkspaceRepository.swift index 359b63ff1..14a295fce 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/WorkspaceRepository.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/Persistence/WorkspaceRepository.swift @@ -385,7 +385,10 @@ public final class WorkspacePaneGroupRepository: PaneGroupRepository, @unchecked self.repository = repository } - public func load(sessionId: UUID, placement: PaneGroupPlacement) -> PaneGroupState? { + /// The session key is deliberately unused: this repository is keyed by + /// workspace and leaf, so a workspace with no chat persists exactly like one + /// that has several. + public func load(sessionId: UUID?, placement: PaneGroupPlacement) -> PaneGroupState? { guard let workspace = repository.workspace(id: workspaceId) else { return nil } switch placement { case .bottom: @@ -396,7 +399,7 @@ public final class WorkspacePaneGroupRepository: PaneGroupRepository, @unchecked } } - public func save(_ state: PaneGroupState, sessionId: UUID, placement: PaneGroupPlacement) { + public func save(_ state: PaneGroupState, sessionId: UUID?, placement: PaneGroupPlacement) { guard var workspace = repository.workspace(id: workspaceId) else { return } switch placement { case .bottom: diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+Layout.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+Layout.swift index 039711e11..d55cc7472 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+Layout.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+Layout.swift @@ -60,7 +60,7 @@ extension WorkspaceSyncModel { pane.chatSessionId.map { "session:\($0.uuidString.lowercased())" } case .terminal: "terminal:\(pane.terminalKey.lowercased())" - case .newTab, .plugin, .document, .browser: + case .newTab, .plugin, .document, .browser, .screenSharing: nil } } diff --git a/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+ServerRecords.swift b/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+ServerRecords.swift index 197579500..8b302d998 100644 --- a/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+ServerRecords.swift +++ b/packages/swift/CodevisorCore/Sources/CodevisorCore/ViewModels/WorkspaceSyncModel+ServerRecords.swift @@ -61,6 +61,13 @@ extension WorkspaceSyncModel { if let data = try? JSONEncoder().encode(BrowserPaneMetadata(url: pane.browserURL)) { metadata = String(data: data, encoding: .utf8) } + case .screenSharing: + paneType = "screen-sharing" + resourceKind = nil + resourceId = nil + if let data = try? JSONEncoder().encode(pane.screenSharing ?? ScreenSharingPanePreferences()) { + metadata = String(data: data, encoding: .utf8) + } case .document: paneType = "markdown" resourceKind = "file" @@ -112,6 +119,14 @@ extension WorkspaceSyncModel { // forward-compatible; renderer support is a client capability. guard record.providerId == "codevisor" else { return nil } switch record.paneType { + case "screen-sharing": + guard let data = record.metadata?.data(using: .utf8), + let preferences = try? JSONDecoder().decode(ScreenSharingPanePreferences.self, from: data), + preferences.schemaVersion == 1 + else { return nil } + return PaneDescriptorState( + id: id, kind: .screenSharing, name: record.title, + terminalKey: id.uuidString, screenSharing: preferences) case "browser": let value = record.metadata.flatMap { $0.data(using: .utf8) } .flatMap { try? JSONDecoder().decode(BrowserPaneMetadata.self, from: $0) } diff --git a/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ChatlessWorkspacePaneTests.swift b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ChatlessWorkspacePaneTests.swift new file mode 100644 index 000000000..2a1b6afd7 --- /dev/null +++ b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ChatlessWorkspacePaneTests.swift @@ -0,0 +1,193 @@ +import Foundation +import Testing + +@testable import CodevisorCore + +/// A workspace can exist before any chat does. These pin the pane-layer +/// contracts that makes such a workspace usable without inventing a session: +/// its layout persists by workspace identity, its first center group is the +/// ordinary New Tab page, and anything that needs a real session identity +/// declines instead of fabricating one. +@Suite("Chatless workspace panes") +struct ChatlessWorkspacePaneTests { + private func workspace(centerTree: SplitNode) -> Workspace { + Workspace( + name: "Verification", + rootDirectory: "/tmp/project", + serverId: "stage3v", + projectId: UUID(), + centerTree: centerTree, + bottomGroup: PaneGroupState() + ) + } + + @Test func aCenterGroupWithoutAChatStartsOnTheNewTabPage() { + let state = PaneGroupState.centerInitialWithoutChat() + + #expect(state.panes.count == 1) + #expect(state.panes[0].kind == .newTab) + #expect(state.panes[0].chatSessionId == nil) + #expect(state.selectedPaneId == state.panes[0].id) + #expect(state.isVisible) + // The placeholder keys on its own pane id, never on a session. + #expect(state.panes[0].terminalKey == state.panes[0].id.uuidString) + } + + @Test func workspaceKeyedPanesPersistWithoutASessionKey() { + let repository = DefaultWorkspaceRepository(store: InMemoryStore()) + var space = workspace(centerTree: .leaf(PaneGroupState.centerInitialWithoutChat())) + let leafId = space.centerTree.allGroups[0].id + repository.save(space) + + let bridge = WorkspacePaneGroupRepository( + workspaceId: space.id, groupId: leafId, repository: repository) + + // Loading with no session key still finds the workspace's own leaf state. + let loaded = bridge.load(sessionId: nil, placement: .center) + #expect(loaded?.panes.first?.kind == .newTab) + + var updated = try! #require(loaded) + updated.addNewTabPane() + bridge.save(updated, sessionId: nil, placement: .center) + + let reloaded = bridge.load(sessionId: nil, placement: .center) + #expect(reloaded?.panes.count == 2) + space = try! #require(repository.workspace(id: space.id)) + #expect(space.centerTree.group(id: leafId)?.panes.count == 2) + } + + @Test func theSessionKeyedStoreDeclinesAGroupWithoutASession() { + let legacy = DefaultPaneGroupRepository(store: InMemoryStore()) + var state = PaneGroupState() + state.addNewTabPane() + + // Saving without a key is a no-op rather than a substitute key that could + // collide with a real session's legacy entry. + legacy.save(state, sessionId: nil, placement: .center) + #expect(legacy.load(sessionId: nil, placement: .center) == nil) + + let session = UUID() + legacy.save(state, sessionId: session, placement: .center) + #expect(legacy.load(sessionId: session, placement: .center) == state) + #expect(legacy.load(sessionId: nil, placement: .center) == nil) + } + + @Test func convertingAPlaceholderWithoutASessionRefusesOnlyTheTerminal() { + var state = PaneGroupState.centerInitialWithoutChat() + let paneId = state.panes[0].id + + #expect(state.convertNewTabPane(id: paneId, to: .terminal, sessionId: nil) == nil) + #expect(state.panes[0].kind == .newTab) // the placeholder survives untouched + + let converted = state.convertNewTabPane(id: paneId, to: .screenSharing, sessionId: nil) + #expect(converted?.kind == .screenSharing) + #expect(converted?.terminalKey == paneId.uuidString) + #expect(converted?.chatSessionId == nil) + } + + @Test func aTerminalStillConvertsWhenARealSessionIsPresent() { + var state = PaneGroupState.centerInitialWithoutChat() + let paneId = state.panes[0].id + let session = UUID() + + let converted = state.convertNewTabPane(id: paneId, to: .terminal, sessionId: session) + #expect(converted?.kind == .terminal) + #expect(converted?.terminalKey == "\(session.uuidString):\(paneId.uuidString)") + } + + /// End to end over the real models: a workspace authored on the server with + /// one New Tab pane and no chat reconciles into a usable layout, its leaf is + /// selectable, and activating it addresses the workspace itself. + @Test @MainActor func aServerAuthoredChatlessWorkspaceBecomesSelectableLayout() throws { + var space = workspace(centerTree: .leaf(PaneGroupState())) + let paneId = UUID() + let record = ServerWorkspacePane( + id: paneId.uuidString, + workspaceId: space.id.uuidString, + providerId: "codevisor", + paneType: "new-tab", + title: "New tab", + createdAt: "2026-09-13T00:00:00.000Z" + ) + + WorkspaceSyncModel.reconcilePanes(in: &space, records: [record], protectedLocalPaneIds: []) + + let pane = try #require( + space.centerTabs.flatMap { $0.root.allGroups }.flatMap(\.state.panes) + .first { $0.id == paneId }) + #expect(pane.kind == .newTab) + let leafId = try #require( + space.centerTabs.flatMap { $0.root.allGroups }.first { $0.state.panes.contains { $0.id == paneId } }?.id) + + let selected = space.selectDestination(.leaf(leafId)) + #expect(selected) + + // No chat in the destination, none in the workspace, nothing already + // routed: the selection must address the workspace. + #expect( + space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + == .workspace(serverId: space.serverId, id: space.id)) + } + + /// The transition root asked to pin: the SAME leaf is entered first without a + /// chat and then with one. Once the placeholder becomes a real chat pane, the + /// leaf routes to that chat instead of the workspace, which is what remounts + /// the container as `.chat` and lets the cached group adopt the session. + @Test func aLeafThatGainsAChatRoutesToItInsteadOfTheWorkspace() throws { + var space = workspace(centerTree: .leaf(PaneGroupState.centerInitialWithoutChat())) + let leafId = space.centerTree.allGroups[0].id + let placeholderId = try #require(space.centerTree.group(id: leafId)?.panes.first?.id) + + // Before: no chat anywhere, so activating the leaf addresses the workspace. + #expect( + space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + == .workspace(serverId: space.serverId, id: space.id)) + + // The ordinary user-triggered creation path binds a real session to the + // placeholder in place (no new pane, no fabricated identity). + let chat = UUID() + var state = try #require(space.centerTree.group(id: leafId)) + let conversion = state.convertNewTabPane( + id: placeholderId, to: .chat, sessionId: nil, chatSessionId: chat) + let converted = try #require(conversion) + space.centerTree = space.centerTree.updatingGroup(id: leafId) { _ in state } + + #expect(converted.id == placeholderId) // same slot, live pane preserved + #expect(converted.chatSessionId == chat) + #expect(space.centerTree.groupId(containingChat: chat) == leafId) + + // After: the same activation now resolves to the real chat. + #expect( + space.selectionRoute( + activatedChatSessionId: chat, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + == .session(serverId: space.serverId, id: chat)) + } + + @Test func paneWorkAnchorsOnTheChatWhenThereIsOneAndOnTheWorkspaceOtherwise() { + let workspaceRoot = "/tmp/project/.worktrees/verification" + let projectRoot = "/tmp/project" + + // Session-rooted behaviour is untouched: its cwd wins, project folder backs it. + #expect( + PaneWorkingDirectory.resolve( + anchor: .session(cwd: "/tmp/project/.worktrees/chat"), + workspaceRootDirectory: workspaceRoot, projectFolderPath: projectRoot) + == "/tmp/project/.worktrees/chat") + #expect( + PaneWorkingDirectory.resolve( + anchor: .session(cwd: nil), workspaceRootDirectory: workspaceRoot, + projectFolderPath: projectRoot) == projectRoot) + + // No chat: the workspace's OWN directory, not the project root. + #expect( + PaneWorkingDirectory.resolve( + anchor: .workspace, workspaceRootDirectory: workspaceRoot, projectFolderPath: projectRoot) + == workspaceRoot) + #expect( + PaneWorkingDirectory.resolve( + anchor: .workspace, workspaceRootDirectory: nil, projectFolderPath: projectRoot) + == projectRoot) + } +} diff --git a/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ConvertedPaneTitleTests.swift b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ConvertedPaneTitleTests.swift new file mode 100644 index 000000000..92066e7b8 --- /dev/null +++ b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ConvertedPaneTitleTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing + +@testable import CodevisorCore + +/// A New Tab converted in place must be persisted under its new name and kind +/// immediately, because that persisted record is what a sidebar row re-reads. +/// The tab's own user-given title is separate and must survive the conversion. +@Suite("Converted pane title") +struct ConvertedPaneTitleTests { + private func workspace(centerTree: SplitNode) -> Workspace { + Workspace( + name: "Verification", + rootDirectory: "/tmp/project", + serverId: "stage3v", + projectId: UUID(), + centerTree: centerTree, + bottomGroup: PaneGroupState() + ) + } + + @Test func convertingAPlaceholderPersistsItsNewNameUnderTheSamePaneId() throws { + let repository = DefaultWorkspaceRepository(store: InMemoryStore()) + // A server-authored placeholder: the record's title is what the sidebar + // shows until the conversion is persisted. + var seeded = PaneGroupState() + let placeholder = seeded.addNewTabPane() + let space = workspace(centerTree: .leaf(seeded)) + let leafId = space.centerTree.allGroups[0].id + repository.save(space) + let bridge = WorkspacePaneGroupRepository( + workspaceId: space.id, groupId: leafId, repository: repository) + + var state = try #require(bridge.load(sessionId: nil, placement: .center)) + #expect(state.selectedPane?.name == "New tab") + + let conversion = state.convertNewTabPane(id: placeholder.id, to: .screenSharing, sessionId: nil) + let converted = try #require(conversion) + bridge.save(state, sessionId: nil, placement: .center) + + // Same slot, new identity-free name: what a re-read of the record yields. + #expect(converted.id == placeholder.id) + let reloaded = try #require(bridge.load(sessionId: nil, placement: .center)) + #expect(reloaded.selectedPane?.id == placeholder.id) + #expect(reloaded.selectedPane?.kind == .screenSharing) + #expect(reloaded.selectedPane?.name == "Screen Sharing") + #expect(reloaded.selectedPane?.name != "New tab") + + let persisted = try #require(repository.workspace(id: space.id)) + #expect(persisted.centerTree.group(id: leafId)?.selectedPane?.name == "Screen Sharing") + } + + @Test func aUserRenamedTabKeepsItsTitleAcrossTheConversion() throws { + var seeded = PaneGroupState() + let placeholder = seeded.addNewTabPane() + var tab = WorkspaceTab(root: .leaf(seeded)) + tab.customTitle = "Remote screen" + var space = workspace(centerTree: .leaf(PaneGroupState())) + space.centerTabs = [tab] + let leafId = try #require(tab.root.allGroups.first?.id) + + var state = try #require(space.centerTabs[0].root.group(id: leafId)) + let conversion = state.convertNewTabPane(id: placeholder.id, to: .screenSharing, sessionId: nil) + _ = try #require(conversion) + space.centerTabs[0].root = space.centerTabs[0].root.updatingGroup(id: leafId) { _ in state } + + // The pane's own name follows the conversion; the tab's user title does not. + #expect(space.centerTabs[0].root.group(id: leafId)?.selectedPane?.name == "Screen Sharing") + #expect(space.centerTabs[0].customTitle == "Remote screen") + } +} diff --git a/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ScreenSharingPaneMappingTests.swift b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ScreenSharingPaneMappingTests.swift new file mode 100644 index 000000000..09fc2c27b --- /dev/null +++ b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/ScreenSharingPaneMappingTests.swift @@ -0,0 +1,44 @@ +import Foundation +import Testing +@testable import CodevisorCore + +@MainActor +struct ScreenSharingPaneMappingTests { + @Test func nativePaneRoundTripsOnlyPresentationPreferences() throws { + let id = UUID() + let preferences = ScreenSharingPanePreferences(preferredDisplayId: "display-uuid", fitToWindow: false) + let pane = PaneDescriptorState( + id: id, kind: .screenSharing, name: "Screen Sharing", terminalKey: id.uuidString, + screenSharing: preferences) + let record = WorkspaceSyncModel.serverPane( + from: pane, workspaceId: UUID(), createdAt: Date(timeIntervalSince1970: 0)) + #expect(record.providerId == "codevisor") + #expect(record.paneType == "screen-sharing") + #expect(record.resourceId == nil) + #expect(record.resourceKind == nil) + let data = try #require(record.metadata?.data(using: .utf8)) + let fields = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect(Set(fields.keys) == ["schemaVersion", "preferredDisplayId", "fitToWindow"]) + #expect(WorkspaceSyncModel.descriptor(from: record) == pane) + #expect(try JSONDecoder().decode(PaneDescriptorState.self, from: JSONEncoder().encode(pane)) == pane) + } + + @Test func unsupportedMetadataDoesNotDiscardSiblingPanes() throws { + let id = UUID() + var record = ServerWorkspacePane( + id: id.uuidString, workspaceId: UUID().uuidString, providerId: "codevisor", + paneType: "screen-sharing", title: "Screen Sharing", metadata: #"{"schemaVersion":2,"fitToWindow":true}"#, + createdAt: "2026-01-01T00:00:00.000Z") + #expect(WorkspaceSyncModel.descriptor(from: record) == nil) + record.metadata = "invalid" + #expect(WorkspaceSyncModel.descriptor(from: record) == nil) + var state = PaneGroupState.centerInitial(sessionId: UUID()) + let initial = state.panes + let placeholder = state.addNewTabPane() + let result = state.convertNewTabPane(id: placeholder.id, to: .screenSharing, sessionId: UUID()) + let converted = try #require(result) + #expect(converted.screenSharing == ScreenSharingPanePreferences()) + #expect(state.panes.first == initial.first) + #expect(state.selectedPaneId == converted.id) + } +} diff --git a/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/WorkspaceSelectionRouteTests.swift b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/WorkspaceSelectionRouteTests.swift new file mode 100644 index 000000000..e111032dc --- /dev/null +++ b/packages/swift/CodevisorCore/Tests/CodevisorCoreTests/WorkspaceSelectionRouteTests.swift @@ -0,0 +1,96 @@ +import Foundation +import Testing + +@testable import CodevisorCore + +/// A workspace owns its server identity, project and layout whether or not it +/// has ever hosted a chat. These pin what the sidebar selection becomes when +/// one of its destinations is activated — the decision that previously left a +/// chatless workspace unreachable. +@Suite("Workspace selection route") +struct WorkspaceSelectionRouteTests { + private func workspace(id: UUID = UUID(), serverId: String = "server-a") -> Workspace { + Workspace( + id: id, + name: "Verification", + rootDirectory: "/tmp/project", + serverId: serverId, + projectId: UUID(), + centerTree: .leaf(PaneGroupState()), + bottomGroup: PaneGroupState() + ) + } + + @Test func anActivatedChatAlwaysWins() { + let chat = UUID() + let routing = UUID() + let space = workspace() + + #expect( + space.selectionRoute( + activatedChatSessionId: chat, routingSessionId: routing, + selectionAlreadyRoutesWorkspace: true + ) == .session(serverId: space.serverId, id: chat)) + } + + @Test func aChatlessDestinationFallsBackToTheWorkspacesOwnRoutingChat() { + let routing = UUID() + let space = workspace() + + #expect( + space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: routing, + selectionAlreadyRoutesWorkspace: false + ) == .session(serverId: space.serverId, id: routing)) + } + + @Test func anAlreadyRoutedSelectionIsLeftUntouched() { + let space = workspace() + + #expect( + space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: UUID(), + selectionAlreadyRoutesWorkspace: true + ) == nil) + } + + /// The defect: no chat in the destination, none anywhere in the workspace, + /// and nothing already routed — previously every branch was skipped and the + /// selection stayed on whatever was showing. + @Test func aChatlessWorkspaceAddressesItself() { + let id = UUID() + let space = workspace(id: id, serverId: "stage3v") + + #expect( + space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, + selectionAlreadyRoutesWorkspace: false + ) == .workspace(serverId: "stage3v", id: id)) + } + + @Test func theRouteCarriesThisWorkspacesOwnServerIdentity() { + let first = workspace(serverId: "stage3v") + let second = workspace(serverId: "local") + + #expect( + first.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + == .workspace(serverId: "stage3v", id: first.id)) + #expect( + second.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + == .workspace(serverId: "local", id: second.id)) + } + + /// Re-activating a destination in a workspace that is already the selection + /// is idempotent rather than oscillating with the chat fallback. + @Test func reactivatingAChatlessWorkspaceIsStable() { + let space = workspace() + let first = space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + let second = space.selectionRoute( + activatedChatSessionId: nil, routingSessionId: nil, selectionAlreadyRoutesWorkspace: false) + + #expect(first == second) + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/NativeScreenSharingBridge.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/NativeScreenSharingBridge.swift new file mode 100644 index 000000000..4ee4b295b --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/NativeScreenSharingBridge.swift @@ -0,0 +1,79 @@ +import Foundation +import CodevisorCore + +/// Adapts the existing authenticated Unix socket to the main-actor media host. +/// The socket's client queue may wait; the app's main thread never blocks. +final class NativeScreenSharingBridge: @unchecked Sendable { + @MainActor private var host: ScreenSharingHostService? + @MainActor private var hostGeneration = 0 + private let lock = NSLock() + private var generation = 0 + private var enabled = false + + func start() { + lock.withLock { + generation += 1; enabled = true + } + } + + func request(_ data: Data) throws -> Data { + let generation = try lock.withLock { + guard enabled else { throw BridgeError("Screen Sharing host is stopped.") } + return self.generation + } + let result = Reply() + let task = Task { @MainActor in + do { + try Task.checkCancellation() + guard self.lock.withLock({ self.enabled && self.generation == generation }) else { + throw BridgeError("Screen Sharing host is stopped.") + } + let request = try JSONDecoder().decode(ServerScreenSharingRequest.self, from: data) + if self.hostGeneration != generation { + let previous = self.host + self.host = nil + await previous?.shutdown() + guard self.lock.withLock({ self.enabled && self.generation == generation }) else { + throw BridgeError("Screen Sharing host is stopped.") + } + } + let host = self.host ?? ScreenSharingHostService() + self.host = host + self.hostGeneration = generation + result.finish(.success(try JSONEncoder().encode(await host.handle(request)))) + } catch { result.finish(.failure(error)) } + } + do { return try result.wait() } catch { task.cancel(); throw error } + } + + func stop() { + let generation = lock.withLock { + enabled = false; return self.generation + } + Task { @MainActor in + guard self.hostGeneration == generation else { return } + let host = self.host + self.host = nil + await host?.shutdown() + } + } + + private final class Reply: @unchecked Sendable { + private let condition = NSCondition() + private var value: Result? + func finish(_ value: Result) { + condition.lock(); defer { condition.unlock() } + guard self.value == nil else { return } + self.value = value + condition.broadcast() + } + func wait() throws -> Data { + condition.lock(); defer { condition.unlock() } + let deadline = Date().addingTimeInterval(25) + while value == nil { + guard condition.wait(until: deadline) else { throw BridgeError("Screen Sharing host timed out.") } + } + return try value!.get() + } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostConnectivity.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostConnectivity.swift new file mode 100644 index 000000000..d15c5965e --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostConnectivity.swift @@ -0,0 +1,50 @@ +import CodevisorClient +import CodevisorScreenSharing +import CryptoKit +import Foundation + +/// TURN REST credentials expire five minutes after issuance. Only the host +/// process has the shared secret; the viewer receives an HMAC-derived password. +struct ScreenSharingHostConnectivity { + let environment: [String: String] + var now: () -> TimeInterval = { Date().timeIntervalSince1970 } + + func make(viewerId: UUID) throws -> ServerScreenSharingConnectivity { + func urls(_ key: String) -> [String] { + (environment[key] ?? "").split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + } + let stun = urls("CODEVISOR_SCREEN_SHARING_STUN_URLS") + let turn = urls("CODEVISOR_SCREEN_SHARING_TURN_URLS") + let expires = Int(now()) + 300 + var servers: [ServerScreenSharingConnectivity.Server] = [] + guard stun.allSatisfy({ $0.hasPrefix("stun:") || $0.hasPrefix("stuns:") }), + turn.allSatisfy({ $0.hasPrefix("turn:") || $0.hasPrefix("turns:") }) + else { + throw ScreenSharingError.invalid("Screen Sharing connectivity URLs use the wrong protocol.") + } + if !stun.isEmpty { servers.append(.init(urls: stun)) } + if !turn.isEmpty { + guard let secret = environment["CODEVISOR_SCREEN_SHARING_TURN_SECRET"], !secret.isEmpty, secret.utf8.count <= 1024 + else { throw ScreenSharingError.unavailable("Configure the Screen Sharing TURN secret on the host Mac.") } + let username = "\(expires):codevisor:\(viewerId.uuidString.lowercased())" + let password = Data( + HMAC.authenticationCode( + for: Data(username.utf8), using: SymmetricKey(data: Data(secret.utf8))) + ).base64EncodedString() + servers.append(.init(urls: turn, username: username, credential: password)) + } + let policy = environment["CODEVISOR_SCREEN_SHARING_RELAY_ONLY"] ?? "0" + guard ["0", "1"].contains(policy) else { throw ScreenSharingError.invalid("Invalid Screen Sharing relay policy.") } + let result = ServerScreenSharingConnectivity(servers: servers, relayOnly: policy == "1", expiresAt: expires) + _ = try result.native() + return result + } +} + +extension ServerScreenSharingConnectivity { + func native() throws -> ScreenSharingICEConfiguration { + try .init( + servers: servers.map { try .init(urls: $0.urls, username: $0.username, credential: $0.credential) }, + relayOnly: relayOnly) + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostControl.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostControl.swift new file mode 100644 index 000000000..87fb1b9da --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostControl.swift @@ -0,0 +1,118 @@ +import CodevisorScreenSharing +import Foundation + +/// The session owns this controller and closes it before stopping media. +/// No input is accepted until the active peer explicitly acquires control. +@MainActor +final class ScreenSharingHostControl { + private(set) var lease: UUID? + private(set) var heldKeys = Set() + private(set) var heldButtons = Set() + private var pointer = ScreenSharingPointer(x: 0.5, y: 0.5) + private var modifiers: UInt8 = 0 + private var sequence: UInt64 = 0 + private var deadline: TimeInterval = 0 + private let now: () -> TimeInterval + private let availability: () -> String? + private let inject: (ScreenSharingInputEvent) -> Void + private let send: (ScreenSharingControlMessage) -> Bool + var onChanged: ((Bool) -> Void)? + + init( + now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, + availability: @escaping () -> String?, inject: @escaping (ScreenSharingInputEvent) -> Void, + send: @escaping (ScreenSharingControlMessage) -> Bool + ) { + self.now = now; self.availability = availability; self.inject = inject; self.send = send + } + + func receive(_ message: ScreenSharingControlMessage) { + checkDeadline() + switch message { + case .request(let request): + if let reason = availability() { _ = send(.denied(request: request, reason: reason)); return } + guard lease == nil else { + _ = send(.denied(request: request, reason: "Control is already active.")); return + } + let id = UUID() + lease = id; deadline = now() + 3; sequence = 0 + onChanged?(true) + if !send(.grant(request: request, lease: id)) { revoke("The control channel closed.") } + case .release(let id): + if lease == id { revoke("Control released.") } + case .heartbeat(let id): + guard lease == id else { return } + if let reason = availability() { revoke(reason); return } + deadline = now() + 3 + case .input(let id, let next, let event): + guard lease == id, next > sequence else { return } + if let reason = availability() { revoke(reason); return } + guard event.isValid else { revoke("Invalid input received."); return } + sequence = next + apply(event) + default: break + } + } + + func checkDeadline() { + if lease != nil, now() >= deadline { revoke("Control timed out. Request control again.") } + } + + func revoke(_ reason: String) { + guard let previous = lease else { return } + // Invalidate first: reentrant callbacks and queued packets cannot inject. + lease = nil + // Release ordinary keys before modifiers; each source tracks only its own input. + let modifierCodes: Set = [54, 55, 56, 58, 59, 60, 61, 62, 63] + let keys = heldKeys.sorted { a, b in + if modifierCodes.contains(a) != modifierCodes.contains(b) { return !modifierCodes.contains(a) } + return a < b + } + for key in keys { + modifiers &= ~Self.modifier(for: key) + inject(.key(code: key, down: false, repeatKey: false, modifiers: modifiers)) + } + for button in heldButtons.sorted() { + inject(.button(pointer, button: button, down: false, clicks: 1, modifiers: 0)) + } + heldKeys = []; heldButtons = []; modifiers = 0 + onChanged?(false) + _ = send(.revoked(lease: previous, reason: reason)) + } + + private func apply(_ event: ScreenSharingInputEvent) { + switch event { + case .key(let code, let down, let repeated, let flags): + if down { + guard repeated ? heldKeys.contains(code) : !heldKeys.contains(code) else { return } + heldKeys.insert(code) + } else { + guard heldKeys.remove(code) != nil else { return } + } + modifiers = flags + case .button(let point, let button, let down, _, let flags): + if down { + guard heldButtons.insert(button).inserted else { return } + } else { + guard heldButtons.remove(button) != nil else { return } + } + pointer = point; modifiers = flags + case .move(let point, let flags), .scroll(let point, _, _, let flags): + pointer = point; modifiers = flags + case .text: + guard heldKeys.isEmpty, heldButtons.isEmpty else { return } + } + inject(event) + } + + private static func modifier(for key: UInt16) -> UInt8 { + switch key { + case 56, 60: 1 + case 59, 62: 2 + case 58, 61: 4 + case 54, 55: 8 + case 63: 32 + default: 0 + } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostIndicator.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostIndicator.swift new file mode 100644 index 000000000..06e3fadc1 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostIndicator.swift @@ -0,0 +1,29 @@ +import AppKit + +@MainActor +final class ScreenSharingHostIndicator: NSObject { + private var item: NSStatusItem? + private var stop: (() -> Void)? + + func show(display: String, stop: @escaping () -> Void) { + hide() + self.stop = stop + let item = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) + item.button?.image = NSImage(systemSymbolName: "display", accessibilityDescription: "Screen Sharing active") + item.button?.title = " Sharing" + let menu = NSMenu() + menu.addItem(withTitle: "Sharing \(display)", action: nil, keyEquivalent: "") + let end = menu.addItem(withTitle: "Stop Sharing", action: #selector(stopSharing), keyEquivalent: "") + end.target = self + item.menu = menu + self.item = item + } + func setControlling(_ active: Bool) { + item?.button?.title = active ? " Controlled" : " Sharing" + } + func hide() { + if let item { NSStatusBar.system.removeStatusItem(item) } + item = nil; stop = nil + } + @objc private func stopSharing() { stop?() } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostLease.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostLease.swift new file mode 100644 index 000000000..0bde5fece --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostLease.swift @@ -0,0 +1,44 @@ +import Foundation +import CodevisorCore + +/// Time is supplied by the host, so ownership and expiry are deterministic. +struct ScreenSharingHostLease { + struct Owner: Equatable, Sendable { + let workspaceId: UUID + let paneId: UUID + let viewerId: UUID + init(_ request: ServerScreenSharingRequest) { + workspaceId = request.workspaceId; paneId = request.paneId; viewerId = request.viewerId + } + } + private(set) var owner: Owner? + private(set) var expiresAt: TimeInterval = 0 + static let lifetime: TimeInterval = 25 + + mutating func reserve(_ candidate: Owner, now: TimeInterval) -> Bool { + guard owner == nil else { return false } + owner = candidate; expiresAt = now + Self.lifetime + return true + } + func owns(_ candidate: Owner, now: TimeInterval) -> Bool { owner == candidate && now < expiresAt } + struct ReplacementPermit { + let revision: Int + let deadline: TimeInterval + func isValid(revision: Int, now: TimeInterval) -> Bool { self.revision == revision && now < deadline } + } + func replacementPermit(_ candidate: Owner, revision: Int, now: TimeInterval) -> ReplacementPermit? { + guard owns(candidate, now: now) else { return nil } + return .init(revision: revision, deadline: expiresAt) + } + mutating func renew(_ candidate: Owner, now: TimeInterval) -> Bool { + guard owns(candidate, now: now) else { return false } + expiresAt = now + Self.lifetime + return true + } + func isExpired(now: TimeInterval) -> Bool { owner != nil && now >= expiresAt } + mutating func release(_ candidate: Owner) -> Bool { + guard owner == candidate else { return false } + owner = nil + return true + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostService.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostService.swift new file mode 100644 index 000000000..c525a4a06 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingHostService.swift @@ -0,0 +1,401 @@ +import AppKit +import CodevisorCore +import CodevisorScreenSharing +import Foundation +import OSLog + +/// One authorized viewer per native host. Capture starts only after the peer +/// carrying the authenticated SDP connects, and a missed lease stops capture. +@MainActor +final class ScreenSharingHostService { + typealias Display = (id: UInt32, description: ServerScreenSharingDisplay) + private static let logger = Logger(subsystem: "com.851labs.Codevisor", category: "ScreenSharing") + @MainActor private final class Session { + let owner: ScreenSharingHostLease.Owner + let peer: ScreenSharingPeer + let capture: ScreenSharingCapture + /// The explicit experimental profile in force for this process, or nil when it is OFF (the default). + let profile: ScreenSharingDiagnosticProfile? + let metrics: ScreenSharingMetrics + let display: ServerScreenSharingDisplay + let displayID: UInt32 + let configuration: ScreenSharingVideoConfiguration + var state = "connecting" + var control: ScreenSharingHostControl? + var clipboard: ScreenSharingClipboardTransfer? + var stopping = false + var watchdog: Task? + var captureTask: Task? + var qualityTask: Task? + + init( + request: ServerScreenSharingRequest, display: ServerScreenSharingDisplay, displayID: UInt32, + connectivity: ServerScreenSharingConnectivity, profile: ScreenSharingDiagnosticProfile? + ) throws { + owner = .init(request) + self.profile = profile + self.display = display; self.displayID = displayID + // Level 0 is where a session starts; lower levels request the video rate through the same validated path. + capture = ScreenSharingCapture(captureIntervalFPS: profile?.captureIntervalFPS(adaptiveLevel: 0)) + let scale = min(1, min(1920.0 / Double(display.width), 1080.0 / Double(display.height))) + configuration = try ScreenSharingVideoConfiguration( + width: max(64, Int(Double(display.width) * scale) / 2 * 2), + height: max(64, Int(Double(display.height) * scale) / 2 * 2)) + metrics = ScreenSharingMetrics() + // install(profile:) throws unless the process trial map equals what this profile requires, so reaching the next + // line means the profile's settings below are the ones actually wired. "Active" therefore names THIS validated + // profile; the first installer's provenance is a separate fact published by the peer as fieldTrialProvenance. + try ScreenSharingFieldTrials.process.install(profile: profile) + metrics.label("diagnosticProfileRequested", profile?.name ?? "none") + metrics.label("diagnosticProfileActive", profile?.name ?? "none") + if let profile { + metrics.label( + "diagnosticProfileCaptureRequest", + "\(profile.captureIntervalFPSAtLevel0) fps at adaptive level 0, video rate below") + } + peer = try ScreenSharingPeer( + sending: true, configuration: configuration, metrics: metrics, + connectivity: connectivity.native()) + } + } + private var current: Session? + // Enumeration can suspend before a session owns the lease. Stop must invalidate those attempts by owner too. + private var pendingStarts: [UUID: ScreenSharingHostLease.Owner] = [:] + private var isShutdown = false + private var stopGeneration = 0 + private var lease = ScreenSharingHostLease() + private let indicator = ScreenSharingHostIndicator() + private var observers: [NSObjectProtocol] = [] + private let connectivity = ScreenSharingHostConnectivity(environment: ProcessInfo.processInfo.environment) + private let captureAccess: () -> Bool + private let enumerateDisplays: () async throws -> [Display] + private let notificationCenter: NotificationCenter + private let workspaceNotificationCenter: NotificationCenter + + convenience init() { + self.init( + captureAccess: { CGPreflightScreenCaptureAccess() }, notificationCenter: .default, + workspaceNotificationCenter: NSWorkspace.shared.notificationCenter, enumerateDisplays: Self.displays) + } + + /// Keep OS permission, notifications and display enumeration at the boundary for request-ordering tests. + init( + captureAccess: @escaping () -> Bool, notificationCenter: NotificationCenter, + workspaceNotificationCenter: NotificationCenter, enumerateDisplays: @escaping () async throws -> [Display] + ) { + self.captureAccess = captureAccess + self.enumerateDisplays = enumerateDisplays + self.notificationCenter = notificationCenter + self.workspaceNotificationCenter = workspaceNotificationCenter + } + + func handle(_ request: ServerScreenSharingRequest) async -> ServerScreenSharingReply { + guard !isShutdown, !Task.isCancelled else { return .init(status: "stopped") } + guard request.version == 1 else { + return .init(status: "unavailable", message: "Update Codevisor to use Screen Sharing.") + } + // Register before any suspension, including cleanup of an expired previous session. + let attempt: UUID? + if request.operation == .start || request.operation == .restart { + let id = UUID() + pendingStarts[id] = .init(request) + attempt = id + } else { + attempt = nil + } + defer { if let attempt { pendingStarts.removeValue(forKey: attempt) } } + installObservers() + if lease.isExpired(now: ProcessInfo.processInfo.systemUptime), let current { await end(current) } + guard !isShutdown else { return .init(status: "stopped") } + if let attempt, Task.isCancelled || pendingStarts[attempt] == nil { return .init(status: "stopped") } + switch request.operation { + case .stop: + stopGeneration += 1 + let owner = ScreenSharingHostLease.Owner(request) + pendingStarts = pendingStarts.filter { $0.value != owner } + if let current, current.owner == .init(request) { await end(current) } + return .init(status: "stopped") + case .heartbeat: + guard let current, !current.stopping, + lease.renew(.init(request), now: ProcessInfo.processInfo.systemUptime) + else { + return .init(status: "stopped", message: "Screen sharing ended on the host Mac.") + } + let labels = current.metrics.snapshot().labels + if let error = labels["captureError"] ?? labels["encoderError"] { + await end(current) + return .init(status: "failed", message: error) + } + return .init(status: current.state) + case .capabilities: + guard captureAccess() else { return permissionRequired() } + do { + return .init( + status: current == nil ? "available" : "busy", displays: try await enumerateDisplays().map(\.description), + connectivity: try connectivity.make(viewerId: request.viewerId)) + } catch { return .init(status: "unavailable", message: error.localizedDescription) } + case .start, .restart: + var replacement: ScreenSharingHostLease.ReplacementPermit? + if request.operation == .restart { + // A fresh media peer avoids replaying old decoder, cursor or input + // state. Renewal still requires the existing live host lease. + guard let old = current, !old.stopping, old.owner == .init(request), + old.display.id == request.displayId, + let permit = lease.replacementPermit( + old.owner, revision: stopGeneration, now: ProcessInfo.processInfo.systemUptime) + else { + return .init( + status: "stopped", message: "The host ended this sharing session. Connect again to start a new one.") + } + replacement = permit + await end(old) + guard !isShutdown, permit.isValid(revision: stopGeneration, now: ProcessInfo.processInfo.systemUptime) else { + return .init(status: "stopped", message: "Screen sharing ended on the host Mac.") + } + } + guard current == nil else { + return .init(status: "busy", message: "This Mac is already sharing with another viewer.") + } + guard captureAccess() else { return permissionRequired() } + guard let offer = request.offer, offer.utf8.count <= 256 * 1024, + offer.contains("a=fingerprint:sha-256 "), let displayId = request.displayId + else { + return .init(status: "failed", message: "Invalid Screen Sharing request.") + } + do { + let available = try await enumerateDisplays() + try Task.checkCancellation() + guard !isShutdown, let attempt, pendingStarts[attempt] != nil else { return .init(status: "stopped") } + if let replacement, !replacement.isValid(revision: stopGeneration, now: ProcessInfo.processInfo.systemUptime) { + return .init(status: "stopped", message: "Screen sharing ended on the host Mac.") + } + // Display enumeration suspends; another request may reserve the host. + guard current == nil else { + return .init(status: "busy", message: "This Mac is already sharing with another viewer.") + } + guard let display = available.first(where: { $0.description.id == displayId }) else { + return .init(status: "unavailable", message: "The selected display is no longer available.") + } + // Parsed once per process (failures cached too); an unknown value fails the request rather than selecting + // the candidate, and both roles in this app process read the same answer. + let profile = try ScreenSharingDiagnosticProfile.process() + let session = try Session( + request: request, display: display.description, displayID: display.id, + connectivity: connectivity.make(viewerId: request.viewerId), profile: profile) + guard lease.reserve(session.owner, now: ProcessInfo.processInfo.systemUptime) else { + session.peer.close() + return .init(status: "busy", message: "This Mac is already sharing with another viewer.") + } + current = session + configure(session) + do { + try await session.peer.accept(.init(kind: "offer", sdp: offer)) + let answer = try await session.peer.makeDescription(offer: false) + try Task.checkCancellation() + guard current === session, !session.stopping else { throw CancellationError() } + return .init(status: "connecting", answer: answer.sdp) + } catch { await end(session); throw error } + } catch { return .init(status: "failed", message: error.localizedDescription) } + } + } + + func shutdown() async { + isShutdown = true + pendingStarts.removeAll() + stopGeneration += 1 + if let current { await end(current) } + for observer in observers { + notificationCenter.removeObserver(observer); + workspaceNotificationCenter.removeObserver(observer) + } + observers = [] + } + + private func configure(_ session: Session) { + session.qualityTask = Task { [weak self, weak session] in + guard let initial = session?.configuration else { return } + var quality = ScreenSharingAdaptiveQuality(configuration: initial) + while !Task.isCancelled { + do { try await Task.sleep(for: .seconds(1)) } catch { return } + guard let session, !session.stopping else { return } + guard session.state == "viewing" else { continue } + let statistics = await session.peer.statistics() + guard !Task.isCancelled, !session.stopping else { return } + guard session.state == "viewing" else { continue } + let bandwidth = statistics.first { $0.key.hasSuffix(".availableOutgoingBitrate") }.flatMap { Double($0.value) } + if let configuration = quality.update(availableBitrate: bandwidth, now: ProcessInfo.processInfo.systemUptime) { + session.peer.updateVideoConfiguration(configuration) + // Level-aware request: the override only at level 0, the video rate below it. Validation, the SCK call and + // the label commit all happen inside the single capture path. + do { + try await session.capture.update( + configuration: configuration, + captureIntervalFPS: session.profile?.captureIntervalFPS(adaptiveLevel: quality.level)) + } catch { + if !Task.isCancelled { + session.metrics.label("captureError", "Unable to adjust capture quality.") + self?.scheduleEnd(session) + } + return + } + session.metrics.label("adaptiveQualityLevel", String(quality.level)) + } + } + } + let pasteboard = ScreenSharingPasteboard() + let clipboard = ScreenSharingClipboardTransfer( + send: { [weak session] in session?.peer.clipboard.send($0) ?? false }, + canReceiveUnsolicited: { [weak session] in session?.state == "viewing" && session?.stopping == false }, + read: { try pasteboard.read() }, write: { try pasteboard.write($0) }) + session.clipboard = clipboard + session.peer.clipboard.onMessage = { [weak clipboard] in clipboard?.receive($0) } + session.peer.clipboard.onAvailabilityChanged = { [weak clipboard] available in + if !available { clipboard?.cancel(reason: "The clipboard channel closed.") } + } + let injector = ScreenSharingInputInjector(displayBounds: CGDisplayBounds(session.displayID)) + let control = ScreenSharingHostControl( + availability: { [weak session] in + guard let session, !session.stopping, session.state == "viewing" else { + return "Wait for live video before requesting control." + } + guard injector.isAvailable else { return "Native input is unavailable on this Mac." } + guard AXIsProcessTrusted() else { + return + "Allow Codevisor in System Settings → Privacy & Security → Accessibility on the host Mac, then request control again." + } + return nil + }, + inject: { [weak session] in + session?.metrics.increment("controlInputEvents"); injector.post($0) + }, send: { [weak session] in session?.peer.control.send($0) ?? false }) + session.control = control + session.peer.control.onMessage = { [weak control] in control?.receive($0) } + session.peer.control.onAvailabilityChanged = { [weak control] available in + if !available { control?.revoke("The control channel closed.") } + } + control.onChanged = { [weak self] active in self?.indicator.setControlling(active) } + + session.capture.onStopped = { [weak self, weak session] _ in + guard let self, let session else { return } + self.scheduleEnd(session) + } + session.peer.onConnectionChanged = { [weak self, weak session] state in + guard let self, let session, self.current === session, !session.stopping else { return } + if state == "connected", session.captureTask == nil { + session.captureTask = Task { [weak self, weak session] in + guard let self, let session else { return } + do { + try await session.capture.start( + displayID: session.displayID, configuration: session.configuration, + sender: session.peer.frameSender, metrics: session.metrics) + guard self.current === session, !session.stopping else { try? await session.capture.stop(); return } + session.state = "viewing" + self.indicator.show(display: session.display.name) { [weak self, weak session] in + guard let self, let session else { return } + self.scheduleEnd(session) + } + } catch { await self.end(session) } + } + } else if ["disconnected", "failed", "closed"].contains(state) { + session.state = "reconnecting" + session.control?.revoke("The connection was interrupted.") + session.clipboard?.cancel(reason: "The connection was interrupted.") + let pending = session.captureTask + pending?.cancel() + session.captureTask = Task { + await pending?.value + try? await session.capture.stop() + } + } + } + session.watchdog = Task { [weak self, weak session] in + while !Task.isCancelled { + do { try await Task.sleep(for: .milliseconds(250)) } catch { return } + guard let self, let session, self.current === session else { return } + session.control?.checkDeadline() + session.clipboard?.tick() + if self.lease.isExpired(now: ProcessInfo.processInfo.systemUptime) { await self.end(session); return } + } + } + } + + private func scheduleEnd(_ session: Session) { + guard current === session else { return } + stopGeneration += 1 + session.state = "stopping" + session.control?.revoke("Screen sharing ended.") + session.clipboard?.cancel(reason: "Screen sharing ended.") + Task { await end(session) } + } + + private func end(_ session: Session) async { + guard current === session, !session.stopping else { return } + session.stopping = true + session.control?.revoke("Screen sharing ended.") + session.clipboard?.cancel(reason: "Screen sharing ended.") + let metrics = session.metrics.snapshot() + Self.logger.info( + "Host ended: \(session.state, privacy: .public), \(String(describing: metrics.counters), privacy: .public), \(String(describing: metrics.labels), privacy: .public)" + ) + session.watchdog?.cancel() + session.qualityTask?.cancel() + session.peer.close() + session.captureTask?.cancel() + // Capture invalidates its generation; a late startup stops its own stream. + try? await session.capture.stop() + indicator.hide() + _ = lease.release(session.owner) + if current === session { current = nil } + } + + private func permissionRequired() -> ServerScreenSharingReply { + .init( + status: "permission-required", + message: + "Allow Codevisor in System Settings → Privacy & Security → Screen & System Audio Recording on the host Mac, then retry." + ) + } + + private static func displays() async throws -> [Display] { + try await ScreenSharingCapture.displays().compactMap { display in + guard let uuid = CGDisplayCreateUUIDFromDisplayID(display.id)?.takeRetainedValue() else { return nil } + let identity = CFUUIDCreateString(nil, uuid) as String + let mode = CGDisplayCopyDisplayMode(display.id) + let pixelScale = mode.map { Double($0.pixelWidth) / Double(max(1, $0.width)) } ?? 1 + let name = + NSScreen.screens.first { + ($0.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == display.id + }?.localizedName ?? "Display" + return ( + display.id, + ServerScreenSharingDisplay( + id: identity, name: name, + width: Int(Double(display.width) * pixelScale), height: Int(Double(display.height) * pixelScale)) + ) + } + } + + /// Shared by the system notification adapter and deterministic request-ordering coverage. + func systemStopped() { + stopGeneration += 1 + pendingStarts.removeAll() + if let current { scheduleEnd(current) } + } + + private func installObservers() { + guard observers.isEmpty else { return } + let stop: @Sendable (Notification) -> Void = { [weak self] _ in + MainActor.assumeIsolated { + self?.systemStopped() + } + } + for name in [NSWorkspace.screensDidSleepNotification, NSWorkspace.sessionDidResignActiveNotification] { + observers.append( + workspaceNotificationCenter.addObserver(forName: name, object: nil, queue: .main, using: stop)) + } + observers.append( + notificationCenter.addObserver( + forName: NSApplication.didChangeScreenParametersNotification, + object: nil, queue: .main, using: stop)) + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputInjector.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputInjector.swift new file mode 100644 index 000000000..ecc3dbdd9 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputInjector.swift @@ -0,0 +1,87 @@ +import AppKit +import CodevisorScreenSharing + +/// Public Quartz events only, without automation's waits or app activation. +/// Display bounds are Quartz global points, including negative display origins. +@MainActor +final class ScreenSharingInputInjector { + static let eventTag: Int64 = 0x435653435245454E + private let source = CGEventSource(stateID: .privateState) + private let displayBounds: CGRect + private var buttons = Set() + private let deliver: (CGEvent) -> Void + var isAvailable: Bool { source != nil } + + init(displayBounds: CGRect, deliver: @escaping (CGEvent) -> Void = { $0.post(tap: .cghidEventTap) }) { + self.deliver = deliver + self.displayBounds = displayBounds + source?.userData = Self.eventTag + source?.localEventsSuppressionInterval = 0 + } + + func post(_ input: ScreenSharingInputEvent) { + guard let source else { return } + switch input { + case .move(let pointer, let modifiers): + let button = buttons.sorted().first + let type: CGEventType = + button == 0 + ? .leftMouseDragged + : button == 1 + ? .rightMouseDragged + : button == 2 ? .otherMouseDragged : .mouseMoved + emit( + CGEvent( + mouseEventSource: source, mouseType: type, mouseCursorPosition: location(pointer), + mouseButton: CGMouseButton(rawValue: UInt32(button ?? 0)) ?? .left), modifiers: modifiers) + case .button(let pointer, let button, let down, let clicks, let modifiers): + if down { buttons.insert(button) } else { buttons.remove(button) } + let type: CGEventType = + button == 0 + ? (down ? .leftMouseDown : .leftMouseUp) + : button == 1 ? (down ? .rightMouseDown : .rightMouseUp) : (down ? .otherMouseDown : .otherMouseUp) + let event = CGEvent( + mouseEventSource: source, mouseType: type, mouseCursorPosition: location(pointer), + mouseButton: CGMouseButton(rawValue: UInt32(button)) ?? .left) + event?.setIntegerValueField(.mouseEventClickState, value: Int64(clicks)) + emit(event, modifiers: modifiers) + case .scroll(let pointer, let x, let y, let modifiers): + let event = CGEvent( + scrollWheelEvent2Source: source, units: .pixel, wheelCount: 2, wheel1: y, wheel2: x, wheel3: 0) + event?.location = location(pointer) + emit(event, modifiers: modifiers) + case .key(let code, let down, let repeated, let modifiers): + let event = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: down) + event?.setIntegerValueField(.keyboardEventAutorepeat, value: repeated ? 1 : 0) + emit(event, modifiers: modifiers) + case .text(let text): + let units = Array(text.utf16) + for down in [true, false] { + let event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: down) + units.withUnsafeBufferPointer { buffer in + if let base = buffer.baseAddress { + event?.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: base) + } + } + emit(event, modifiers: 0) + } + } + } + + func location(_ pointer: ScreenSharingPointer) -> CGPoint { + CGPoint( + x: displayBounds.minX + min(displayBounds.width - 0.001, pointer.x * displayBounds.width), + y: displayBounds.minY + min(displayBounds.height - 0.001, pointer.y * displayBounds.height)) + } + + private func emit(_ event: CGEvent?, modifiers: UInt8) { + let masks: [CGEventFlags] = [ + .maskShift, .maskControl, .maskAlternate, .maskCommand, .maskAlphaShift, .maskSecondaryFn, + ] + var flags: CGEventFlags = [] + for (index, mask) in masks.enumerated() where modifiers & (1 << index) != 0 { flags.insert(mask) } + event?.flags = flags + event?.setIntegerValueField(.eventSourceUserData, value: Self.eventTag) + if let event { deliver(event) } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputSurface.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputSurface.swift new file mode 100644 index 000000000..5f1b66b36 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingInputSurface.swift @@ -0,0 +1,137 @@ +import AppKit +import CodevisorScreenSharing + +/// Responder and event routing for the native surface. AppKit delivers ordinary +/// physical key events; the host's keyboard layout and input method interpret them. +@MainActor +final class ScreenSharingInputSurface { + private weak var view: ScreenSharingVideoSurface? + var onInput: ((ScreenSharingInputEvent) -> Void)? + var onRelease: (() -> Void)? + private var monitor: Any? + private var observers: [NSObjectProtocol] = [] + private var motionTask: Task? + private var pendingMotion: ScreenSharingInputEvent? + private var buttons = Set() + private var modifiers: UInt8 = 0 + private var keys = Set() + private var scroll = ScreenSharingScrollAccumulator() + private(set) var active = false + + init(view: ScreenSharingVideoSurface) { self.view = view } + + func begin() -> Bool { + guard let view, let window = view.window, window.isKeyWindow, + window.makeFirstResponder(view) + else { return false } + active = true + view.controlCursorChanged() + monitor = NSEvent.addLocalMonitorForEvents( + matching: [.keyDown, .keyUp, .flagsChanged, .leftMouseDown, .rightMouseDown, .otherMouseDown] + ) { [weak self] event in + guard let self else { return event } + return self.route(event) + } + let release: @Sendable (Notification) -> Void = { [weak self] _ in + MainActor.assumeIsolated { self?.onRelease?() } + } + observers = [ + NotificationCenter.default.addObserver( + forName: NSWindow.didResignKeyNotification, object: window, queue: .main, using: release), + NotificationCenter.default.addObserver( + forName: NSApplication.didResignActiveNotification, object: nil, queue: .main, using: release), + ] + motionTask = Task { [weak self] in + while !Task.isCancelled { + do { try await Task.sleep(for: .milliseconds(16)) } catch { return } + self?.flushMotion() + } + } + return true + } + + func end() { + active = false + view?.controlCursorChanged() + if let monitor { NSEvent.removeMonitor(monitor) } + monitor = nil + for observer in observers { NotificationCenter.default.removeObserver(observer) } + observers = []; motionTask?.cancel(); motionTask = nil + pendingMotion = nil; buttons = []; keys = []; modifiers = 0; scroll = .init() + } + + private func route(_ event: NSEvent) -> NSEvent? { + guard active, let view else { return event } + if event.cgEvent?.getIntegerValueField(.eventSourceUserData) == ScreenSharingInputInjector.eventTag { return event } + guard view.window?.isKeyWindow == true, view.window?.firstResponder === view else { + onRelease?(); return event + } + if [.leftMouseDown, .rightMouseDown, .otherMouseDown].contains(event.type) { + if event.window !== view.window || !view.bounds.contains(view.convert(event.locationInWindow, from: nil)) { + onRelease?() + } + return event + } + if event.type == .keyDown, event.keyCode == 53, event.modifierFlags.contains([.control, .option]) { + onRelease?(); return nil + } + flushMotion() + syncModifiers(event.modifierFlags) + guard active else { return nil } + if event.type == .flagsChanged { return nil } + let down = event.type == .keyDown + if down { keys.insert(event.keyCode) } else if keys.remove(event.keyCode) == nil { return nil } + onInput?(.key(code: event.keyCode, down: down, repeatKey: event.isARepeat, modifiers: modifiers)) + return nil + } + + func mouse(_ event: NSEvent) { + guard active, + event.cgEvent?.getIntegerValueField(.eventSourceUserData) != ScreenSharingInputInjector.eventTag, + let point = view?.pointer(event, clamp: !buttons.isEmpty) + else { return } + let flags = Self.flags(event.modifierFlags) + switch event.type { + case .mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: + pendingMotion = .move(point, modifiers: flags) + case .leftMouseDown, .rightMouseDown, .otherMouseDown, .leftMouseUp, .rightMouseUp, .otherMouseUp: + guard (0...2).contains(event.buttonNumber) else { return } + flushMotion(); syncModifiers(event.modifierFlags) + let button = UInt8(event.buttonNumber) + let down = [.leftMouseDown, .rightMouseDown, .otherMouseDown].contains(event.type) + if down { buttons.insert(button) } else { buttons.remove(button) } + onInput?( + .button(point, button: button, down: down, clicks: UInt8(min(3, max(1, event.clickCount))), modifiers: flags)) + case .scrollWheel: + flushMotion(); syncModifiers(event.modifierFlags) + let scale = event.hasPreciseScrollingDeltas ? 1.0 : 12.0 + let delta = scroll.add(x: event.scrollingDeltaX * scale, y: event.scrollingDeltaY * scale) + if delta.x != 0 || delta.y != 0 { onInput?(.scroll(point, x: delta.x, y: delta.y, modifiers: flags)) } + default: break + } + } + + private func flushMotion() { + guard active, let pending = pendingMotion else { return } + pendingMotion = nil + onInput?(pending) + } + + private func syncModifiers(_ flags: NSEvent.ModifierFlags) { + let next = Self.flags(flags) + let codes: [(UInt8, UInt16)] = [(1, 56), (2, 59), (4, 58), (8, 55), (32, 63)] + for (mask, code) in codes where (next & mask) != (modifiers & mask) { + let down = next & mask != 0 + if down { modifiers |= mask } else { modifiers &= ~mask } + onInput?(.key(code: code, down: down, repeatKey: false, modifiers: modifiers)) + } + modifiers = next + } + + private static func flags(_ flags: NSEvent.ModifierFlags) -> UInt8 { + let values: [NSEvent.ModifierFlags] = [.shift, .control, .option, .command, .capsLock, .function] + return values.enumerated().reduce(UInt8(0)) { result, entry in + result | (flags.contains(entry.element) ? 1 << entry.offset : 0) + } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingPasteboard.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingPasteboard.swift new file mode 100644 index 000000000..5533f167b --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingPasteboard.swift @@ -0,0 +1,33 @@ +import AppKit +import CodevisorScreenSharing + +@MainActor +final class ScreenSharingPasteboard { + private let pasteboard: NSPasteboard + var changeCount: Int { pasteboard.changeCount } + init(_ pasteboard: NSPasteboard = .general) { self.pasteboard = pasteboard } + + func read() throws -> String { + guard let text = pasteboard.string(forType: .string) else { + throw ClipboardError("The clipboard does not contain plain text.") + } + guard text.utf8.count <= ScreenSharingClipboardMessage.maximumTextBytes else { + throw ClipboardError("Clipboard text must be 64 KiB or smaller.") + } + return text + } + func write(_ text: String, expectedChangeCount: Int? = nil) throws { + if let expectedChangeCount, expectedChangeCount != pasteboard.changeCount { + throw ClipboardError("The local clipboard changed during transfer. Get the remote clipboard again.") + } + let item = NSPasteboardItem() + guard item.setString(text, forType: .string) else { throw ClipboardError("Cannot prepare clipboard text.") } + pasteboard.clearContents() + guard pasteboard.writeObjects([item]) else { throw ClipboardError("Cannot write clipboard text.") } + } + struct ClipboardError: LocalizedError { + let message: String + init(_ message: String) { self.message = message } + var errorDescription: String? { message } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingVideoSurface.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingVideoSurface.swift new file mode 100644 index 000000000..f88ad7aa5 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingVideoSurface.swift @@ -0,0 +1,247 @@ +import AppKit +import CodevisorClient +import CodevisorScreenSharing +import OSLog + +@MainActor +final class ScreenSharingVideoSurface: NSView { + var onFocusChanged: ((Bool) -> Void)? + let metal: ScreenSharingMetalView + lazy var input = ScreenSharingInputSurface(view: self) + private var tracking: NSTrackingArea? + private static let remoteCursor = NSCursor(image: NSImage(size: NSSize(width: 1, height: 1)), hotSpot: .zero) + private let scroll = NSScrollView() + private var videoSize = CGSize(width: 1920, height: 1080) + var fitToWindow = true { didSet { needsLayout = true; metal.fitToWindow = fitToWindow } } + + /// `profile` nil (the default) keeps the product renderer exactly as it was: display-link drive, three drawables, + /// main-actor preparation. The explicit profile forwards to the EXISTING worker/arrival2 initializer; no pacing, + /// render-queue rewrite or auditing feature is added here. + init(peer: ScreenSharingPeer, profile: ScreenSharingDiagnosticProfile? = nil) throws { + metal = try ScreenSharingMetalView( + mailbox: peer.mailbox, metrics: peer.metrics, renderOnArrival: profile?.renderOnArrival ?? false, + maximumDrawableCount: profile?.maximumDrawableCount ?? 3, + offMainPreparation: profile?.offMainPreparation ?? false) + super.init(frame: .zero) + scroll.drawsBackground = false + scroll.autohidesScrollers = true + scroll.documentView = metal + addSubview(scroll) + metal.onFrameSize = { [weak self] size in + self?.videoSize = size + self?.needsLayout = true + } + } + required init?(coder: NSCoder) { nil } + override func layout() { + super.layout() + scroll.frame = bounds + scroll.hasHorizontalScroller = !fitToWindow + scroll.hasVerticalScroller = !fitToWindow + let viewport = scroll.contentSize + let pixels = convertFromBacking(videoSize) + metal.frame = CGRect( + origin: .zero, + size: fitToWindow + ? viewport + : CGSize( + width: max(viewport.width, pixels.width), height: max(viewport.height, pixels.height))) + window?.invalidateCursorRects(for: self) + } + override func viewDidChangeBackingProperties() { super.viewDidChangeBackingProperties(); needsLayout = true } + /// Input first, then the renderer's terminal stop (arrival subscription, + /// mailbox, cached frame and callbacks released; an in-flight submission + /// keeps its buffers until the GPU completes it). + func stop() { input.end(); metal.stop() } + override var acceptsFirstResponder: Bool { true } + override func becomeFirstResponder() -> Bool { + let accepted = super.becomeFirstResponder() + if accepted { onFocusChanged?(true) } + return accepted + } + override func resignFirstResponder() -> Bool { + if input.active { input.onRelease?() } + let accepted = super.resignFirstResponder() + if accepted { onFocusChanged?(false) } + return accepted + } + override func hitTest(_ point: NSPoint) -> NSView? { + let hit = super.hitTest(point) + return input.active && hit != nil ? self : hit + } + override func updateTrackingAreas() { + super.updateTrackingAreas() + if let tracking { removeTrackingArea(tracking) } + let area = NSTrackingArea( + rect: .zero, options: [.mouseMoved, .mouseEnteredAndExited, .cursorUpdate, .activeInKeyWindow, .inVisibleRect], + owner: self) + tracking = area; addTrackingArea(area) + } + override func resetCursorRects() { + super.resetCursorRects() + guard input.active else { return } + let drawable = metal.convertToBacking(metal.bounds).size + let scale = fitToWindow ? min(drawable.width / videoSize.width, drawable.height / videoSize.height) : 1 + let video = CGRect( + x: (drawable.width - videoSize.width * scale) / 2, + y: (drawable.height - videoSize.height * scale) / 2, + width: videoSize.width * scale, height: videoSize.height * scale) + let rect = convert(metal.convertFromBacking(video), from: metal).intersection(bounds) + if !rect.isEmpty { addCursorRect(rect, cursor: Self.remoteCursor) } + } + func controlCursorChanged() { + window?.invalidateCursorRects(for: self) + if !input.active, NSCursor.current === Self.remoteCursor { NSCursor.arrow.set() } + } + override func cursorUpdate(with event: NSEvent) { + if input.active, pointer(event, clamp: false) != nil { Self.remoteCursor.set() } else { NSCursor.arrow.set() } + } + override func mouseExited(with event: NSEvent) { + if NSCursor.current === Self.remoteCursor { NSCursor.arrow.set() } + } + override func mouseEntered(with event: NSEvent) { cursorUpdate(with: event) } + override func mouseMoved(with event: NSEvent) { + if input.active { cursorUpdate(with: event) } + input.mouse(event) + } + override func mouseDown(with event: NSEvent) { input.mouse(event) } + override func mouseUp(with event: NSEvent) { input.mouse(event) } + override func rightMouseDown(with event: NSEvent) { input.mouse(event) } + override func rightMouseUp(with event: NSEvent) { input.mouse(event) } + override func otherMouseDown(with event: NSEvent) { input.mouse(event) } + override func otherMouseUp(with event: NSEvent) { input.mouse(event) } + override func mouseDragged(with event: NSEvent) { input.mouse(event) } + override func rightMouseDragged(with event: NSEvent) { input.mouse(event) } + override func otherMouseDragged(with event: NSEvent) { input.mouse(event) } + override func scrollWheel(with event: NSEvent) { + if input.active { input.mouse(event) } else { super.scrollWheel(with: event) } + } + func pointer(_ event: NSEvent, clamp: Bool) -> ScreenSharingPointer? { + let point = metal.convertToBacking(metal.convert(event.locationInWindow, from: nil)) + let size = metal.convertToBacking(metal.bounds).size + return ScreenSharingVideoGeometry.pointer( + x: point.x, y: metal.isFlipped ? point.y : size.height - point.y, + surfaceWidth: size.width, surfaceHeight: size.height, + videoWidth: videoSize.width, videoHeight: videoSize.height, fit: fitToWindow, clamp: clamp) + } + +} + +@MainActor +protocol ScreenSharingViewingPeer: AnyObject { + var view: NSView { get } + var failure: String? { get } + var control: ScreenSharingViewerControl { get } + var clipboard: ScreenSharingViewerClipboard? { get } + var diagnostics: ScreenSharingViewerDiagnostics { get } + var onReady: (() -> Void)? { get set } + var onConnectionChanged: ((String) -> Void)? { get set } + var onFocusChanged: ((Bool) -> Void)? { get set } + func offer() async throws -> String + func accept(_ answer: String) async throws + func fit(_ enabled: Bool) + func close() +} + +@MainActor +final class NativeScreenSharingViewingPeer: ScreenSharingViewingPeer { + private static let logger = Logger(subsystem: "com.851labs.Codevisor", category: "ScreenSharing") + let peer: ScreenSharingPeer + let surface: ScreenSharingVideoSurface + let control: ScreenSharingViewerControl + let clipboard: ScreenSharingViewerClipboard? + let diagnostics = ScreenSharingViewerDiagnostics() + private var controlTask: Task? + private var diagnosticsTask: Task? + var view: NSView { surface } + var failure: String? { peer.metrics.snapshot().labels["decoderError"] } + var onReady: (() -> Void)? + var onConnectionChanged: ((String) -> Void)? + var onFocusChanged: ((Bool) -> Void)? + private var presented = false + + init(connectivity: ServerScreenSharingConnectivity? = nil) throws { + // Parsed once per process (failures cached too), so both roles in this app process agree; an unknown value fails + // here rather than selecting the candidate. + let profile = try ScreenSharingDiagnosticProfile.process() + // Trials are process-wide and irreversible: install (or prove already installed) BEFORE the peer exists. A + // conflict with a selection already installed by the host role in this same app process throws here, so reaching + // the next line means this profile's settings are the ones actually wired below. + try ScreenSharingFieldTrials.process.install(profile: profile) + let metrics = ScreenSharingMetrics() + metrics.label("diagnosticProfileRequested", profile?.name ?? "none") + metrics.label("diagnosticProfileActive", profile?.name ?? "none") + if let profile { + metrics.label( + "diagnosticProfileRenderer", + "arrival rendering, \(profile.maximumDrawableCount) drawables, off-main preparation") + } + peer = try ScreenSharingPeer( + sending: false, configuration: .init(), metrics: metrics, + connectivity: connectivity?.native()) + surface = try ScreenSharingVideoSurface(peer: peer, profile: profile) + clipboard = ScreenSharingViewerClipboard(channel: peer.clipboard) + let channel = peer.control + control = ScreenSharingViewerControl(send: { [weak channel] in channel?.send($0) ?? false }) + channel.onMessage = { [weak control] in control?.receive($0) } + channel.onAvailabilityChanged = { [weak control] in control?.setAvailable($0) } + control.setAvailable(channel.isAvailable) + control.onActiveChanged = { [weak self] active in + guard let self else { return } + if active { + if !self.surface.input.begin() { self.control.release(reason: "Focus this window and request control again.") } + } else { + self.surface.input.end() + } + } + surface.onFocusChanged = { [weak self] in self?.onFocusChanged?($0) } + surface.input.onInput = { [weak control] in control?.input($0) } + surface.input.onRelease = { [weak control] in control?.release() } + controlTask = Task { [weak self] in + while !Task.isCancelled { + do { try await Task.sleep(for: .seconds(1)) } catch { return } + self?.clipboard?.tick() + if self?.failure != nil { + self?.control.release(reason: "Video decoding failed. Reconnect before controlling.") + } else { + self?.control.tick() + } + } + } + // Statistics callbacks must never delay input lease heartbeats. + diagnosticsTask = Task { [weak self] in + while !Task.isCancelled { + do { try await Task.sleep(for: .seconds(1)) } catch { return } + if let self, self.presented { + let statistics = await self.peer.statistics() + guard !Task.isCancelled else { return } + self.diagnostics.update( + metrics: self.peer.metrics.snapshot(), statistics: statistics, + now: ProcessInfo.processInfo.systemUptime) + } + } + } + peer.onConnectionChanged = { [weak self] in self?.onConnectionChanged?($0) } + surface.metal.onPresented = { [weak self] _ in + guard let self, !self.presented else { return } + self.presented = true + self.onReady?() + } + } + func offer() async throws -> String { try await peer.makeDescription(offer: true).sdp } + func accept(_ answer: String) async throws { try await peer.accept(.init(kind: "answer", sdp: answer)) } + func fit(_ enabled: Bool) { surface.fitToWindow = enabled } + func close() { + guard !closed else { return } + closed = true + clipboard?.close() + control.release(); controlTask?.cancel(); controlTask = nil + diagnosticsTask?.cancel(); diagnosticsTask = nil + let metrics = peer.metrics.snapshot() + Self.logger.info( + "Viewer ended: \(String(describing: metrics.counters), privacy: .public), \(String(describing: metrics.labels), privacy: .public), drawable \(String(describing: self.surface.metal.drawableSize), privacy: .public)" + ) + onReady = nil; onConnectionChanged = nil; surface.stop(); peer.close() + } + private var closed = false +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerClipboard.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerClipboard.swift new file mode 100644 index 000000000..26d61f944 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerClipboard.swift @@ -0,0 +1,56 @@ +import CodevisorScreenSharing +import Foundation +import Observation + +@MainActor +@Observable +public final class ScreenSharingViewerClipboard { + public private(set) var available = false + public private(set) var busy = false + public private(set) var message: String? + @ObservationIgnored private let pasteboard: ScreenSharingPasteboard + @ObservationIgnored private var transfer: ScreenSharingClipboardTransfer! + @ObservationIgnored private var expectedChangeCount: Int? + @ObservationIgnored private var receiving = false + + init(channel: ScreenSharingClipboardChannel, pasteboard: ScreenSharingPasteboard = .init()) { + self.pasteboard = pasteboard + transfer = ScreenSharingClipboardTransfer( + send: { [weak channel] in channel?.send($0) ?? false }, + read: { try pasteboard.read() }, + write: { [weak self] text in + guard let self else { return } + try pasteboard.write(text, expectedChangeCount: self.expectedChangeCount) + }) + transfer.onFinished = { [weak self] error in + guard let self else { return } + self.busy = false + self.message = + error ?? (self.receiving ? "Remote text copied to this Mac’s clipboard." : "Text sent to the host’s clipboard.") + self.expectedChangeCount = nil + } + channel.onMessage = { [weak transfer] in transfer?.receive($0) } + channel.onAvailabilityChanged = { [weak self] available in + self?.available = available + if !available { self?.transfer.cancel(reason: "The clipboard channel closed.") } + } + available = channel.isAvailable + } + + public func sendLocalText() { + guard available, !busy else { return } + do { + let text = try pasteboard.read() + busy = true; receiving = false; message = "Sending clipboard text…" + transfer.sendText(text) + } catch { message = error.localizedDescription } + } + public func getRemoteText() { + guard available, !busy else { return } + busy = true; receiving = true; message = "Receiving clipboard text…" + expectedChangeCount = pasteboard.changeCount + transfer.requestText() + } + func tick() { transfer.tick() } + func close() { available = false; transfer.cancel(reason: "Clipboard transfer ended with the connection.") } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift new file mode 100644 index 000000000..6d6001cdc --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift @@ -0,0 +1,79 @@ +import CodevisorScreenSharing +import Foundation +import Observation + +@MainActor +@Observable +public final class ScreenSharingViewerControl { + public enum State: Equatable { case viewing, requesting, controlling } + public private(set) var state: State = .viewing + public private(set) var available = false + public private(set) var message: String? + @ObservationIgnored var onActiveChanged: ((Bool) -> Void)? + @ObservationIgnored private let send: (ScreenSharingControlMessage) -> Bool + @ObservationIgnored private let now: () -> TimeInterval + @ObservationIgnored private var requestID: UUID? + @ObservationIgnored private var deadline: TimeInterval = 0 + @ObservationIgnored private var lease: UUID? + @ObservationIgnored private var sequence: UInt64 = 0 + + init( + now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, + send: @escaping (ScreenSharingControlMessage) -> Bool + ) { self.now = now; self.send = send } + + public func request() { + guard available, state == .viewing else { return } + let id = UUID() + requestID = id; deadline = now() + 3; state = .requesting; message = nil + if !send(.request(id: id)) { release(reason: "The control channel is unavailable.") } + } + + public func release(reason: String? = nil) { + let oldLease = lease + requestID = nil; lease = nil; state = .viewing; message = reason + onActiveChanged?(false) + if let oldLease { _ = send(.release(lease: oldLease)) } + } + + func setAvailable(_ available: Bool) { + let previouslyAvailable = self.available + self.available = available + if !available { + release(reason: previouslyAvailable ? "Control is unavailable on this connection." : nil) + } else if !previouslyAvailable { + message = nil + } + } + + func receive(_ message: ScreenSharingControlMessage) { + switch message { + case .grant(let request, let id): + guard requestID == request, state == .requesting, available, now() < deadline else { + // Cancellation can cross the host's grant. Release that grant immediately. + _ = send(.release(lease: id)); return + } + requestID = nil; lease = id; sequence = 0; state = .controlling; self.message = nil + onActiveChanged?(true) + case .denied(let request, let reason): + if requestID == request { release(reason: reason) } + case .revoked(let id, let reason): + if lease == id { release(reason: reason) } + default: break + } + } + + func tick() { + if state == .requesting, now() >= deadline { release(reason: "The host did not grant control. Try again.") } + if let lease, !send(.heartbeat(lease: lease)) { release(reason: "The control channel closed.") } + } + + func input(_ event: ScreenSharingInputEvent) { + guard let lease, state == .controlling, event.isValid else { return } + guard sequence < UInt64.max else { release(); return } + sequence += 1 + if !send(.input(lease: lease, sequence: sequence, event: event)) { + release(reason: "Control paused because the connection could not keep up. Request control again.") + } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerDiagnostics.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerDiagnostics.swift new file mode 100644 index 000000000..5f5bef45b --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerDiagnostics.swift @@ -0,0 +1,44 @@ +import CodevisorScreenSharing +import Foundation +import Observation + +@MainActor +@Observable +public final class ScreenSharingViewerDiagnostics { + public private(set) var framesPerSecond: Double? + public private(set) var megabitsPerSecond: Double? + public private(set) var roundTripMilliseconds: Double? + public private(set) var route = "Connecting" + public private(set) var resolution = "Waiting for video" + public private(set) var decoder = "Waiting for decoder" + public private(set) var decodeMilliseconds: Double? + public private(set) var droppedFrames = 0 + @ObservationIgnored private var previous: (time: TimeInterval, frames: Int, bytes: Double)? + + func update(metrics: ScreenSharingMetrics.Snapshot, statistics: [String: String], now: TimeInterval) { + let frames = metrics.counters["presentedFrames", default: 0] + let bytes = statistics.filter { $0.key.hasPrefix("inbound-rtp.") && $0.key.hasSuffix(".bytesReceived") } + .values.compactMap(Double.init).reduce(0, +) + if let previous, now > previous.time { + let elapsed = now - previous.time + framesPerSecond = Double(max(0, frames - previous.frames)) / elapsed + megabitsPerSecond = max(0, bytes - previous.bytes) * 8 / elapsed / 1_000_000 + } + previous = (now, frames, bytes) + roundTripMilliseconds = statistics.first { $0.key.hasSuffix(".currentRoundTripTime") } + .flatMap { Double($0.value) }.map { $0 * 1000 } + let types = statistics.filter { $0.key.hasSuffix(".candidateType") }.values + // A TURN allocation may carry UDP media over a TCP/TLS connection to the + // relay. Report that client transport when the selected candidate exposes it. + let transport = + (statistics.first { $0.key.hasSuffix(".relayProtocol") } + ?? statistics.first { $0.key.hasSuffix(".protocol") })?.value.uppercased() ?? "" + route = + types.isEmpty + ? "Connecting" : (types.contains("relay") ? "Relay" : "Direct") + (transport.isEmpty ? "" : " · " + transport) + resolution = metrics.labels["videoSize"] ?? resolution + decoder = metrics.labels["decoder"] ?? decoder + decodeMilliseconds = metrics.timings["decode"]?.p95Ms + droppedFrames = metrics.counters["renderDrops", default: 0] + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift new file mode 100644 index 000000000..1a0eaa75a --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift @@ -0,0 +1,267 @@ +import AppKit +import CodevisorCore +import Foundation +import Observation + +@MainActor +@Observable +public final class ScreenSharingViewerModel { + public enum Phase: Equatable { case idle, loading, ready, connecting, reconnecting, viewing, suspended, failed } + public private(set) var phase: Phase = .idle + public private(set) var message: String? + public private(set) var displays: [ServerScreenSharingDisplay] = [] + public private(set) var selectedDisplayId: String? + public private(set) var preferences: ScreenSharingPanePreferences + public private(set) var videoView: NSView? + public var control: ScreenSharingViewerControl? { endpoint?.control } + public var clipboard: ScreenSharingViewerClipboard? { endpoint?.clipboard } + public var diagnostics: ScreenSharingViewerDiagnostics? { endpoint?.diagnostics } + @ObservationIgnored public var onFocusChanged: ((Bool) -> Void)? + @ObservationIgnored public var onPreferencesChanged: ((ScreenSharingPanePreferences) -> Void)? + @ObservationIgnored private let client: any CodevisorServerClienting + @ObservationIgnored private let workspaceId: UUID + @ObservationIgnored private let paneId: UUID + @ObservationIgnored private let makePeer: (ServerScreenSharingConnectivity?) throws -> any ScreenSharingViewingPeer + @ObservationIgnored private let sleep: @Sendable (Duration) async throws -> Void + @ObservationIgnored private var task: Task? + @ObservationIgnored private var endpoint: (any ScreenSharingViewingPeer)? + @ObservationIgnored private var generation = 0 + @ObservationIgnored private var visible = false + @ObservationIgnored private var wantsConnection = false + @ObservationIgnored private var activeViewerId: UUID? + + public convenience init( + client: any CodevisorServerClienting, workspaceId: UUID, paneId: UUID, + preferences: ScreenSharingPanePreferences = .init() + ) { + self.init( + client: client, workspaceId: workspaceId, paneId: paneId, preferences: preferences, + makePeer: { try NativeScreenSharingViewingPeer(connectivity: $0) }, sleep: { try await Task.sleep(for: $0) }) + } + + init( + client: any CodevisorServerClienting, workspaceId: UUID, paneId: UUID, + preferences: ScreenSharingPanePreferences = .init(), + makePeer: @escaping (ServerScreenSharingConnectivity?) throws -> any ScreenSharingViewingPeer, + sleep: @escaping @Sendable (Duration) async throws -> Void + ) { + self.client = client; self.workspaceId = workspaceId; self.paneId = paneId + self.preferences = preferences; self.makePeer = makePeer; self.sleep = sleep + } + + public func setVisible(_ visible: Bool) { + guard self.visible != visible else { return } + self.visible = visible + if visible { refresh() } else { cancel(); phase = .suspended } + } + + public func refresh() { + guard visible else { return } + let previous = cancel() + let generation = generation + phase = .loading; message = nil + task = Task { [self] in + await previous?.value + guard isCurrent(generation) else { return } + do { + let reply = try await client.screenSharing(request(.capabilities, viewerId: UUID())) + guard isCurrent(generation) else { return } + guard reply.version == 1, ["available", "busy"].contains(reply.status) else { + throw ViewerError(reply.message ?? "Screen Sharing is unavailable on this Mac.") + } + displays = reply.displays + if let preferred = preferences.preferredDisplayId { + selectedDisplayId = displays.first(where: { $0.id == preferred })?.id + if selectedDisplayId == nil { + throw ViewerError("The selected display is unavailable. Choose another display to connect.") + } + } else { + selectedDisplayId = displays.first?.id + } + guard selectedDisplayId != nil else { throw ViewerError("No displays are available on this Mac.") } + phase = .ready + if wantsConnection { await runConnection(generation: generation) } + } catch { if isCurrent(generation) { phase = .failed; message = serverErrorMessage(error) } } + } + } + + public func selectDisplay(_ id: String) { + guard displays.contains(where: { $0.id == id }) else { return } + preferences.preferredDisplayId = id + selectedDisplayId = id + onPreferencesChanged?(preferences) + if wantsConnection { connect() } else { message = nil; phase = .ready } + } + + public func setFitToWindow(_ fit: Bool) { + preferences.fitToWindow = fit + endpoint?.fit(fit) + onPreferencesChanged?(preferences) + } + + /// Apply a registry update without replacing the live surface or echoing + /// the write. A display change from another client requires a new Connect. + public func applyPreferences(_ preferences: ScreenSharingPanePreferences) { + guard self.preferences != preferences else { return } + let displayChanged = self.preferences.preferredDisplayId != preferences.preferredDisplayId + self.preferences = preferences + endpoint?.fit(preferences.fitToWindow) + if displayChanged { wantsConnection = false; refresh() } + } + + public func connect() { + guard visible, selectedDisplayId != nil else { return } + preferences.preferredDisplayId = selectedDisplayId + onPreferencesChanged?(preferences) + wantsConnection = true + let previous = cancel() + let generation = generation + phase = .connecting; message = nil + task = Task { [self] in + await previous?.value + guard isCurrent(generation) else { return } + await runConnection(generation: generation) + } + } + + public func disconnect() { + wantsConnection = false + cancel() + phase = visible ? .ready : .suspended + message = nil + } + + public func close() async { + wantsConnection = false; visible = false; let pending = cancel(); phase = .suspended; await pending?.value + } + + @discardableResult + private func cancel() -> Task? { + generation += 1 + let previous = task + previous?.cancel() + endpoint?.close() + endpoint = nil; videoView = nil + guard let viewerId = activeViewerId else { return previous } + activeViewerId = nil + let stop = request(.stop, viewerId: viewerId) + let client = client + return Task { + await previous?.value + _ = try? await client.screenSharing(stop) + } + } + + private func runConnection(generation: Int, viewerId: UUID = UUID(), restarts: Int = 0) async { + guard let display = selectedDisplayId else { return } + activeViewerId = viewerId + var started = false + var handingOff = false + var peer: (any ScreenSharingViewingPeer)? + do { + // Fetch fresh short-lived relay credentials at connection time. The + // display picker may have been left open much longer than their lifetime. + let capabilities = try await client.screenSharing(request(.capabilities, viewerId: viewerId)) + try requireCurrent(generation) + guard capabilities.version == 1, ["available", "busy"].contains(capabilities.status) else { + throw ViewerError(capabilities.message ?? "Screen Sharing is unavailable on this Mac.") + } + let created = try makePeer(capabilities.connectivity) + peer = created; endpoint = created; videoView = created.view + created.fit(preferences.fitToWindow) + created.onFocusChanged = { [weak self] in self?.onFocusChanged?($0) } + phase = restarts == 0 ? .connecting : .reconnecting + created.onReady = { [weak self] in + guard let self, self.isCurrent(generation), [.connecting, .reconnecting].contains(self.phase) else { return } + self.phase = .viewing + self.message = nil + } + created.onConnectionChanged = { [weak self] state in + guard let self, self.isCurrent(generation), ["failed", "disconnected", "closed"].contains(state) else { return } + if self.phase == .viewing, restarts < 3, state != "closed" { + handingOff = true + self.recover(generation: generation, viewerId: viewerId, restarts: restarts + 1) + return + } + self.phase = .failed + self.message = "The screen-sharing connection ended. Reconnect to continue." + self.task?.cancel() + } + let offer = try await created.offer() + try requireCurrent(generation) + started = true + let reply = try await client.screenSharing( + request( + restarts == 0 ? .start : .restart, + viewerId: viewerId, displayId: display, offer: offer)) + try requireCurrent(generation) + guard reply.version == 1, reply.status == "connecting", let answer = reply.answer else { + throw ViewerError(reply.message ?? "This Mac cannot start screen sharing right now.") + } + try await created.accept(answer) + try requireCurrent(generation) + var heartbeatsBeforeVideo = 0 + while isCurrent(generation) { + try await sleep(.seconds(8)) + try requireCurrent(generation) + let reply = try await client.screenSharing(request(.heartbeat, viewerId: viewerId)) + try requireCurrent(generation) + guard reply.version == 1, ["connecting", "viewing"].contains(reply.status) else { + throw ViewerError(reply.message ?? "Screen sharing ended on the host Mac.") + } + if let failure = created.failure { throw ViewerError(failure) } + if phase == .connecting || phase == .reconnecting { + heartbeatsBeforeVideo += 1 + guard heartbeatsBeforeVideo < 3 else { + throw ViewerError( + "No video arrived. Check the connection between these Macs or the configured relay, then retry.") + } + } + } + } catch { + if isCurrent(generation), !isTaskCancellation(error) { phase = .failed; message = serverErrorMessage(error) } + } + peer?.close() + if started, !handingOff { + // URLSession inherits task cancellation. Cleanup needs a fresh task so + // hiding the tab can still deliver the authenticated stop request. + let stop = request(.stop, viewerId: viewerId) + let client = client + await Task { _ = try? await client.screenSharing(stop) }.value + } + if generation == self.generation { endpoint = nil; videoView = nil; activeViewerId = nil } + } + + private func recover(generation: Int, viewerId: UUID, restarts: Int) { + guard isCurrent(generation), visible, wantsConnection else { return } + let previous = task + self.generation += 1 + let next = self.generation + previous?.cancel() + endpoint?.close() // Releases every held input and discards the old render mailbox. + endpoint = nil; videoView = nil + phase = .reconnecting + message = "Reconnecting to this Mac…" + task = Task { [self] in + await previous?.value + guard isCurrent(next) else { return } + await runConnection(generation: next, viewerId: viewerId, restarts: restarts) + } + } + + private func isCurrent(_ generation: Int) -> Bool { visible && generation == self.generation && !Task.isCancelled } + private func requireCurrent(_ generation: Int) throws { if !isCurrent(generation) { throw CancellationError() } } + private func request( + _ operation: ServerScreenSharingRequest.Operation, viewerId: UUID, + displayId: String? = nil, offer: String? = nil + ) -> ServerScreenSharingRequest { + .init( + operation: operation, workspaceId: workspaceId, paneId: paneId, viewerId: viewerId, displayId: displayId, + offer: offer) + } + private struct ViewerError: LocalizedError { + let message: String + init(_ message: String) { self.message = message } + var errorDescription: String? { message } + } +} diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Dispatch.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Dispatch.swift index 695a2be84..4433e7c78 100644 --- a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Dispatch.swift +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge+Dispatch.swift @@ -5,6 +5,16 @@ import Foundation extension ComputerUseBridge { func handle(_ message: [String: Any]) throws -> [String: Any] { let type = message["type"] as? String + if type == "screenSharing" { + guard let request = message["request"] as? [String: Any] else { + throw BridgeError("Missing screen-sharing request.") + } + let data = try screenSharing.request(JSONSerialization.data(withJSONObject: request)) + guard let response = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BridgeError("Invalid screen-sharing response.") + } + return response + } let sessionID = message["sessionId"] as? String ?? "" let agentLabel = message["agentLabel"] as? String if type == "closeSession" { diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge.swift index 4161a42ea..e1dccb339 100644 --- a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge.swift +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/Server/ComputerUseBridge.swift @@ -41,6 +41,7 @@ public final class ComputerUseBridge: @unchecked Sendable { attributes: .concurrent ) private let supportDirectory: URL + let screenSharing = NativeScreenSharingBridge() let recordings: ComputerUseRecordings let lock = NSLock() private var listener: Int32 = -1 @@ -106,6 +107,7 @@ public final class ComputerUseBridge: @unchecked Sendable { listener = descriptor let ready = Configuration(socketPath: socketPath, token: token) configuration = ready + screenSharing.start() listenerQueue.async { [weak self] in self?.acceptLoop(descriptor, token: token) } return ready } @@ -128,6 +130,7 @@ public final class ComputerUseBridge: @unchecked Sendable { if let socketPath { unlink(socketPath) } ComputerUsePresentation.endAll() recordings.end() + screenSharing.stop() } private func bindSocket(_ descriptor: Int32, path: String) throws { diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingConnectivityTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingConnectivityTests.swift new file mode 100644 index 000000000..1f00cde9d --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingConnectivityTests.swift @@ -0,0 +1,44 @@ +import CodevisorClient +import Foundation +import Testing +@testable import CodevisorCoreMac + +struct ScreenSharingConnectivityTests { + @Test func defaultConfigurationDoesNotContactThirdPartyServers() throws { + let result = try ScreenSharingHostConnectivity(environment: [:], now: { 1_700_000_000 }).make(viewerId: UUID()) + #expect(result.servers.isEmpty && !result.relayOnly) + } + + @Test func credentialsUseTURNRESTHMACAndExpireWithoutExposingTheSharedSecret() throws { + let configuration = ScreenSharingHostConnectivity( + environment: [ + "CODEVISOR_SCREEN_SHARING_STUN_URLS": "stun:relay.example.test:3478", + "CODEVISOR_SCREEN_SHARING_TURN_URLS": + "turn:relay.example.test:3478?transport=udp, turns:relay.example.test:443?transport=tcp", + "CODEVISOR_SCREEN_SHARING_TURN_SECRET": "fixture-secret", + "CODEVISOR_SCREEN_SHARING_RELAY_ONLY": "1", + ], now: { 1_700_000_000 }) + let result = try configuration.make(viewerId: UUID(uuidString: "00000000-0000-0000-0000-000000000001")!) + #expect(result.expiresAt == 1_700_000_300) + #expect(result.relayOnly) + #expect(result.servers[1].credential == "IAVGLGXf33m6RhQKjUfVHiRskOc=") + #expect(result.servers[1].username == "1700000300:codevisor:00000000-0000-0000-0000-000000000001") + let wire = String(decoding: try JSONEncoder().encode(result), as: UTF8.self) + #expect(!wire.contains("fixture-secret")) + let other = try configuration.make(viewerId: UUID(uuidString: "00000000-0000-0000-0000-000000000002")!) + #expect(other.servers[1].credential != result.servers[1].credential) + } + + @Test func incompleteOrMalformedRelayConfigurationFailsBeforeCreatingAPeer() { + for environment in [ + ["CODEVISOR_SCREEN_SHARING_TURN_URLS": "turn:relay.example.test"], + ["CODEVISOR_SCREEN_SHARING_RELAY_ONLY": "1"], + ["CODEVISOR_SCREEN_SHARING_RELAY_ONLY": "typo"], + ["CODEVISOR_SCREEN_SHARING_STUN_URLS": "https://example.test"], + ] { + #expect(throws: (any Error).self) { + try ScreenSharingHostConnectivity(environment: environment, now: { 1_700_000_000 }).make(viewerId: UUID()) + } + } + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingControlTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingControlTests.swift new file mode 100644 index 000000000..803451f4c --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingControlTests.swift @@ -0,0 +1,184 @@ +import AppKit +import CodevisorScreenSharing +import Testing +@testable import CodevisorCoreMac + +@MainActor +struct ScreenSharingControlTests { + @Test func leaseExpiryReleasesAllHeldInputAndRejectsDelayedPackets() throws { + let fixture = HostControlFixture() + let lease = try fixture.acquire() + fixture.receive(lease, 1, .key(code: 55, down: true, repeatKey: false, modifiers: 8)) + fixture.receive(lease, 2, .key(code: 0, down: true, repeatKey: false, modifiers: 8)) + fixture.receive(lease, 3, .button(.init(x: 0.25, y: 0.75), button: 0, down: true, clicks: 1, modifiers: 8)) + fixture.time = 2.999 + fixture.host.checkDeadline() + #expect(fixture.events.count == 3) + fixture.time = 3 + fixture.host.checkDeadline() + #expect( + fixture.events.suffix(3) == [ + .key(code: 0, down: false, repeatKey: false, modifiers: 8), + .key(code: 55, down: false, repeatKey: false, modifiers: 0), + .button(.init(x: 0.25, y: 0.75), button: 0, down: false, clicks: 1, modifiers: 0), + ]) + #expect(fixture.host.heldKeys.isEmpty && fixture.host.heldButtons.isEmpty) + fixture.host.revoke("Duplicate close") + fixture.host.receive(.heartbeat(lease: lease)) + fixture.receive(lease, 4, .key(code: 1, down: true, repeatKey: false, modifiers: 0)) + #expect(fixture.events.count == 6) + let fresh = try fixture.acquire() + #expect(fresh != lease) + fixture.receive(lease, 5, .key(code: 1, down: true, repeatKey: false, modifiers: 0)) + #expect(fixture.events.count == 6) + fixture.host.revoke("Test complete") + } + + @Test func heartbeatRequiresMatchingLeaseAndCannotReviveAnExpiredGrant() throws { + let fixture = HostControlFixture() + let lease = try fixture.acquire() + fixture.time = 2 + fixture.host.receive(.heartbeat(lease: UUID())) + fixture.host.receive(.heartbeat(lease: lease)) + fixture.time = 4.999 + fixture.host.checkDeadline() + #expect(fixture.host.lease == lease) + fixture.time = 5 + fixture.host.receive(.heartbeat(lease: lease)) + #expect(fixture.host.lease == nil) + } + + @Test func deniedPermissionAndRevocationCannotInjectOrKeepKeysHeld() throws { + let fixture = HostControlFixture() + fixture.reason = "Accessibility required" + fixture.host.receive(.request(id: UUID())) + #expect(fixture.host.lease == nil && fixture.events.isEmpty) + fixture.reason = nil + let lease = try fixture.acquire() + fixture.receive(lease, 1, .key(code: 0, down: true, repeatKey: false, modifiers: 0)) + fixture.reason = "Permission revoked" + fixture.receive(lease, 2, .key(code: 1, down: true, repeatKey: false, modifiers: 0)) + #expect( + fixture.events == [ + .key(code: 0, down: true, repeatKey: false, modifiers: 0), + .key(code: 0, down: false, repeatKey: false, modifiers: 0), + ]) + #expect(fixture.host.lease == nil) + } + + @Test func duplicateAndOutOfOrderEventsDoNotReplayTransitions() throws { + let fixture = HostControlFixture() + let lease = try fixture.acquire() + let down = ScreenSharingInputEvent.key(code: 0, down: true, repeatKey: false, modifiers: 0) + fixture.receive(lease, 1, down) + fixture.receive(lease, 1, down) + fixture.receive(lease, 2, down) + fixture.receive(lease, 3, .key(code: 0, down: true, repeatKey: true, modifiers: 0)) + fixture.receive(lease, 2, .key(code: 0, down: false, repeatKey: false, modifiers: 0)) + #expect(fixture.events.count == 2) + fixture.host.receive(.release(lease: lease)) + #expect(fixture.events.count == 3) + #expect(fixture.host.heldKeys.isEmpty) + } + + @Test func invalidInputRevokesControlAndReleasesAButton() throws { + let fixture = HostControlFixture() + let lease = try fixture.acquire() + fixture.receive(lease, 1, .button(.init(x: 0, y: 0), button: 1, down: true, clicks: 1, modifiers: 0)) + fixture.receive(lease, 2, .move(.init(x: -1, y: 0), modifiers: 0)) + #expect(fixture.host.lease == nil) + #expect(fixture.events.count == 2) + #expect(fixture.host.heldButtons.isEmpty) + } + + @Test func cancelledAndTimedOutRequestsReleaseLateGrants() throws { + var time = 0.0 + var sent: [ScreenSharingControlMessage] = [] + let viewer = ScreenSharingViewerControl( + now: { time }, + send: { + sent.append($0); return true + }) + viewer.setAvailable(true) + viewer.request() + guard case .request(let request) = try #require(sent.last) else { Issue.record("Missing request"); return } + viewer.release() + let lease = UUID() + viewer.receive(.grant(request: request, lease: lease)) + #expect(viewer.state == .viewing) + #expect(sent.last == .release(lease: lease)) + viewer.request() + guard case .request(let second) = try #require(sent.last) else { Issue.record("Missing request"); return } + time = 3 + viewer.receive(.grant(request: second, lease: lease)) + viewer.tick() + #expect(viewer.state == .viewing) + #expect(viewer.message?.contains("did not grant") == true) + } + + @Test func congestionStopsViewerInputAndRequiresExplicitControlAgain() throws { + var writable = true + var sent: [ScreenSharingControlMessage] = [] + let viewer = ScreenSharingViewerControl(send: { + sent.append($0); return writable + }) + viewer.setAvailable(true); viewer.request() + guard case .request(let request) = try #require(sent.last) else { Issue.record("Missing request"); return } + let lease = UUID() + viewer.receive(.grant(request: request, lease: lease)) + #expect(viewer.state == .controlling) + viewer.input(.move(.init(x: 0, y: 0), modifiers: 0)) + #expect(sent.last == .input(lease: lease, sequence: 1, event: .move(.init(x: 0, y: 0), modifiers: 0))) + writable = false + viewer.tick() + #expect(viewer.state == .viewing) + let count = sent.count + viewer.input(.key(code: 0, down: true, repeatKey: false, modifiers: 0)) + writable = true; viewer.tick() + #expect(sent.count == count) + #expect(viewer.state == .viewing) + } + + @Test func quartzMappingUsesGlobalPointsAndProducesNativeDragAndReleaseEvents() throws { + var events: [CGEvent] = [] + let injector = ScreenSharingInputInjector(displayBounds: CGRect(x: -1440, y: -900, width: 1440, height: 900)) { + events.append($0) + } + let point = ScreenSharingPointer(x: 0.25, y: 0.75) + injector.post(.button(point, button: 0, down: true, clicks: 2, modifiers: 8)) + injector.post(.move(point, modifiers: 8)) + injector.post(.button(point, button: 0, down: false, clicks: 2, modifiers: 0)) + injector.post(.move(point, modifiers: 0)) + #expect(events.map(\.type) == [.leftMouseDown, .leftMouseDragged, .leftMouseUp, .mouseMoved]) + #expect(events.allSatisfy { $0.location == CGPoint(x: -1080, y: -225) }) + #expect(events[0].flags == .maskCommand) + #expect(events[0].getIntegerValueField(.mouseEventClickState) == 2) + #expect(events.allSatisfy { $0.getIntegerValueField(.eventSourceUserData) == ScreenSharingInputInjector.eventTag }) + injector.post(.key(code: 0, down: true, repeatKey: true, modifiers: 1)) + #expect(events.last?.getIntegerValueField(.keyboardEventAutorepeat) == 1) + #expect(events.last?.flags == .maskShift) + #expect(injector.location(.init(x: 1, y: 1)).x < 0) + #expect(injector.location(.init(x: 1, y: 1)).y < 0) + } +} + +@MainActor +private final class HostControlFixture { + var time = 0.0 + var reason: String? + var events: [ScreenSharingInputEvent] = [] + var sent: [ScreenSharingControlMessage] = [] + lazy var host = ScreenSharingHostControl( + now: { [unowned self] in time }, availability: { [unowned self] in reason }, + inject: { [unowned self] in events.append($0) }, + send: { [unowned self] in + sent.append($0); return true + }) + func acquire() throws -> UUID { + host.receive(.request(id: UUID())) + return try #require(host.lease) + } + func receive(_ lease: UUID, _ sequence: UInt64, _ event: ScreenSharingInputEvent) { + host.receive(.input(lease: lease, sequence: sequence, event: event)) + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingDiagnosticsTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingDiagnosticsTests.swift new file mode 100644 index 000000000..6a95eab34 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingDiagnosticsTests.swift @@ -0,0 +1,52 @@ +import CodevisorScreenSharing +import Testing +@testable import CodevisorCoreMac + +@MainActor +struct ScreenSharingDiagnosticsTests { + @Test func relayTransportDescribesTheClientConnectionRatherThanItsUDPAllocation() { + let diagnostics = ScreenSharingViewerDiagnostics() + diagnostics.update( + metrics: ScreenSharingMetrics().snapshot(), + statistics: [ + "local-candidate.selected.candidateType": "relay", + "local-candidate.selected.protocol": "udp", + "local-candidate.selected.relayProtocol": "tls", + ], now: 10) + #expect(diagnostics.route == "Relay · TLS") + } + @Test func intervalRatesAndSelectedRouteUseOnlyMeasuredValues() { + let metrics = ScreenSharingMetrics() + let diagnostics = ScreenSharingViewerDiagnostics() + diagnostics.update(metrics: metrics.snapshot(), statistics: [:], now: 10) + #expect(diagnostics.framesPerSecond == nil) + #expect(diagnostics.roundTripMilliseconds == nil) + #expect(diagnostics.route == "Connecting") + metrics.increment("presentedFrames", by: 120) + metrics.increment("renderDrops", by: 3) + metrics.observe("decode", milliseconds: 2.5) + metrics.label("videoSize", "1920 × 1080") + metrics.label("decoder", "VideoToolbox hardware") + diagnostics.update( + metrics: metrics.snapshot(), + statistics: [ + "inbound-rtp.video.bytesReceived": "3000000", + "candidate-pair.selected.currentRoundTripTime": "0.012", + "local-candidate.selected.candidateType": "relay", + "remote-candidate.selected.candidateType": "host", + "local-candidate.selected.protocol": "udp", + ], now: 12) + #expect(diagnostics.framesPerSecond == 60) + #expect(diagnostics.megabitsPerSecond == 12) + #expect(diagnostics.roundTripMilliseconds == 12) + #expect(diagnostics.route == "Relay · UDP") + #expect(diagnostics.resolution == "1920 × 1080") + #expect(diagnostics.decodeMilliseconds == 2.5) + #expect(diagnostics.droppedFrames == 3) + // Reset or missing RTC counters cannot produce negative bitrate. + diagnostics.update(metrics: metrics.snapshot(), statistics: [:], now: 13) + #expect(diagnostics.framesPerSecond == 0) + #expect(diagnostics.megabitsPerSecond == 0) + #expect(diagnostics.roundTripMilliseconds == nil) + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostLeaseTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostLeaseTests.swift new file mode 100644 index 000000000..f7de7544a --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostLeaseTests.swift @@ -0,0 +1,63 @@ +import CodevisorCore +import Foundation +import Testing +@testable import CodevisorCoreMac + +struct ScreenSharingHostLeaseTests { + @Test func replacementMustStillBeAuthorizedAfterAsynchronousTeardownAndDisplayLookup() throws { + let first = request() + var lease = ScreenSharingHostLease() + _ = lease.reserve(.init(first), now: 100) + #expect(lease.replacementPermit(.init(request()), revision: 7, now: 110) == nil) + let permit = try #require(lease.replacementPermit(.init(first), revision: 7, now: 110)) + _ = lease.release(.init(first)) // The old peer has been torn down. + #expect(permit.isValid(revision: 7, now: 124.999)) + #expect(!permit.isValid(revision: 7, now: 125)) + #expect(!permit.isValid(revision: 8, now: 111)) // Host stop while enumeration was suspended. + #expect(lease.replacementPermit(.init(first), revision: 7, now: 111) == nil) + } + @Test func secondViewerCannotDisplaceOrRenewAnotherLease() { + let first = request() + let second = request() + var lease = ScreenSharingHostLease() + let reserved = lease.reserve(.init(first), now: 100) + let displaced = lease.reserve(.init(second), now: 101) + let foreignRenewed = lease.renew(.init(second), now: 102) + let foreignReleased = lease.release(.init(second)) + #expect(reserved) + #expect(!displaced) + #expect(!foreignRenewed) + #expect(!foreignReleased) + #expect(lease.owner == .init(first)) + #expect(!lease.isExpired(now: 124)) + #expect(lease.isExpired(now: 125)) + let expiredRenewed = lease.renew(.init(first), now: 125) + let released = lease.release(.init(first)) + let nextReserved = lease.reserve(.init(second), now: 125) + #expect(!expiredRenewed) + #expect(released) + #expect(nextReserved) + } + + @Test func heartbeatExtendsOnlyAnUnexpiredMatchingWorkspaceAndPane() { + let first = request() + let wrongPane = ServerScreenSharingRequest( + operation: .heartbeat, workspaceId: first.workspaceId, + paneId: UUID(), viewerId: first.viewerId) + var lease = ScreenSharingHostLease() + let reserved = lease.reserve(.init(first), now: 0) + let foreignRenewed = lease.renew(.init(wrongPane), now: 5) + let renewed = lease.renew(.init(first), now: 24) + #expect(reserved) + #expect(!foreignRenewed) + #expect(renewed) + #expect(!lease.isExpired(now: 48)) + #expect(lease.isExpired(now: 49)) + let released = lease.release(.init(first)) + #expect(released) + #expect(!lease.isExpired(now: 100)) + } + private func request() -> ServerScreenSharingRequest { + .init(operation: .start, workspaceId: UUID(), paneId: UUID(), viewerId: UUID()) + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostRequestTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostRequestTests.swift new file mode 100644 index 000000000..ea79502c5 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingHostRequestTests.swift @@ -0,0 +1,157 @@ +import CodevisorCore +import CodevisorTestSupport +import Foundation +import Testing +@testable import CodevisorCoreMac + +@MainActor +struct ScreenSharingHostRequestTests { + @Test func stopDuringInitialDisplayLookupPreventsTheStartFromContinuing() async { + let entered = TestSignal() + let release = TestSignal() + let host = makeHost { + entered.signal() + await release.wait() + return [] + } + let start = request() + let pending = Task { await host.handle(start) } + await entered.wait() + let stopped = await host.handle(stop(start)) + #expect(stopped.status == "stopped") + release.signal() + let result = await pending.value + // An empty enumeration deliberately avoids creating a native peer. Reaching display selection at all is wrong + // after Stop; the same continuation with an available display would otherwise construct a live media session. + #expect(result.status == "stopped") + await host.shutdown() + } + + @Test func aLaterExplicitStartIsNotCancelledByTheEarlierStop() async { + let entered = TestSignal() + let release = TestSignal() + var lookups = 0 + let host = makeHost { + lookups += 1 + entered.signal() + await release.wait() + return [] + } + let start = request() + let pending = Task { await host.handle(start) } + await entered.wait() + _ = await host.handle(stop(start)) + release.signal() + #expect(await pending.value.status == "stopped") + // The signal remains open. A newly issued Start, even for the same owner, gets its own admission. + let next = await host.handle(start) + #expect(next.status == "unavailable") + #expect(lookups == 2) + await host.shutdown() + } + + enum ForeignIdentity: CaseIterable, Sendable { case workspace, pane, viewer } + + @Test(arguments: ForeignIdentity.allCases) + func anotherOwnersStopDoesNotCancelThePendingStart(_ identity: ForeignIdentity) async { + let entered = TestSignal() + let release = TestSignal() + let host = makeHost { + entered.signal() + await release.wait() + return [] + } + let start = request() + let pending = Task { await host.handle(start) } + await entered.wait() + let foreign = ServerScreenSharingRequest( + operation: .stop, workspaceId: identity == .workspace ? UUID() : start.workspaceId, + paneId: identity == .pane ? UUID() : start.paneId, + viewerId: identity == .viewer ? UUID() : start.viewerId) + _ = await host.handle(foreign) + release.signal() + // Display selection is reached: a different workspace, pane or viewer cannot cancel this owner. + #expect(await pending.value.status == "unavailable") + await host.shutdown() + } + + @Test func stoppingOneOfTwoPendingOwnersDoesNotCancelTheOther() async { + let entered = TestSignal() + let release = TestSignal() + let host = makeHost { + entered.signal() + await release.wait() + return [] + } + let first = request() + let second = request() + let pendingFirst = Task { await host.handle(first) } + await entered.wait() + let pendingSecond = Task { await host.handle(second) } + await entered.wait(for: 2) + _ = await host.handle(stop(first)) + release.signal() + #expect(await pendingFirst.value.status == "stopped") + #expect(await pendingSecond.value.status == "unavailable") + await host.shutdown() + } + + @Test func shutdownInvalidatesPendingDisplayLookup() async { + let entered = TestSignal() + let release = TestSignal() + let host = makeHost { + entered.signal() + await release.wait() + return [] + } + let start = request() + let pending = Task { await host.handle(start) } + await entered.wait() + await host.shutdown() + release.signal() + #expect(await pending.value.status == "stopped") + #expect(await host.handle(start).status == "stopped") + } + + @Test func aSystemStopInvalidatesEveryPendingOwner() async { + let entered = TestSignal() + let release = TestSignal() + let host = makeHost { + entered.signal() + await release.wait() + return [] + } + let first = request() + let second = request() + let pendingFirst = Task { await host.handle(first) } + await entered.wait() + let pendingSecond = Task { await host.handle(second) } + await entered.wait(for: 2) + // Exercise the same main-actor entry as the OS notification adapter without posting a process-global event. + host.systemStopped() + release.signal() + #expect(await pendingFirst.value.status == "stopped") + #expect(await pendingSecond.value.status == "stopped") + // A system stop invalidates attempts already admitted; it does not shut down the host service permanently. + #expect(await host.handle(request()).status == "unavailable") + await host.shutdown() + } + + private func makeHost( + enumerate: @escaping () async throws -> [ScreenSharingHostService.Display] + ) -> ScreenSharingHostService { + ScreenSharingHostService( + captureAccess: { true }, notificationCenter: NotificationCenter(), + workspaceNotificationCenter: NotificationCenter(), enumerateDisplays: enumerate) + } + + private func request() -> ServerScreenSharingRequest { + .init( + operation: .start, workspaceId: UUID(), paneId: UUID(), viewerId: UUID(), displayId: "display", + offer: "a=fingerprint:sha-256 fixture") + } + + private func stop(_ start: ServerScreenSharingRequest) -> ServerScreenSharingRequest { + .init(operation: .stop, workspaceId: start.workspaceId, paneId: start.paneId, viewerId: start.viewerId) + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingPasteboardTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingPasteboardTests.swift new file mode 100644 index 000000000..dcf9a5a68 --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingPasteboardTests.swift @@ -0,0 +1,30 @@ +import AppKit +import Testing +@testable import CodevisorCoreMac + +@MainActor +struct ScreenSharingPasteboardTests { + @Test func aNewerLocalCopyIsNotOverwrittenByAnInFlightDownload() throws { + let pasteboard = NSPasteboard(name: .init("screen-sharing-test-\(UUID())")) + defer { pasteboard.releaseGlobally() } + let adapter = ScreenSharingPasteboard(pasteboard) + try adapter.write("original") + let expected = adapter.changeCount + try adapter.write("newer local copy") + #expect(throws: (any Error).self) { try adapter.write("late remote copy", expectedChangeCount: expected) } + #expect(try adapter.read() == "newer local copy") + try adapter.write("remote ✓", expectedChangeCount: adapter.changeCount) + #expect(try adapter.read() == "remote ✓") + } + + @Test func unsupportedAndOversizedContentsDoNotBecomeTextTransfers() throws { + let pasteboard = NSPasteboard(name: .init("screen-sharing-test-\(UUID())")) + defer { pasteboard.releaseGlobally() } + let adapter = ScreenSharingPasteboard(pasteboard) + pasteboard.setData(Data([1, 2, 3]), forType: .png) + #expect(throws: (any Error).self) { try adapter.read() } + pasteboard.clearContents() + pasteboard.setString(String(repeating: "a", count: 65537), forType: .string) + #expect(throws: (any Error).self) { try adapter.read() } + } +} diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift new file mode 100644 index 000000000..c1b26435f --- /dev/null +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift @@ -0,0 +1,273 @@ +import AppKit +import CodevisorCore +import CodevisorTestSupport +import Foundation +import Testing +@testable import CodevisorCoreMac + +@MainActor +struct ScreenSharingViewerTests { + @Test func networkRecoveryUsesFreshMediaAndTheSameLeaseWithoutReacquiringControl() async throws { + let transport = SharingTransport() + let clock = TestClock() + var peers: [SharingPeer] = [] + let model = makeModel(transport, clock: clock) { + let peer = SharingPeer(); peers.append(peer); return peer + } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + let first = try #require(peers.first) + let staleFrame = first.onReady + first.onConnectionChanged?("disconnected") + #expect(model.phase == .reconnecting && first.closed) + #expect(model.videoView == nil) + staleFrame?() + #expect(model.phase == .reconnecting) + await awaitObserved { model.phase == .viewing && peers.count == 2 } + let requests = await transport.requests + let start = try #require(requests.first { $0.operation == .start }) + let restart = try #require(requests.first { $0.operation == .restart }) + #expect(start.viewerId == restart.viewerId && start.displayId == restart.displayId) + #expect(requests.filter { $0.operation == .start }.count == 1) + #expect(requests.filter { $0.operation == .stop }.isEmpty) + #expect(requests.filter { $0.operation == .capabilities && $0.viewerId == start.viewerId }.count == 2) + #expect(peers[1].control.state == .viewing) + await model.close() + #expect(peers.allSatisfy { $0.closed }) + #expect(clock.pendingCount == 0) + } + + @Test func aHostStopDuringRecoveryCannotBecomeANewStart() async { + let transport = SharingTransport(restartStatus: "stopped") + let clock = TestClock() + var peers: [SharingPeer] = [] + let model = makeModel(transport, clock: clock) { + let peer = SharingPeer(); peers.append(peer); return peer + } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + peers[0].onConnectionChanged?("disconnected") + await awaitObserved { model.phase == .failed } + #expect(await transport.requests.filter { $0.operation == .start }.count == 1) + #expect(await transport.requests.filter { $0.operation == .restart }.count == 1) + await model.close() + #expect(peers.allSatisfy { $0.closed }) + #expect(clock.pendingCount == 0) + } + + @Test func hidingDuringStartRejectsLateCallbacksAndStopsBeforeResuming() async throws { + let transport = SharingTransport(blockFirstStart: true) + let clock = TestClock() + var peers: [SharingPeer] = [] + let model = makeModel(transport, clock: clock) { + let peer = SharingPeer(); peers.append(peer); return peer + } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + #expect(peers.isEmpty) + model.connect() + await transport.started.wait() + let staleCallback = peers.first?.onReady + model.setVisible(false) + #expect(model.videoView == nil) + #expect(model.phase == .suspended) + staleCallback?() + #expect(model.phase == .suspended) + model.setVisible(true) + transport.releaseFirstStart.signal() + await clock.waitForSleep(.seconds(8)) + #expect(model.phase == .viewing) + let requests = await transport.requests + let starts = requests.filter { $0.operation == .start } + #expect(starts.count == 2) + #expect(starts[0].viewerId != starts[1].viewerId) + let oldStop = try #require(requests.firstIndex { $0.operation == .stop && $0.viewerId == starts[0].viewerId }) + let newStart = try #require(requests.firstIndex { $0.operation == .start && $0.viewerId == starts[1].viewerId }) + #expect(oldStop < newStart) + await model.close() + #expect(clock.pendingCount == 0) + #expect(peers.allSatisfy { $0.closed }) + #expect(await transport.stopWasCancelled == false) + } + + @Test func aMissingPreferredDisplayNeverSelectsAnotherDisplay() async { + let transport = SharingTransport() + let model = makeModel(transport, clock: TestClock(), preferences: .init(preferredDisplayId: "missing")) { + SharingPeer() + } + model.setVisible(true) + await awaitObserved { model.phase == .failed } + #expect(model.selectedDisplayId == nil) + #expect(model.displays.count == 1) + #expect(await transport.requests.allSatisfy { $0.operation == .capabilities }) + await model.close() + } + + @Test func hostStopEndsViewingAtTheHeartbeatBoundary() async { + let transport = SharingTransport(heartbeatStatus: "stopped") + let clock = TestClock() + let peer = SharingPeer() + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.setFitToWindow(false) + model.connect() + await clock.waitForSleep(.seconds(8)) + #expect(model.phase == .viewing) + #expect(peer.fitToWindow == false) + clock.advance(by: .seconds(7)) + #expect(await transport.requests.filter { $0.operation == .heartbeat }.isEmpty) + clock.advance(by: .seconds(1)) + await awaitObserved { model.phase == .failed } + await model.close() + #expect(peer.closed) + #expect(model.videoView == nil) + #expect(clock.pendingCount == 0) + } + + @Test func missingVideoTimesOutAndLateFirstFrameCannotReviveTheConnection() async { + let transport = SharingTransport() + let clock = TestClock() + let peer = SharingPeer() + peer.deliversVideo = false + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + let lateFrame = peer.onReady + for _ in 0..<2 { + clock.advance(by: .seconds(8)) + await clock.waitForSleep(.seconds(8)) + #expect(model.phase == .connecting) + } + clock.advance(by: .seconds(7)) + #expect(model.phase == .connecting) + clock.advance(by: .seconds(1)) + await awaitObserved { model.phase == .failed } + lateFrame?() + #expect(model.phase == .failed) + #expect(model.message?.contains("No video arrived") == true) + await model.close() + #expect(peer.closed) + #expect(clock.pendingCount == 0) + } + + @Test func decoderFailureEndsViewingAtTheNextHeartbeat() async { + let transport = SharingTransport() + let clock = TestClock() + let peer = SharingPeer() + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + peer.failure = "Fixture hardware decoder failed" + clock.advance(by: .seconds(8)) + await awaitObserved { model.phase == .failed } + #expect(model.message == peer.failure) + await model.close() + #expect(peer.closed) + } + + @Test func incomingPreferencesUpdateTheLiveSurfaceWithoutEchoOrReconnect() async { + let transport = SharingTransport() + let clock = TestClock() + let peer = SharingPeer() + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + var echoed = false + model.onPreferencesChanged = { _ in echoed = true } + var preferences = model.preferences + preferences.fitToWindow = false + model.applyPreferences(preferences) + model.applyPreferences(preferences) + #expect(model.videoView === peer.view) + #expect(model.phase == .viewing) + #expect(!peer.fitToWindow) + #expect(!peer.closed) + #expect(!echoed) + #expect(await transport.requests.filter { $0.operation == .start }.count == 1) + + preferences.preferredDisplayId = "missing" + model.applyPreferences(preferences) + await awaitObserved { model.phase == .failed } + #expect(model.videoView == nil) + #expect(peer.closed) + #expect(await transport.requests.filter { $0.operation == .start }.count == 1) + #expect(!echoed) + await model.close() + } + + private func makeModel( + _ transport: SharingTransport, clock: TestClock, + preferences: ScreenSharingPanePreferences = .init(), makePeer: @escaping () -> SharingPeer + ) -> ScreenSharingViewerModel { + let client = CodevisorServerClient(config: .init(requestTransport: transport)) + return ScreenSharingViewerModel( + client: client, workspaceId: UUID(), paneId: UUID(), preferences: preferences, + makePeer: { _ in makePeer() }, sleep: { try await clock.sleep(for: $0) }) + } +} + +@MainActor +private final class SharingPeer: ScreenSharingViewingPeer { + let view = NSView() + let control = ScreenSharingViewerControl(send: { _ in true }) + var clipboard: ScreenSharingViewerClipboard? { nil } + let diagnostics = ScreenSharingViewerDiagnostics() + var failure: String? + var deliversVideo = true + var onReady: (() -> Void)? + var onConnectionChanged: ((String) -> Void)? + var onFocusChanged: ((Bool) -> Void)? + var closed = false + var fitToWindow = true + func offer() async throws -> String { "fixture offer" } + func accept(_ answer: String) async throws { if deliversVideo { onReady?() } } + func fit(_ enabled: Bool) { fitToWindow = enabled } + func close() { closed = true; onReady = nil; onConnectionChanged = nil } +} + +private actor SharingTransport: ServerRequestTransport { + nonisolated let started = TestSignal() + nonisolated let releaseFirstStart = TestSignal() + private let blockFirstStart: Bool + private let heartbeatStatus: String + private let restartStatus: String + private(set) var requests: [ServerScreenSharingRequest] = [] + private(set) var stopWasCancelled = false + init(blockFirstStart: Bool = false, heartbeatStatus: String = "viewing", restartStatus: String = "connecting") { + self.blockFirstStart = blockFirstStart; self.heartbeatStatus = heartbeatStatus + self.restartStatus = restartStatus + } + func data(for request: URLRequest) async throws -> (Data, HTTPURLResponse) { + let payload = try JSONDecoder().decode(ServerScreenSharingRequest.self, from: request.httpBody ?? Data()) + requests.append(payload) + let reply: ServerScreenSharingReply + switch payload.operation { + case .capabilities: + reply = .init(status: "available", displays: [.init(id: "display", name: "Display", width: 1920, height: 1080)]) + case .start, .restart: + let first = requests.filter { $0.operation == .start }.count == 1 + started.signal() + if first, blockFirstStart { await releaseFirstStart.wait() } + reply = .init(status: payload.operation == .restart ? restartStatus : "connecting", answer: "fixture answer") + case .heartbeat: reply = .init(status: heartbeatStatus) + case .stop: + stopWasCancelled = stopWasCancelled || Task.isCancelled + reply = .init(status: "stopped") + } + return ( + try JSONEncoder().encode(reply), + HTTPURLResponse(url: request.url!, statusCode: 200, httpVersion: nil, headerFields: nil)! + ) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeCapabilities.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeCapabilities.swift new file mode 100644 index 000000000..9355e3058 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeCapabilities.swift @@ -0,0 +1,37 @@ +import Foundation +import VideoToolbox + +/// Capability discovery is deliberately separate from a measured round trip. +/// An advertised HEVC encoder does not establish negotiated HEVC or 4:4:4 support. +struct ProbeCapabilities: Encodable { + struct Codec: Encodable { + let name: String + let hardwareDecodeAdvertised: Bool + let encoders: [Encoder] + } + struct Encoder: Encodable { + let identifier: String + let hardwareAdvertised: Bool? + } + let operatingSystem: String + let negotiatedProbeCodec = "H.264 constrained baseline, 8-bit 4:2:0 SDR" + let codecs: [Codec] + + static func read() throws -> Self { + var encoders: CFArray? + let status = VTCopyVideoEncoderList(nil, &encoders) + guard status == noErr else { throw NSError(domain: NSOSStatusErrorDomain, code: Int(status)) } + let entries = encoders as? [[CFString: Any]] ?? [] + let codecs = [("H.264", kCMVideoCodecType_H264), ("HEVC", kCMVideoCodecType_HEVC)].map { name, type in + Codec( + name: name, hardwareDecodeAdvertised: VTIsHardwareDecodeSupported(type), + encoders: entries.compactMap { + guard ($0[kVTVideoEncoderList_CodecType] as? NSNumber)?.uint32Value == type else { return nil } + return Encoder( + identifier: $0[kVTVideoEncoderList_EncoderID] as? String ?? "unknown", + hardwareAdvertised: $0[kVTVideoEncoderList_IsHardwareAccelerated] as? Bool) + }) + } + return Self(operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString, codecs: codecs) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeCapturePicker.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeCapturePicker.swift new file mode 100644 index 000000000..585fb4d14 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeCapturePicker.swift @@ -0,0 +1,79 @@ +import AppKit +import CodevisorScreenSharing +@preconcurrency import ScreenCaptureKit + +/// A diagnostic process owns one picker and one stream. Keep the picker active +/// until capture stops so the system can display and revoke the selected share. +@MainActor +final class ProbeCapturePicker: NSObject, SCContentSharingPickerObserver { + private var selection: CheckedContinuation? + private var selectedFilter: SCContentFilter? + private var timeout: Task? + private var active = false + + func choose(window: Bool = false) async throws -> SCContentFilter { + let picker = SCContentSharingPicker.shared + var configuration = SCContentSharingPickerConfiguration() + configuration.allowedPickerModes = window ? [.singleWindow] : [.singleDisplay] + configuration.allowsChangingSelectedContent = false + picker.defaultConfiguration = configuration + picker.maximumStreamCount = 1 + picker.add(self) + active = true + picker.isActive = true + NSApplication.shared.activate() + try await withTaskCancellationHandler { + try Task.checkCancellation() + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + selection = continuation + timeout = Task { [weak self] in + do { try await Task.sleep(for: .seconds(120)) } catch { return } + self?.finish(.failure(ScreenSharingError.unavailable("Capture selection timed out."))) + } + picker.present(using: window ? .window : .display) + } + } onCancel: { + Task { @MainActor [weak self] in self?.finish(.failure(CancellationError())) } + } + guard let selectedFilter else { throw ScreenSharingError.unavailable("No capture content was selected.") } + return selectedFilter + } + + func stop() { + finish(.failure(CancellationError())) + if active { + let picker = SCContentSharingPicker.shared + picker.isActive = false + picker.remove(self) + active = false + } + } + + private func finish(_ result: Result) { + timeout?.cancel() + timeout = nil + let continuation = selection + selection = nil + guard let continuation else { return } + switch result { + case .success(let filter): + selectedFilter = filter + continuation.resume() + case .failure(let error): continuation.resume(throwing: error) + } + } + + nonisolated func contentSharingPicker( + _ picker: SCContentSharingPicker, didUpdateWith filter: SCContentFilter, for stream: SCStream? + ) { + Task { @MainActor [weak self] in self?.finish(.success(filter)) } + } + + nonisolated func contentSharingPicker(_ picker: SCContentSharingPicker, didCancelFor stream: SCStream?) { + Task { @MainActor [weak self] in self?.finish(.failure(CancellationError())) } + } + + nonisolated func contentSharingPickerStartDidFailWithError(_ error: any Error) { + Task { @MainActor [weak self] in self?.finish(.failure(error)) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeClipboardCheck.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeClipboardCheck.swift new file mode 100644 index 000000000..b8d400542 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeClipboardCheck.swift @@ -0,0 +1,71 @@ +import CodevisorScreenSharing +import Foundation + +/// Exercises the real encrypted channel using in-memory clipboards. Never +/// reads or writes the diagnostic machine's general pasteboard. +@MainActor +final class ProbeClipboardCheck { + private let hostChannel: ScreenSharingClipboardChannel + private let viewerChannel: ScreenSharingClipboardChannel + private let localText = String(repeating: "Codevisor café 👩🏽‍💻 日本語\n", count: 1000) + private let remoteText = String(repeating: "Remote ✓ Ελληνικά 한국어\n", count: 1000) + private var hostText = "" + private var viewerText = "" + private var stage = 0 + private var completion: CheckedContinuation? + private var timeout: Task? + private lazy var host = ScreenSharingClipboardTransfer( + send: { [weak self] in self?.hostChannel.send($0) ?? false }, + canReceiveUnsolicited: { true }, read: { [unowned self] in hostText }, + write: { [unowned self] in hostText = $0 }) + private lazy var viewer = ScreenSharingClipboardTransfer( + send: { [weak self] in self?.viewerChannel.send($0) ?? false }, + read: { [unowned self] in localText }, write: { [unowned self] in viewerText = $0 }) + + init(host: ScreenSharingClipboardChannel, viewer: ScreenSharingClipboardChannel) { + hostChannel = host; viewerChannel = viewer + } + func run() async throws { + defer { + timeout?.cancel() + host.onFinished = nil; viewer.onFinished = nil + host.cancel(); viewer.cancel() + hostChannel.onMessage = nil; viewerChannel.onMessage = nil + hostChannel.onAvailabilityChanged = nil; viewerChannel.onAvailabilityChanged = nil + } + try await withCheckedThrowingContinuation { continuation in + completion = continuation + hostChannel.onMessage = { [weak self] in self?.host.receive($0) } + viewerChannel.onMessage = { [weak self] in self?.viewer.receive($0) } + viewer.onFinished = { [weak self] in self?.transferred(error: $0) } + hostChannel.onAvailabilityChanged = { [weak self] _ in self?.startIfReady() } + viewerChannel.onAvailabilityChanged = { [weak self] _ in self?.startIfReady() } + timeout = Task { [weak self] in + do { try await Task.sleep(for: .seconds(15)) } catch { return } + self?.finish(false) + } + startIfReady() + } + } + private func startIfReady() { + guard stage == 0, hostChannel.isAvailable, viewerChannel.isAvailable else { return } + stage = 1; viewer.sendText(localText) + } + private func transferred(error: String?) { + guard error == nil else { finish(false); return } + if stage == 1, hostText == localText { + stage = 2; hostText = remoteText; viewer.requestText() + } else { + finish(stage == 2 && viewerText == remoteText) + } + } + private func finish(_ passed: Bool) { + guard let completion else { return } + self.completion = nil + if passed { + completion.resume() + } else { + completion.resume(throwing: ScreenSharingError.unavailable("Clipboard channel check failed.")) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeClockSync.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeClockSync.swift new file mode 100644 index 000000000..8d38c6acb --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeClockSync.swift @@ -0,0 +1,31 @@ +import CodevisorScreenSharing +import Foundation +import QuartzCore + +/// A bounded stdin/stdout clock responder for a trusted diagnostic SSH session. +/// The caller brackets each response with its own clock; no symmetry assumption +/// is required to derive an offset interval from a request's round-trip time. +enum ProbeClockSync { + static func run() throws { + guard CommandLine.arguments.count == 2 else { + throw ScreenSharingError.invalid("--clock-sync must run alone.") + } + for _ in 0..<10_000 { + guard let line = readLine() else { return } + let received = CACurrentMediaTime() + guard let id = Int(line), (0...1_000_000).contains(id) else { + throw ScreenSharingError.invalid("Clock requests require an integer identifier.") + } + let reply = Reply(id: id, receivedAtSeconds: received, sentAtSeconds: CACurrentMediaTime()) + var data = try JSONEncoder().encode(reply) + data.append(0x0a) + try FileHandle.standardOutput.write(contentsOf: data) + } + } + + private struct Reply: Encodable { + let id: Int + let receivedAtSeconds: Double + let sentAtSeconds: Double + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecCheck.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecCheck.swift new file mode 100644 index 000000000..eb094cc18 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecCheck.swift @@ -0,0 +1,324 @@ +import CodevisorScreenSharing +import Foundation +import VideoToolbox + +/// Isolated hardware experiment. No WebRTC, capture permission, private codec +/// identifiers or software fallback. Unsupported cases are reported individually. +enum ProbeCodecCheck { + struct Variant { + let name: String + let codec: CMVideoCodecType + let profile: CFString? + let lowLatency: Bool + var nv24 = false + var advertised444 = false + var prioritizeSpeed = false + var nv12 = false + } + struct Result: Encodable { + let name: String + let completedRoundTrip: Bool + let error: String? + let rgbRMSE: Double? + let metrics: ScreenSharingMetrics.Snapshot + } + struct Report: Encodable { + let operatingSystem: String + let configuration: ScreenSharingVideoConfiguration + let note = + "180 paced frames per case; first 60 excluded from timings. Two frames in flight. Hardware required. No network or presentation latency. RGB RMSE measures frame 90, including color conversion. HEVC output chroma is read from hvcC, not inferred from input." + let results: [Result] + } + + static func run(configuration: ScreenSharingVideoConfiguration, report: URL?, caseName: String?) throws { + var variants = [ + Variant( + name: "h264-low-delay", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: true), + Variant( + name: "h264-low-delay-speed", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: true, prioritizeSpeed: true), + Variant( + name: "h264-realtime", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: false), + Variant( + name: "h264-realtime-speed", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: false, prioritizeSpeed: true), + Variant( + name: "h264-low-delay-nv12", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: true, nv12: true), + Variant( + name: "h264-realtime-nv12", codec: kCMVideoCodecType_H264, + profile: kVTProfileLevel_H264_Baseline_AutoLevel, lowLatency: false, nv12: true), + Variant( + name: "hevc-main", codec: kCMVideoCodecType_HEVC, + profile: kVTProfileLevel_HEVC_Main_AutoLevel, lowLatency: false), + Variant(name: "hevc-auto", codec: kCMVideoCodecType_HEVC, profile: nil, lowLatency: false), + Variant(name: "hevc-low-delay", codec: kCMVideoCodecType_HEVC, profile: nil, lowLatency: true), + Variant(name: "hevc-nv24", codec: kCMVideoCodecType_HEVC, profile: nil, lowLatency: false, nv24: true), + Variant(name: "hevc-nv24-low-delay", codec: kCMVideoCodecType_HEVC, profile: nil, lowLatency: true, nv24: true), + Variant( + name: "hevc-advertised444-bgra", codec: kCMVideoCodecType_HEVC, profile: nil, lowLatency: false, + advertised444: true), + Variant( + name: "hevc-main42210", codec: kCMVideoCodecType_HEVC, + profile: kVTProfileLevel_HEVC_Main42210_AutoLevel, lowLatency: false), + ] + if let caseName { + variants = variants.filter { $0.name == caseName } + guard !variants.isEmpty else { throw ScreenSharingError.invalid("Unknown codec case: \(caseName)") } + } + let images = report.map { URL(fileURLWithPath: $0.path + ".images", isDirectory: true) } + if let images { try FileManager.default.createDirectory(at: images, withIntermediateDirectories: true) } + var results: [Result] = [] + for variant in variants { + let runner = CodecRunner(configuration: configuration, variant: variant) + var failure: String? + do { try runner.run(images: images) } catch { failure = error.localizedDescription } + runner.stop() + let metrics = runner.metrics.snapshot() + let completed = + failure == nil && metrics.counters["decodedFrames", default: 0] >= 90 + && metrics.counters["codecErrors", default: 0] == 0 + results.append( + Result( + name: variant.name, completedRoundTrip: completed, error: failure, + rgbRMSE: runner.rgbRMSE, metrics: metrics)) + print("Codec \(variant.name): \(completed ? "round trip passed" : "unsupported or failed") \(failure ?? "")") + } + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode( + Report( + operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString, + configuration: configuration, results: results)) + if let report { try data.write(to: report, options: .atomic) } + print(String(decoding: data, as: UTF8.self)) + guard results.first?.completedRoundTrip == true else { + throw ScreenSharingError.unavailable("The first codec control case failed; inspect the recorded results.") + } + } +} + +private final class CodecRunner: @unchecked Sendable { + let metrics = ScreenSharingMetrics() + let configuration: ScreenSharingVideoConfiguration + let variant: ProbeCodecCheck.Variant + private let lock = NSLock() + private let decodeQueue = DispatchQueue(label: "codevisor.codec-probe.decode") + private var pending = 0 + private var encoder: VTCompressionSession? + private var decoder: VTDecompressionSession? + private var transfer: VTPixelTransferSession? + private var saved: (source: CVPixelBuffer, decoded: CVPixelBuffer)? + private(set) var rgbRMSE: Double? + + init(configuration: ScreenSharingVideoConfiguration, variant: ProbeCodecCheck.Variant) { + self.configuration = configuration; self.variant = variant + } + + func run(images: URL?) throws { + var specification: [CFString: Any] = [kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: true] + if variant.lowLatency { specification[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true } + try check( + VTCompressionSessionCreate( + allocator: nil, width: Int32(configuration.width), + height: Int32(configuration.height), codecType: variant.codec, + encoderSpecification: specification as CFDictionary, + imageBufferAttributes: nil, compressedDataAllocator: nil, outputCallback: nil, refcon: nil, + compressionSessionOut: &encoder), "Create hardware encoder") + guard let encoder else { throw ScreenSharingError.invalid("Missing encoder.") } + var supported: CFDictionary? + if VTSessionCopySupportedPropertyDictionary(encoder, supportedPropertyDictionaryOut: &supported) == noErr, + let entry = (supported as? [String: Any])?[kVTCompressionPropertyKey_ProfileLevel as String] + { + metrics.label("supportedProfiles", String(describing: entry)) + } + if variant.advertised444 { + // Select only a value returned by the public property-query API. No + // assumption that an undeclared profile exists on another OS or encoder. + let profileInfo = + (supported as? [String: Any])?[kVTCompressionPropertyKey_ProfileLevel as String] as? [String: Any] + let profiles = profileInfo?[kVTPropertySupportedValueListKey as String] as? [String] ?? [] + guard let profile = profiles.first(where: { $0.contains("_Main444_") }) else { + throw ScreenSharingError.unavailable("Encoder does not advertise an 8-bit Main444 profile.") + } + try property(kVTCompressionPropertyKey_ProfileLevel, profile as CFString) + metrics.label("selectedAdvertisedProfile", profile) + } + try property(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue) + try property(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse) + if variant.prioritizeSpeed { + try property(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, kCFBooleanTrue) + } + metrics.label("encoderSpeedPolicy", variant.prioritizeSpeed ? "prioritize speed" : "encoder default") + if let profile = variant.profile { try property(kVTCompressionPropertyKey_ProfileLevel, profile) } + try property(kVTCompressionPropertyKey_ExpectedFrameRate, configuration.framesPerSecond as CFNumber) + try property(kVTCompressionPropertyKey_AverageBitRate, configuration.bitrate as CFNumber) + try property(kVTCompressionPropertyKey_MaxKeyFrameInterval, (configuration.framesPerSecond * 2) as CFNumber) + let delayStatus = VTSessionSetProperty( + encoder, key: kVTCompressionPropertyKey_MaxFrameDelayCount, value: 1 as CFNumber) + metrics.label("maxFrameDelayCountStatus", String(delayStatus)) + try property(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2) + try property(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2) + try property(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2) + try check(VTCompressionSessionPrepareToEncodeFrames(encoder), "Prepare encoder") + hardware(encoder, key: kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, label: "encoderHardware") + let started = ScreenSharingMetrics.nowNs + for sequence in 0..<180 { + // This is paced diagnostic traffic, not correctness-test synchronization. + let deadline = started + Int64(sequence) * 1_000_000_000 / Int64(configuration.framesPerSecond) + let wait = Double(deadline - ScreenSharingMetrics.nowNs) / 1_000_000_000 + if wait > 0 { Thread.sleep(forTimeInterval: wait) } + let admitted = lock.withLock { + if pending >= 2 { return false }; pending += 1; return true + } + guard admitted else { metrics.increment("admissionDrops"); continue } + do { + try autoreleasepool { + let original = try ProbeCodecPattern.make( + width: configuration.width, height: configuration.height, sequence: sequence) + let input = try source(original) + metrics.label("inputPixelFormatCode", String(CVPixelBufferGetPixelFormatType(input))) + let retained = ScreenSharingVideoFrame(pixelBuffer: original, timestampNs: 0) + let submitted = ScreenSharingMetrics.nowNs + let status = VTCompressionSessionEncodeFrame( + encoder, imageBuffer: input, + presentationTimeStamp: CMTime(value: Int64(sequence), timescale: Int32(configuration.framesPerSecond)), + duration: CMTime(value: 1, timescale: Int32(configuration.framesPerSecond)), frameProperties: nil, + infoFlagsOut: nil + ) { [self] status, flags, sample in + defer { lock.withLock { pending -= 1 } } + guard status == noErr, !flags.contains(.frameDropped), let sample else { + metrics.increment(status == noErr ? "encoderDrops" : "codecErrors"); return + } + metrics.increment("encodedFrames") + metrics.increment("encodedBytes", by: CMSampleBufferGetTotalSampleSize(sample)) + if sequence >= 60 { + metrics.observe("encode", milliseconds: Double(ScreenSharingMetrics.nowNs - submitted) / 1_000_000) + } + decodeQueue.sync { + do { try decode(sample, original: retained, sequence: sequence) } catch { + metrics.increment("codecErrors"); metrics.label("decodeError", error.localizedDescription) + } + } + } + try check(status, "Submit encode") + } + } catch { lock.withLock { pending -= 1 }; throw error } + } + try check(VTCompressionSessionCompleteFrames(encoder, untilPresentationTimeStamp: .invalid), "Drain encoder") + decodeQueue.sync {} + if let saved { + rgbRMSE = ProbeCodecPattern.rgbRMSE(saved.source, saved.decoded) + if let images { + try ProbeCodecPattern.write(saved.source, to: images.appendingPathComponent(variant.name + "-source.png")) + try ProbeCodecPattern.write(saved.decoded, to: images.appendingPathComponent(variant.name + "-decoded.png")) + } + } + } + + func stop() { + if let encoder { + VTCompressionSessionCompleteFrames(encoder, untilPresentationTimeStamp: .invalid); + VTCompressionSessionInvalidate(encoder) + } + encoder = nil + decodeQueue.sync { + if let decoder { + VTDecompressionSessionWaitForAsynchronousFrames(decoder); VTDecompressionSessionInvalidate(decoder) + } + decoder = nil + } + if let transfer { VTPixelTransferSessionInvalidate(transfer) }; transfer = nil + } + + private func source(_ original: CVPixelBuffer) throws -> CVPixelBuffer { + guard variant.nv24 || variant.nv12 else { return original } + let started = ScreenSharingMetrics.nowNs + if transfer == nil { + try check( + VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer), "Create pixel transfer") + } + var converted: CVPixelBuffer? + try check( + CVPixelBufferCreate( + nil, configuration.width, configuration.height, + variant.nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, + [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &converted), "Allocate YCbCr input") + guard let transfer, let converted else { throw ScreenSharingError.invalid("Missing transfer buffer.") } + CVBufferSetAttachment( + converted, kCVImageBufferColorPrimariesKey, kCVImageBufferColorPrimaries_ITU_R_709_2, .shouldPropagate) + CVBufferSetAttachment( + converted, kCVImageBufferTransferFunctionKey, kCVImageBufferTransferFunction_ITU_R_709_2, .shouldPropagate) + CVBufferSetAttachment( + converted, kCVImageBufferYCbCrMatrixKey, kCVImageBufferYCbCrMatrix_ITU_R_709_2, .shouldPropagate) + try check(VTPixelTransferSessionTransferImage(transfer, from: original, to: converted), "Transfer BGRA to YCbCr") + metrics.observe("inputConversion", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + return converted + } + + private func decode(_ sample: CMSampleBuffer, original: ScreenSharingVideoFrame, sequence: Int) throws { + guard let format = CMSampleBufferGetFormatDescription(sample) else { + throw ScreenSharingError.invalid("Missing format.") + } + if decoder == nil { + let atoms = + CMFormatDescriptionGetExtension( + format, extensionKey: kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms) as? [String: Any] + if let hvcc = atoms?["hvcC"] as? Data, hvcc.count >= 23 { + metrics.label("encodedHEVCProfileIDC", String(hvcc[1] & 31)) + metrics.label("encodedChroma", ["monochrome", "4:2:0", "4:2:2", "4:4:4"][Int(hvcc[16] & 3)]) + metrics.label("encodedBitDepth", String(8 + Int(hvcc[17] & 7))) + } else if variant.codec == kCMVideoCodecType_H264 { + metrics.label("encodedChroma", "4:2:0 (H.264 baseline)") + } + try check( + VTDecompressionSessionCreate( + allocator: nil, formatDescription: format, + decoderSpecification: [kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder: true] + as CFDictionary, + imageBufferAttributes: [ + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, + kCVPixelBufferIOSurfacePropertiesKey: [:], + ] as CFDictionary, + outputCallback: nil, decompressionSessionOut: &decoder), "Create hardware decoder") + if let decoder { + try check( + VTSessionSetProperty(decoder, key: kVTDecompressionPropertyKey_RealTime, value: kCFBooleanTrue), + "Decoder real-time") + hardware( + decoder, key: kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder, label: "decoderHardware") + } + } + guard let decoder else { throw ScreenSharingError.invalid("Missing decoder.") } + let started = ScreenSharingMetrics.nowNs + try check( + VTDecompressionSessionDecodeFrame(decoder, sampleBuffer: sample, flags: [], infoFlagsOut: nil) { + [self] status, flags, image, _, _ in + guard status == noErr, !flags.contains(.frameDropped), let image else { + metrics.increment("codecErrors"); return + } + metrics.increment("decodedFrames") + if sequence >= 60 { + metrics.observe("decode", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + } + if sequence == 90 { saved = (original.pixelBuffer, image) } + }, "Decode") + } + + private func hardware(_ session: VTSession, key: CFString, label: String) { + var value: Unmanaged? + let status = VTSessionCopyProperty(session, key: key, allocator: nil, valueOut: &value) + let result = value?.takeRetainedValue() as? Bool + metrics.label( + label, status == noErr ? (result == true ? "confirmed" : "not confirmed") : "required; query status \(status)") + } + private func property(_ key: CFString, _ value: CFTypeRef) throws { + guard let encoder else { throw ScreenSharingError.invalid("Missing encoder.") } + try check(VTSessionSetProperty(encoder, key: key, value: value), "Set \(key)") + } + private func check(_ status: OSStatus, _ operation: String) throws { + if status != noErr { throw ScreenSharingError.codec(operation, status) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecPattern.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecPattern.swift new file mode 100644 index 000000000..94d800484 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeCodecPattern.swift @@ -0,0 +1,102 @@ +import CoreGraphics +import CoreText +import CoreVideo +import Foundation +import ImageIO +import CodevisorScreenSharing + +enum ProbeCodecPattern { + static func make(width: Int, height: Int, sequence: Int) throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + let status = CVPixelBufferCreate( + nil, width, height, kCVPixelFormatType_32BGRA, + [ + kCVPixelBufferIOSurfacePropertiesKey: [:], kCVPixelBufferCGImageCompatibilityKey: true, + kCVPixelBufferCGBitmapContextCompatibilityKey: true, + ] as CFDictionary, &pixel) + guard status == noErr, let pixel else { throw ScreenSharingError.codec("Pattern allocation", status) } + CVPixelBufferLockBaseAddress(pixel, []) + defer { CVPixelBufferUnlockBaseAddress(pixel, []) } + guard let context = context(pixel) else { throw ScreenSharingError.invalid("Pattern context unavailable.") } + context.setFillColor(CGColor(gray: 0.06, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + let font = CTFontCreateWithName("Menlo" as CFString, 14, nil) + let colors = [ + CGColor(red: 1, green: 0.25, blue: 0.3, alpha: 1), + CGColor(red: 0.2, green: 1, blue: 0.4, alpha: 1), CGColor(red: 0.3, green: 0.65, blue: 1, alpha: 1), + CGColor(gray: 0.95, alpha: 1), + ] + for row in 0.. Il1O0 @ 60 fps", row + sequence) + let attributes: [NSAttributedString.Key: Any] = [ + NSAttributedString.Key(kCTFontAttributeName as String): font, + NSAttributedString.Key(kCTForegroundColorAttributeName as String): colors[row % colors.count], + ] + let line = CTLineCreateWithAttributedString(NSAttributedString(string: text, attributes: attributes)) + context.textPosition = CGPoint(x: 24, y: height - 30 - row * 22 + sequence % 22) + CTLineDraw(line, context) + } + for x in 0.. Double? { + guard CVPixelBufferGetPixelFormatType(decoded) == kCVPixelFormatType_32BGRA, + CVPixelBufferGetWidth(source) == CVPixelBufferGetWidth(decoded), + CVPixelBufferGetHeight(source) == CVPixelBufferGetHeight(decoded) + else { return nil } + CVPixelBufferLockBaseAddress(source, .readOnly) + CVPixelBufferLockBaseAddress(decoded, .readOnly) + defer { + CVPixelBufferUnlockBaseAddress(decoded, .readOnly) + CVPixelBufferUnlockBaseAddress(source, .readOnly) + } + guard let a = CVPixelBufferGetBaseAddress(source)?.assumingMemoryBound(to: UInt8.self), + let b = CVPixelBufferGetBaseAddress(decoded)?.assumingMemoryBound(to: UInt8.self) + else { return nil } + var squared = 0.0 + let width = CVPixelBufferGetWidth(source) + let height = CVPixelBufferGetHeight(source) + for y in 0.. CGContext? { + CGContext( + data: CVPixelBufferGetBaseAddress(pixel), width: CVPixelBufferGetWidth(pixel), + height: CVPixelBufferGetHeight(pixel), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixel), + space: CGColorSpace(name: CGColorSpace.itur_709)!, + bitmapInfo: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeControlCheck.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeControlCheck.swift new file mode 100644 index 000000000..a003a64e1 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeControlCheck.swift @@ -0,0 +1,88 @@ +import CodevisorScreenSharing +import Foundation + +/// Real encrypted data-channel check with a recording sink. Never posts input +/// to the diagnostic machine. Exercises both directions while video is running. +@MainActor +final class ProbeControlCheck { + private let host: ScreenSharingControlChannel + private let viewer: ScreenSharingControlChannel + private let request = UUID() + private let lease = UUID() + private var started = false + private var received: [ScreenSharingInputEvent] = [] + private var completion: CheckedContinuation? + private var timeout: Task? + private let events: [ScreenSharingInputEvent] = [ + .move(.init(x: 0.25, y: 0.75), modifiers: 0), + .button(.init(x: 0.25, y: 0.75), button: 0, down: true, clicks: 1, modifiers: 0), + .move(.init(x: 0.5, y: 0.5), modifiers: 0), + .button(.init(x: 0.5, y: 0.5), button: 0, down: false, clicks: 1, modifiers: 0), + .key(code: 0, down: true, repeatKey: false, modifiers: 0), + .key(code: 0, down: false, repeatKey: false, modifiers: 0), + .scroll(.init(x: 0.5, y: 0.5), x: -3, y: 12, modifiers: 0), + .text("Codevisor café 日本語"), + ] + init(host: ScreenSharingControlChannel, viewer: ScreenSharingControlChannel) { + self.host = host; self.viewer = viewer + } + func run() async throws { + defer { + timeout?.cancel() + host.onMessage = nil; viewer.onMessage = nil + host.onAvailabilityChanged = nil; viewer.onAvailabilityChanged = nil + } + try await withCheckedThrowingContinuation { continuation in + completion = continuation + host.onAvailabilityChanged = { [weak self] _ in self?.startIfReady() } + viewer.onAvailabilityChanged = { [weak self] _ in self?.startIfReady() } + host.onMessage = { [weak self] in self?.hostReceive($0) } + viewer.onMessage = { [weak self] in self?.viewerReceive($0) } + timeout = Task { [weak self] in + do { try await Task.sleep(for: .seconds(10)) } catch { return } + self?.finish(false) + } + startIfReady() + } + } + private func startIfReady() { + guard !started, host.isAvailable, viewer.isAvailable else { return } + started = true + if !viewer.send(.request(id: request)) { finish(false) } + } + private func hostReceive(_ message: ScreenSharingControlMessage) { + switch message { + case .request(let id) where id == request: + if !host.send(.grant(request: request, lease: lease)) { finish(false) } + case .input(let id, let sequence, let event) where id == lease: + guard sequence == UInt64(received.count + 1) else { finish(false); return } + received.append(event) + case .release(let id) where id == lease: + guard received == events else { finish(false); return } + if !host.send(.revoked(lease: lease, reason: "complete")) { finish(false) } + default: finish(false) + } + } + private func viewerReceive(_ message: ScreenSharingControlMessage) { + switch message { + case .grant(let request, let lease) where request == self.request && lease == self.lease: + for (index, event) in events.enumerated() { + guard viewer.send(.input(lease: lease, sequence: UInt64(index + 1), event: event)) else { + finish(false); return + } + } + if !viewer.send(.release(lease: lease)) { finish(false) } + case .revoked(let lease, let reason) where lease == self.lease && reason == "complete": finish(true) + default: finish(false) + } + } + private func finish(_ passed: Bool) { + guard let completion else { return } + self.completion = nil + if passed { + completion.resume() + } else { + completion.resume(throwing: ScreenSharingError.unavailable("Control channel check failed.")) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopPainter.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopPainter.swift new file mode 100644 index 000000000..cef67c2e2 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopPainter.swift @@ -0,0 +1,69 @@ +import CodevisorScreenSharing +import CoreImage +import CoreText +import Foundation + +/// Immutable drawing resources shared by the visible desktop target and the +/// synthetic source. Each owner draws on its own serial queue. +final class ProbeDesktopPainter: @unchecked Sendable { + let pattern: CGImage + let fps: Int + // Use a concrete, retained CTFont like the source pattern. Repeated AppKit + // system-font resolution in NSString.draw raised a CoreText exception on + // the macOS 26 host during an extended workload run. + private let textAttributes: [NSAttributedString.Key: Any] = [ + NSAttributedString.Key(kCTFontAttributeName as String): CTFontCreateWithName("Menlo" as CFString, 20, nil), + NSAttributedString.Key(kCTForegroundColorAttributeName as String): CGColor(gray: 1, alpha: 1), + ] + + init(pattern: CGImage, fps: Int) { self.pattern = pattern; self.fps = fps } + + static func make(width: Int, height: Int, fps: Int) throws -> ProbeDesktopPainter { + let pixel = try ProbeCodecPattern.make(width: width, height: height, sequence: 0) + let source = CIImage(cvPixelBuffer: pixel) + guard let pattern = CIContext().createCGImage(source, from: source.extent) else { + throw ScreenSharingError.unavailable("Cannot prepare desktop pattern.") + } + return ProbeDesktopPainter(pattern: pattern, fps: fps) + } + + func draw(in context: CGContext, bounds: CGRect, sequence: Int, responses: Int) { + let width = CGFloat(pattern.width) + let height = CGFloat(pattern.height) + context.saveGState() + defer { context.restoreGState() } + context.scaleBy(x: bounds.width / width, y: bounds.height / height) + context.interpolationQuality = .none + let offset = CGFloat(sequence % 22) + context.setFillColor(CGColor(gray: 0.06, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + context.draw(pattern, in: CGRect(x: 0, y: offset, width: width, height: height)) + context.setFillColor(CGColor(gray: 0.06, alpha: 1)) + context.fill(CGRect(x: 0, y: height - 140, width: width, height: 140)) + drawText("Codevisor benchmark · frame \(sequence) · responses \(responses)", x: 24, y: height - 40, in: context) + drawText( + "Click or press Space · time-driven frame code below · \(fps) fps requested", x: 24, y: height - 70, in: context) + // 18 bits, MSB first, fit the full one-hour / 60 fps CLI range. A white + // reference cell precedes the data; a black cell follows it. + for bit in 0..<20 { + let on = bit == 0 || (bit < 19 && (sequence & (1 << (18 - bit))) != 0) + context.setFillColor(CGColor(gray: on ? 1 : 0, alpha: 1)) + context.fill(CGRect(x: 24 + bit * 28, y: Int(height) - 120, width: 24, height: 30)) + } + context.setFillColor( + responses.isMultiple(of: 2) + ? CGColor(red: 0.1, green: 0.4, blue: 1, alpha: 1) + : CGColor(red: 1, green: 0.3, blue: 0.1, alpha: 1)) + context.fill(CGRect(x: width - 300, y: height - 125, width: 276, height: 105)) + drawText("Response \(responses)", x: width - 280, y: height - 75, in: context) + context.setFillColor(CGColor(red: 0.2, green: 1, blue: 0.4, alpha: 1)) + context.fill(CGRect(x: CGFloat((sequence * 7) % pattern.width), y: 70, width: 80, height: 20)) + } + + private func drawText(_ text: String, x: CGFloat, y: CGFloat, in context: CGContext) { + let line = CTLineCreateWithAttributedString(NSAttributedString(string: text, attributes: textAttributes)) + context.textMatrix = .identity + context.textPosition = CGPoint(x: x, y: y) + CTLineDraw(line, context) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopWorkload.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopWorkload.swift new file mode 100644 index 000000000..69c32f54c --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeDesktopWorkload.swift @@ -0,0 +1,94 @@ +import AppKit +import CodevisorScreenSharing +import QuartzCore + +/// A common desktop target for external viewers. Counters describe AppKit draw +/// calls and received events, never claimed source/display presentation times. +@MainActor +enum ProbeDesktopWorkload { + static func run(options: ProbeOptions) async throws { + guard let screen = NSScreen.main else { throw ScreenSharingError.unavailable("No desktop display.") } + let painter = try ProbeDesktopPainter.make( + width: options.configuration.width, height: options.configuration.height, + fps: options.configuration.framesPerSecond) + let view = WorkloadView(painter: painter, recordDrawTimes: options.recordWorkloadTimes) + let contentFrame: NSRect + if options.workloadWindow { + let size = NSSize( + width: Double(options.configuration.width) / screen.backingScaleFactor, + height: Double(options.configuration.height) / screen.backingScaleFactor) + contentFrame = NSRect( + x: screen.frame.minX, y: screen.frame.maxY - size.height, width: size.width, height: size.height) + } else { + contentFrame = screen.frame + } + let window = WorkloadWindow( + contentRect: contentFrame, styleMask: [.borderless], backing: .buffered, defer: false) + window.hasShadow = false + window.isReleasedWhenClosed = false + window.title = "Codevisor · Screen Sharing Benchmark Workload" + window.level = .floating + window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + window.contentView = view + let previousPresentation = NSApplication.shared.presentationOptions + NSApplication.shared.presentationOptions = [.autoHideDock, .autoHideMenuBar] + defer { + view.stop() + window.orderOut(nil) + NSApplication.shared.presentationOptions = previousPresentation + } + window.makeKeyAndOrderFront(nil) + window.makeFirstResponder(view) + NSApplication.shared.activate() + view.start() + if let report = options.reportURL { + let ready: [String: Any] = [ + "windowID": window.windowNumber, + "widthPoints": view.bounds.width, "heightPoints": view.bounds.height, + "backingScaleFactor": window.backingScaleFactor, + "rasterWidth": options.configuration.width, "rasterHeight": options.configuration.height, + "startedAtSeconds": view.startedAt, + "layout": options.workloadWindow ? "exact backing size" : "fullscreen", + ] + try JSONSerialization.data(withJSONObject: ready, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".ready.json"), options: .atomic) + } + print("Desktop workload ready: click or press Space for a response; no capture or input injection.") + try await Task.sleep(for: .seconds(options.duration)) + view.stop() + let report = WorkloadReport( + rasterWidth: options.configuration.width, rasterHeight: options.configuration.height, + windowWidthPoints: view.bounds.width, windowHeightPoints: view.bounds.height, + backingScaleFactor: window.backingScaleFactor, requestedFPS: options.configuration.framesPerSecond, + startedAtSeconds: view.startedAt, + elapsedSeconds: CACurrentMediaTime() - view.startedAt, drawCalls: view.drawCalls, + lastSequence: view.sequence, responses: view.responses, eventSamples: view.events, + eventSamplesTruncated: view.responses > view.events.count, + drawSamples: view.drawSamples, drawSamplesTruncated: view.drawSamplesTruncated) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(report) + if let url = options.reportURL { try data.write(to: url, options: .atomic) } + print(String(decoding: data, as: UTF8.self)) + } +} + +private struct WorkloadReport: Encodable { + let kind = "desktop-workload" + let timingScope = "AppKit draw calls and event receipt; not physical presentation or input-to-photon" + let rasterWidth: Int + let rasterHeight: Int + let windowWidthPoints: Double + let windowHeightPoints: Double + let backingScaleFactor: Double + let requestedFPS: Int + let startedAtSeconds: Double + let elapsedSeconds: Double + let drawCalls: Int + let lastSequence: Int + let responses: Int + let eventSamples: [WorkloadEvent] + let eventSamplesTruncated: Bool + let drawSamples: [WorkloadDraw] + let drawSamplesTruncated: Bool +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeHostRecovery.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeHostRecovery.swift new file mode 100644 index 000000000..64f7bc767 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeHostRecovery.swift @@ -0,0 +1,125 @@ +#if os(macOS) + import AppKit + import CodevisorClient + import CodevisorScreenSharing + + /// A focused integration check against the actual native host API. It uses a + /// supplied test pane, never posts OS input and never accesses either clipboard. + @MainActor + final class ProbeHostRecovery { + private let client: CodevisorServerClient + private let workspace: UUID + private let pane: UUID + private let viewer = UUID() + private let metrics = ScreenSharingMetrics() + private var peer: ScreenSharingPeer? + private var view: ScreenSharingMetalView? + private var window: NSWindow? + private var display: String? + private var lastOffer: String? + + init(url: URL, workspace: UUID, pane: UUID) { + self.workspace = workspace; self.pane = pane + client = CodevisorServerClient( + config: .init( + baseURL: url, + bearerToken: ProcessInfo.processInfo.environment["CODEVISOR_SCREEN_SHARING_PROBE_TOKEN"])) + } + func run(report: URL?) async throws { + do { + try await connect(restarting: false) + let wrongOwner = ServerScreenSharingRequest( + operation: .restart, workspaceId: workspace, + paneId: pane, viewerId: UUID(), displayId: display, offer: lastOffer) + guard try await client.screenSharing(wrongOwner).status == "stopped" else { + throw ScreenSharingError.invalid("Host accepted a replacement from the wrong viewer.") + } + try await waitForVideo() + peer?.close(); peer = nil; view?.isPaused = true + try await connect(restarting: true) + _ = try await client.screenSharing(request(.stop)) + let denied = try await client.screenSharing(request(.restart, offer: lastOffer)) + guard denied.status == "stopped" else { + throw ScreenSharingError.invalid("Host restarted a stopped lease.") + } + let value = Report( + passed: true, + checks: [ + "initial video", "wrong viewer rejected without stopping video", + "fresh media peer on existing lease", "stopped lease cannot restart", + ], receiver: metrics.snapshot()) + let encoder = JSONEncoder(); encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(value) + if let report { try data.write(to: report, options: .atomic) } + print(String(decoding: data, as: UTF8.self)) + await stop() + } catch { await stop(); throw error } + } + private func connect(restarting: Bool) async throws { + let reply = try await client.screenSharing(request(.capabilities)) + guard ["available", "busy"].contains(reply.status), let selected = reply.displays.first else { + throw ScreenSharingError.unavailable(reply.message ?? "No native host display is available.") + } + if display == nil { display = selected.id } + let connectivity = try ScreenSharingICEConfiguration( + servers: reply.connectivity?.servers.map { + try ScreenSharingICEServer(urls: $0.urls, username: $0.username, credential: $0.credential) + } ?? [], relayOnly: reply.connectivity?.relayOnly ?? false) + let receiver = try ScreenSharingPeer( + sending: false, configuration: .init(), metrics: metrics, connectivity: connectivity) + peer = receiver + let view = try ScreenSharingMetalView(mailbox: receiver.mailbox, metrics: metrics) + self.view = view + if window == nil { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 960, height: 540), + styleMask: [.titled, .resizable], backing: .buffered, defer: false) + window.title = "Codevisor · Host Recovery Check"; window.center(); self.window = window + } + window?.contentView = view; window?.makeKeyAndOrderFront(nil) + let offer = try await receiver.makeDescription(offer: true) + lastOffer = offer.sdp + let answer = try await client.screenSharing(request(restarting ? .restart : .start, offer: offer.sdp)) + guard answer.status == "connecting", let sdp = answer.answer else { + throw ScreenSharingError.unavailable(answer.message ?? "The native host rejected the connection.") + } + try await receiver.accept(.init(kind: "answer", sdp: sdp)) + try await waitForVideo() + } + private func waitForVideo() async throws { + guard let view else { throw ScreenSharingError.invalid("No native viewer.") } + let baseline = metrics.snapshot().counters["presentedFrames", default: 0] + var completion: CheckedContinuation? + let timeout = Task { + do { try await Task.sleep(for: .seconds(15)) } catch { return } + let pending = completion; completion = nil + pending?.resume(throwing: ScreenSharingError.unavailable("No frames arrived during native host recovery.")) + } + defer { timeout.cancel(); view.onPresented = nil } + try await withCheckedThrowingContinuation { continuation in + completion = continuation + view.onPresented = { [weak self] _ in + guard let self, self.metrics.snapshot().counters["presentedFrames", default: 0] >= baseline + 3 else { + return + } + let pending = completion; completion = nil; pending?.resume() + } + } + } + private func stop() async { + view?.isPaused = true; peer?.close(); peer = nil; window?.orderOut(nil) + _ = try? await client.screenSharing(request(.stop)) + } + private func request( + _ operation: ServerScreenSharingRequest.Operation, offer: String? = nil + ) -> ServerScreenSharingRequest { + .init( + operation: operation, workspaceId: workspace, paneId: pane, viewerId: viewer, displayId: display, offer: offer) + } + private struct Report: Encodable { + let passed: Bool + let checks: [String] + let receiver: ScreenSharingMetrics.Snapshot + } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeMetalDisplayLink.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeMetalDisplayLink.swift new file mode 100644 index 000000000..ca4a9962c --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeMetalDisplayLink.swift @@ -0,0 +1,57 @@ +import AppKit +import CodevisorScreenSharing +import QuartzCore + +/// Owns the probe's alternative display driver. The application still uses +/// MTKView's ordinary scheduling unless this standalone experiment is selected. +@MainActor +final class ProbeMetalDisplayLink: NSObject, @preconcurrency CAMetalDisplayLinkDelegate { + private weak var view: ScreenSharingMetalView? + private let link: CAMetalDisplayLink + + init(view: ScreenSharingMetalView, framesPerSecond: Int) throws { + guard let layer = view.layer as? CAMetalLayer else { + throw ScreenSharingError.unavailable("The probe viewer has no Metal layer.") + } + self.view = view + link = CAMetalDisplayLink(metalLayer: layer) + super.init() + view.isPaused = true + view.enableSetNeedsDisplay = false + link.preferredFrameLatency = 1 + let fps = Float(framesPerSecond) + link.preferredFrameRateRange = CAFrameRateRange(minimum: fps, maximum: fps, preferred: fps) + link.delegate = self + view.metrics.label("renderScheduling", "CAMetalDisplayLink") + view.metrics.label("frameSelection", "newest from display-link drawable") + view.metrics.label("displayLinkPreferredFrameLatency", "1") + view.metrics.label("renderRequestedFPS", String(framesPerSecond)) + // Supplied drawables: neither MTKView acquisition nor the off-main worker. + view.metrics.label("drawableAcquisitionPath", "CAMetalDisplayLink supplied drawable") + link.add(to: .main, forMode: .common) + } + + func metalDisplayLink(_ link: CAMetalDisplayLink, needsUpdate update: CAMetalDisplayLink.Update) { + guard let view else { return } + let now = CACurrentMediaTime() + view.metrics.event("displayLinkCallbackInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + if update.targetTimestamp < now { + view.metrics.increment("displayLinkMissedSubmissionDeadlines") + } else { + view.metrics.observe("displayLinkSubmissionDeadlineLead", milliseconds: (update.targetTimestamp - now) * 1000) + } + if update.targetPresentationTimestamp < now { + view.metrics.increment("displayLinkLateCallbacks") + } else { + view.metrics.observe( + "displayLinkTargetPresentationLead", milliseconds: (update.targetPresentationTimestamp - now) * 1000) + } + view.draw(displayLinkDrawable: update.drawable) + } + + func stop() { + link.isPaused = true + link.delegate = nil + link.invalidate() + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeOptions.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeOptions.swift new file mode 100644 index 000000000..3a23efd64 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeOptions.swift @@ -0,0 +1,800 @@ +import CodevisorScreenSharing +import CoreVideo +import Foundation + +struct ProbeOptions { + enum Mode: String, Codable { case loopback, send, receive } + enum SyntheticPixelFormat: String { case bgra, nv12 } + let mode: Mode + let configuration: ScreenSharingVideoConfiguration + let duration: Double + let displayID: UInt32? + let capturePicker: Bool + let capturePickerWindow: Bool + let captureQueueDepth: Int + /// Optional SCK minimum-frame-interval request (fps), isolated from the video rate; nil = existing behaviour. + let captureIntervalFPS: Int? + let copyCaptureSurface: Bool + let capturePixelFormat: OSType + let offerURL: URL? + let answerURL: URL? + let reportURL: URL? + let listDisplays: Bool + let capabilities: Bool + let checkQuality: Bool + let checkRecovery: Bool + let headlessRecovery: Bool + let headless: Bool + let dropRecoveryKeyframe: Bool + let pauseSourceAfterSeconds: Double? + let finalBurstSignal: Bool + let captureOwnedWindow: Bool + let recordOwnedWorkloadTimes: Bool + let pauseWorkloadAfterSeconds: Double? + let idleThresholdMs: Int? + let idleGraceMs: Int? + let idleGraceExtensions: Int? + let traceBoundary: Bool + let sampleIntervalSeconds: Int + let syntheticGapMs: Int? + let idleOnDecoderReset: Bool + let checkCodecs: Bool + let showWorkload: Bool + let workloadWindow: Bool + let recordWorkloadTimes: Bool + let requestScreenRecording: Bool + let keepFront: Bool + let renderOnArrival: Bool + let renderFPS: Int? + let metalDisplayLink: Bool + let viewerDisplayID: UInt32? + let drawableCount: Int + let unsyncedPresentation: Bool + /// Diagnostic: drawable acquisition and encoding on a serial worker (viewer/loopback, arrival-driven, synchronized). + let renderOffMain: Bool + let standardRateControl: Bool + let disableLookAhead: Bool + let encoderInFlight: Int + let maintainSourceRate: Bool + let staticCodecRate: Bool + let completeEachFrame: Bool + let prioritizeSpeed: Bool + let desktopPattern: Bool + let userInitiatedActivity: Bool + let keyframeIntervalSeconds: Int + let syntheticPixelFormat: SyntheticPixelFormat + let videoCodec: ScreenSharingVideoCodec + /// The process-wide trial selection these options require, derived by the shared library factory that the boundary + /// and its tests use, so this file cannot drift from what is actually installed. + var fieldTrialSelection: ScreenSharingFieldTrials.Selection { + .probeOptions( + jitterWindowFrames: jitterWindowFrames, lowLatencyPlayout: lowLatencyPlayout, + playoutDelayBoundsMs: playoutDelayBoundsMs) + } + + let jitterWindowFrames: Int? + let lowLatencyPlayout: Bool + /// Receiver-only diagnostic: explicit WebRTC-ForcePlayoutDelay bounds (min_ms, max_ms) in milliseconds. + let playoutDelayBoundsMs: (min: Int, max: Int)? + /// Receiver-only diagnostic frame-delivery audit window (seconds relative to the receiver's media start). + let deliveryAuditWindow: (beginSeconds: Double, durationSeconds: Double)? + /// Receiver-only diagnostic: WebRTC RTC event log window (seconds relative to media start) and its raw output path + /// derived from the report stem; nil = no log object, nothing scheduled. + let rtcEventLogWindow: (beginSeconds: Double, durationSeconds: Double)? + let rtcEventLogPath: String? + /// Sender-only diagnostic: the same WebRTC RTC event log on the SENDING peer (send or loopback; owned-window source and + /// plain headless sender included); raw path REPORT.sender-rtc-event-log.binarypb; nil = no log object, nothing scheduled. + let senderRtcEventLogWindow: (beginSeconds: Double, durationSeconds: Double)? + let senderRtcEventLogPath: String? + let codecCase: String? + let hostCheck: (url: URL, workspace: UUID, pane: UUID)? + + static let usage = """ + screen-sharing-probe [--loopback | --send | --receive] [options] + --width 1920 --height 1080 --fps 60 --bitrate 12000000 + --duration 10 Measured seconds after connection (1...3600) + --report /path.json Write metrics; no SDP, addresses or credentials + --display ID Capture this Mac display instead of synthetic motion + --capture-picker Select a display through macOS's system sharing picker + --capture-picker-window + Select a single window through the system sharing picker + --capture-interval-fps N + Request a ScreenCaptureKit minimum frame interval of 1/N s independent of + --fps (N from the video rate up to 120); real SCK capture only + (display, picker or owned window); request telemetry, not a frame promise + --capture-queue-depth N + SCK surface pool experiment, 3...8 (default 3) + --capture-format nv12|bgra + Compare SCK output formats (default nv12; hevc444 uses bgra) + --copy-capture-surface + Transfer SCK frames into a separate pool of at most six buffers + --capabilities Report local VT codec advertisements (not a HEVC benchmark) + --check-quality Exercise live capture format changes in loopback + --check-recovery Reset decoder state after 120 frames; require recovery within 2 seconds + Loopback only; --keyframe-interval >=10 and --duration >=10 + --headless-recovery Run --check-recovery without a window; verifies decoding, not presentation + --headless Synthetic send/receive/loopback without a window; decoding telemetry only + --drop-recovery-keyframe + Discard one forced encoder output after reset; requires --check-recovery + --pause-source-after N + Headless recovery diagnostic: stop synthetic input, then observe idle output + --capture-owned-window + Headless loopback/send diagnostic: capture this process's own workload + window through ScreenCaptureKit (current-process content, exact window + and owning process; no display or other-window fallback), no viewer + rendering; excludes display/picker/synthetic-pause/workload/quality/ + recovery options + --pause-workload-after N + With --capture-owned-window: stop the workload animation after N + seconds (>=3, leaving >=3); the static window stays captured + --record-owned-workload-times + With --capture-owned-window and a duration <=240: retain the first + AppKit draw-call start per marker code (at most 20000) in + REPORT.workload-times.json; the same schema as --record-workload-times. + Nothing is retained without this flag. + --final-burst-signal With --pause-source-after and --report: publish REPORT.final-burst.json, + keep capturing 250 ms, then stop; a loss relay drops that final content + --idle-threshold-ms N Host idle-notice threshold, 50...2000 (product default 100; the slow + re-offer phase never runs faster than the threshold; 500 = former default) + --idle-grace-ms N Viewer delivery grace before a refresh, 50...5000 (product default 100; + 500 = former default) + --idle-grace-extensions N + Extend the grace up to N more windows while newer content keeps + arriving, 0...10 (product default 4; 0 = former fixed grace) + --trace-boundary Headless diagnostic: record bounded refresh/encoder/decoder boundary + traces and WebRTC drop messages in the report + --sample-interval-seconds N + Resource/cadence timeline sample interval, 1...30 (default 30); + the run may retain at most 360 samples + --synthetic-gap-ms N Bursty synthetic input: 250 ms bursts separated by N ms gaps, 50...5000 + --idle-on-decoder-reset + Stop capture delivery at reset to check recovery from an idle source + --check-codecs Hardware VT round trips, actual HEVC chroma, text images; + optional --report also creates a sibling .images directory + --show-workload Show a timed fullscreen text/motion/input target, without capture; + --width/--height set its raster, --duration bounds its lifetime + --workload-window Give that target its exact backing-pixel size, even beyond display bounds + --record-workload-times + Retain the first draw timestamp for each code; workload only, duration <=240 + --clock-sync Run alone: reply to integer stdin requests with local monotonic timestamps + --observe-window Separate diagnostic: choose a viewer window and record frame codes/timestamps + Accepts --content-top-points N, --width, --height, --duration (1...120), + and a required --report path; source raster defaults to 3840x2160. + Optional --window-id ID uses existing capture permission instead of the picker. + --request-screen-recording + Ask macOS for this diagnostic app's capture permission, then exit + --keep-front Keep the diagnostic viewer above other windows during measurement + --render-on-arrival Compare bounded frame-triggered rendering with the display-link baseline + --render-fps N Request a display-link cadence, 30...240; excludes --render-on-arrival + --metal-display-link Use CAMetalDisplayLink supplied drawables; excludes --render-on-arrival + --viewer-display ID Place the diagnostic viewer on this currently attached local display + --drawable-count 2|3 Compare Metal drawable-pool bounds (default 3) + --render-off-main Diagnostic: acquire drawables and encode on a serial worker; requires + --render-on-arrival; excludes --metal-display-link and --unsynced-presentation + --unsynced-presentation + macOS experiment: disable Metal display synchronization + --standard-rate-control + Experiment with standard VT rate control; same two-frame admission bound + --no-lookahead Request zero encoder lookahead with standard rate control (macOS) + --encoder-inflight N Encoder admission experiment, 1...8 (default 2) + --keyframe-interval N Periodic keyframe interval in nominal seconds, 1...60 (default 2) + --fixed-source-rate Disable WebRTC format adaptation; retain bitrate control and bounded frame admission + --static-codec-rate Isolate encoder reconfiguration by retaining its initial bitrate and FPS + --complete-each-frame Request synchronous completion through each submitted frame's timestamp + --prioritize-encoding-speed + Use the speed preference advertised by the standard encoder's High Speed preset + --desktop-pattern Use the desktop benchmark drawing as synthetic input, without SCK + --user-initiated-activity + Hold a user-initiated process activity during media; allow idle system sleep + --synthetic-format bgra|nv12 + Synthetic sender input (default bgra); nv12 converts before WebRTC + --codec h264|hevc|hevc444 + Native transport experiment (default h264); hevc444 sender requires + --standard-rate-control and preserves full chroma from BGRA capture + --jitter-window-frames N + Receiver experiment: estimate max frame size using p95 over N frames (30...600) + --low-latency-playout Receiver experiment: request zero-delay WebRTC playout; may increase stuttering + --playout-delay-min-ms N --playout-delay-max-ms N + Receiver experiment: force WebRTC playout delay bounds (both required, + 0 <= min <= max <= 500 ms; a positive minimum avoids ASAP rendering); + an experiment, not a latency guarantee; excludes --low-latency-playout + --rtc-event-log-begin S --rtc-event-log-duration S + Receiver-only diagnostic: start WebRTC's RTC event log (shipped API, 8 MiB cap) at + S seconds after media start and stop it duration seconds later, both from the + measurement ticks with CACurrentMediaTime read before and after each call; the raw + log is REPORT.rtc-event-log.binarypb (an existing entry is refused at option time + and again by an exclusive reservation just before the start); the lifecycle record + is REPORT.rtc-event-log.json on every exit path; --receive or --loopback with + --report, duration 1...120, window inside --duration; the file also holds WebRTC's + bounded pre-start history and is judged offline, not here + --sender-rtc-event-log-begin S --sender-rtc-event-log-duration S + Sender-only diagnostic: the same RTC event log on the SENDING peer (--send or --loopback, + owned-window source and headless sender included; may coexist with the receiver log in + loopback); raw REPORT.sender-rtc-event-log.binarypb, record REPORT.sender-rtc-event-log.json; + outgoing packet events are stamped at the post-pacer transport hand-off, not NIC egress + --delivery-audit-begin S --delivery-audit-duration S + Receiver-only diagnostic: record scalar frame-delivery boundaries (decoder input … + presented result) for S..S+duration seconds after media start; viewer or loopback, + not headless/sender/owned capture/observer; duration 1...120, window within --duration + --codec-case NAME Run only this --check-codecs case, e.g. hevc-nv24 + --check-host URL --workspace UUID --pane UUID + Verify authenticated native-host recovery using an existing pane; + optional token comes from CODEVISOR_SCREEN_SHARING_PROBE_TOKEN + --list-displays List displays (requires Screen Recording permission) + --offer /path.json --answer /path.json + Required for send/receive; exchange using a trusted channel + Sender writes offer and waits up to 120 seconds for answer file. + Receiver reads offer, writes answer, then waits for sender to connect. + Loopback creates two real WebRTC peers on this Mac. It is not a LAN benchmark. + """ + + init(arguments: [String]) throws { + var values: [String: String] = [:] + var flags = Set() + let switches = [ + "--loopback", "--send", "--receive", "--list-displays", "--capabilities", "--check-quality", "--check-codecs", + "--keep-front", "--render-on-arrival", "--standard-rate-control", "--capture-picker", "--unsynced-presentation", + "--render-off-main", + "--show-workload", "--no-lookahead", "--fixed-source-rate", "--copy-capture-surface", "--static-codec-rate", + "--complete-each-frame", "--check-recovery", "--headless-recovery", "--drop-recovery-keyframe", + "--idle-on-decoder-reset", "--headless", "--final-burst-signal", "--trace-boundary", "--capture-owned-window", + "--record-owned-workload-times", + "--prioritize-encoding-speed", "--desktop-pattern", "--user-initiated-activity", + "--request-screen-recording", "--capture-picker-window", "--workload-window", + "--record-workload-times", + "--low-latency-playout", + "--metal-display-link", + ] + let valued = [ + "--width", "--height", "--fps", "--bitrate", "--duration", "--display", "--offer", "--answer", "--report", + "--check-host", "--workspace", "--pane", "--codec-case", "--synthetic-format", "--jitter-window-frames", + "--playout-delay-min-ms", "--playout-delay-max-ms", "--delivery-audit-begin", "--delivery-audit-duration", + "--rtc-event-log-begin", "--rtc-event-log-duration", + "--sender-rtc-event-log-begin", "--sender-rtc-event-log-duration", + "--capture-queue-depth", "--capture-interval-fps", + "--codec", + "--drawable-count", + "--render-fps", + "--viewer-display", + "--capture-format", + "--encoder-inflight", + "--keyframe-interval", + "--pause-source-after", + "--pause-workload-after", + "--idle-threshold-ms", + "--idle-grace-ms", + "--idle-grace-extensions", + "--synthetic-gap-ms", + "--sample-interval-seconds", + ] + var index = 0 + while index < arguments.count { + let key = arguments[index] + guard values[key] == nil, !flags.contains(key) else { + throw ScreenSharingError.invalid("Repeated option: \(key)") + } + if switches.contains(key) { + flags.insert(key) + } else if valued.contains(key), index + 1 < arguments.count { + index += 1 + values[key] = arguments[index] + } else { + throw ScreenSharingError.invalid("Unknown or incomplete option: \(key)\n\(Self.usage)") + } + index += 1 + } + guard flags.intersection(["--loopback", "--send", "--receive"]).count <= 1 else { + throw ScreenSharingError.invalid("Choose one probe mode.") + } + mode = flags.contains("--send") ? .send : flags.contains("--receive") ? .receive : .loopback + func integer(_ key: String, fallback: Int) throws -> Int { + guard let value = values[key] else { return fallback } + guard let parsed = Int(value) else { throw ScreenSharingError.invalid("Invalid \(key).") } + return parsed + } + configuration = try ScreenSharingVideoConfiguration( + width: integer("--width", fallback: 1920), height: integer("--height", fallback: 1080), + framesPerSecond: integer("--fps", fallback: 60), bitrate: integer("--bitrate", fallback: 12_000_000)) + guard let duration = Double(values["--duration"] ?? "10"), duration.isFinite, (1...3600).contains(duration) else { + throw ScreenSharingError.invalid("Duration must be 1...3600 seconds.") + } + self.duration = duration + if let value = values["--display"] { + guard let id = UInt32(value), mode != .receive else { + throw ScreenSharingError.invalid("Invalid capture display.") + } + displayID = id + } else { + displayID = nil + } + offerURL = values["--offer"].map { URL(fileURLWithPath: $0).standardizedFileURL } + answerURL = values["--answer"].map { URL(fileURLWithPath: $0).standardizedFileURL } + reportURL = values["--report"].map { URL(fileURLWithPath: $0).standardizedFileURL } + listDisplays = flags.contains("--list-displays") + capabilities = flags.contains("--capabilities") + checkQuality = flags.contains("--check-quality") + checkRecovery = flags.contains("--check-recovery") + headlessRecovery = flags.contains("--headless-recovery") + headless = headlessRecovery || flags.contains("--headless") + dropRecoveryKeyframe = flags.contains("--drop-recovery-keyframe") + idleOnDecoderReset = flags.contains("--idle-on-decoder-reset") + if let raw = values["--pause-source-after"] { + guard headless, mode != .receive, let seconds = Double(raw), seconds.isFinite, seconds >= 3, + seconds + 3 <= duration + else { + throw ScreenSharingError.invalid( + "Source pause requires a headless sender, at least 3 active seconds and 3 remaining seconds.") + } + pauseSourceAfterSeconds = seconds + } else { + pauseSourceAfterSeconds = nil + } + if idleOnDecoderReset, !headlessRecovery || pauseSourceAfterSeconds != nil { + throw ScreenSharingError.invalid("Idle-at-reset requires headless recovery and excludes timed source pause.") + } + finalBurstSignal = flags.contains("--final-burst-signal") + if finalBurstSignal, pauseSourceAfterSeconds == nil || values["--report"] == nil { + throw ScreenSharingError.invalid("The final-burst signal requires --pause-source-after and --report.") + } + captureOwnedWindow = flags.contains("--capture-owned-window") + if captureOwnedWindow { + let excluded: Set = [ + "--display", "--capture-picker", "--capture-picker-window", "--pause-source-after", "--final-burst-signal", + "--show-workload", "--workload-window", "--record-workload-times", "--check-quality", "--check-recovery", + "--headless-recovery", "--idle-on-decoder-reset", "--drop-recovery-keyframe", "--synthetic-gap-ms", + "--desktop-pattern", "--synthetic-format", "--list-displays", "--capabilities", "--check-codecs", + "--request-screen-recording", "--check-host", "--observe-window", "--keep-front", "--viewer-display", + "--render-fps", "--render-on-arrival", "--metal-display-link", "--render-off-main", + "--rtc-event-log-begin", "--rtc-event-log-duration", + ] + let present = excluded.intersection(flags).union(excluded.intersection(values.keys)).sorted() + guard flags.contains("--headless"), mode != .receive, present.isEmpty else { + throw ScreenSharingError.invalid( + "Owned-window capture requires --headless with --loopback or --send and excludes " + + "display/picker/synthetic-pause/workload/quality/recovery/viewer options" + + (present.isEmpty ? "." : " (given: \(present.joined(separator: " "))).") + ) + } + } + recordOwnedWorkloadTimes = flags.contains("--record-owned-workload-times") + if recordOwnedWorkloadTimes, !captureOwnedWindow || duration > 240 { + throw ScreenSharingError.invalid( + "Owned-workload draw timestamps require --capture-owned-window and a duration <=240.") + } + if let raw = values["--pause-workload-after"] { + guard captureOwnedWindow, let seconds = Double(raw), seconds.isFinite, seconds >= 3, seconds + 3 <= duration + else { + throw ScreenSharingError.invalid( + "Workload pause requires --capture-owned-window, at least 3 animated seconds and 3 remaining seconds.") + } + pauseWorkloadAfterSeconds = seconds + } else { + pauseWorkloadAfterSeconds = nil + } + let headlessMedia = headless + func experimentMilliseconds(_ key: String) throws -> Int? { + guard values[key] != nil else { return nil } + let value = try integer(key, fallback: 0) + guard (50...5000).contains(value), headlessMedia else { + throw ScreenSharingError.invalid("\(key) requires a headless media probe and 50...5000 milliseconds.") + } + return value + } + idleThresholdMs = try experimentMilliseconds("--idle-threshold-ms") + idleGraceMs = try experimentMilliseconds("--idle-grace-ms") + syntheticGapMs = try experimentMilliseconds("--synthetic-gap-ms") + if let idleThresholdMs, idleThresholdMs > 2000 { + throw ScreenSharingError.invalid("--idle-threshold-ms must not exceed the 2000 ms slow re-offer interval.") + } + if values["--idle-grace-extensions"] != nil { + let extensions = try integer("--idle-grace-extensions", fallback: 0) + guard (0...10).contains(extensions), headlessMedia, mode != .send else { + throw ScreenSharingError.invalid("--idle-grace-extensions requires a headless receiver and 0...10 windows.") + } + idleGraceExtensions = extensions + } else { + idleGraceExtensions = nil + } + sampleIntervalSeconds = try integer("--sample-interval-seconds", fallback: 30) + guard (1...30).contains(sampleIntervalSeconds), Double(sampleIntervalSeconds) * 360 >= duration else { + throw ScreenSharingError.invalid("--sample-interval-seconds must be 1...30 and allow at most 360 samples.") + } + traceBoundary = flags.contains("--trace-boundary") + if traceBoundary, !headless { + throw ScreenSharingError.invalid("Boundary tracing requires a headless media probe.") + } + if idleThresholdMs != nil, mode == .receive { + throw ScreenSharingError.invalid("Idle threshold applies to a sender.") + } + if idleGraceMs != nil, mode == .send { throw ScreenSharingError.invalid("Delivery grace applies to a receiver.") } + if syntheticGapMs != nil, mode == .receive { throw ScreenSharingError.invalid("Synthetic gaps apply to a sender.") } + checkCodecs = flags.contains("--check-codecs") + showWorkload = flags.contains("--show-workload") + workloadWindow = flags.contains("--workload-window") + recordWorkloadTimes = flags.contains("--record-workload-times") + if recordWorkloadTimes, !showWorkload || duration > 240 { + throw ScreenSharingError.invalid("Workload draw timestamps require --show-workload and a duration <=240.") + } + if workloadWindow, !showWorkload { + throw ScreenSharingError.invalid("--workload-window requires --show-workload.") + } + requestScreenRecording = flags.contains("--request-screen-recording") + userInitiatedActivity = flags.contains("--user-initiated-activity") + if userInitiatedActivity, + listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("A user-initiated activity requires a media probe.") + } + keepFront = flags.contains("--keep-front") + renderOnArrival = flags.contains("--render-on-arrival") + metalDisplayLink = flags.contains("--metal-display-link") + if metalDisplayLink, + renderOnArrival || mode == .send || listDisplays || capabilities || checkCodecs || showWorkload + || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid( + "Metal display link requires a viewer or loopback without arrival-driven rendering.") + } + if values["--render-fps"] != nil { + let requested = try integer("--render-fps", fallback: 60) + guard (30...240).contains(requested), !renderOnArrival, mode != .send, + !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil + else { + throw ScreenSharingError.invalid( + "Render FPS requires a display-link viewer or loopback and a cadence from 30 through 240.") + } + renderFPS = requested + } else { + renderFPS = nil + } + if let value = values["--viewer-display"] { + guard let id = UInt32(value), id > 0, mode != .send, + !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil + else { + throw ScreenSharingError.invalid("Viewer display requires a viewer or loopback and a positive display ID.") + } + viewerDisplayID = id + } else { + viewerDisplayID = nil + } + drawableCount = try integer("--drawable-count", fallback: 3) + unsyncedPresentation = flags.contains("--unsynced-presentation") + if values["--drawable-count"] != nil || unsyncedPresentation { + guard (2...3).contains(drawableCount), mode != .send, !listDisplays, !capabilities, !checkCodecs, + !showWorkload, !requestScreenRecording, values["--check-host"] == nil + else { + throw ScreenSharingError.invalid( + "Presentation experiments require a viewer or loopback, with two or three drawables.") + } + } + renderOffMain = flags.contains("--render-off-main") + if renderOffMain, + !renderOnArrival || metalDisplayLink || unsyncedPresentation || mode == .send || listDisplays || capabilities + || checkCodecs || showWorkload || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid( + "Off-main render preparation requires a viewer or loopback with --render-on-arrival and synchronized presentation; " + + "it excludes --metal-display-link and --unsynced-presentation.") + } + standardRateControl = flags.contains("--standard-rate-control") + prioritizeSpeed = flags.contains("--prioritize-encoding-speed") + if prioritizeSpeed, !standardRateControl { + throw ScreenSharingError.invalid("Encoding speed experiments require standard rate control.") + } + maintainSourceRate = flags.contains("--fixed-source-rate") + staticCodecRate = flags.contains("--static-codec-rate") + completeEachFrame = flags.contains("--complete-each-frame") + if completeEachFrame, + mode == .receive || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Per-frame completion requires sending media.") + } + if staticCodecRate, + mode == .receive || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("A static codec rate requires sending media.") + } + if maintainSourceRate, + mode == .receive || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("A fixed source rate requires sending media.") + } + encoderInFlight = try integer("--encoder-inflight", fallback: 2) + keyframeIntervalSeconds = try integer("--keyframe-interval", fallback: 2) + if values["--keyframe-interval"] != nil { + guard (1...60).contains(keyframeIntervalSeconds), mode != .receive, !listDisplays, !capabilities, + !checkCodecs, !showWorkload, !requestScreenRecording, values["--check-host"] == nil + else { throw ScreenSharingError.invalid("Keyframe interval requires sending media and 1...60 seconds.") } + } + if values["--encoder-inflight"] != nil { + guard (1...8).contains(encoderInFlight), mode != .receive, !listDisplays, !capabilities, + !checkCodecs, !showWorkload, !requestScreenRecording, values["--check-host"] == nil + else { throw ScreenSharingError.invalid("Encoder admission requires sending media and 1...8 frames.") } + } + disableLookAhead = flags.contains("--no-lookahead") + if disableLookAhead, !standardRateControl { + throw ScreenSharingError.invalid("--no-lookahead requires --standard-rate-control.") + } + capturePickerWindow = flags.contains("--capture-picker-window") + if capturePickerWindow, flags.contains("--capture-picker") { + throw ScreenSharingError.invalid("Choose one capture picker mode.") + } + capturePicker = flags.contains("--capture-picker") || capturePickerWindow + desktopPattern = flags.contains("--desktop-pattern") + if desktopPattern, + mode == .receive || displayID != nil || capturePicker || listDisplays || capabilities || checkCodecs + || showWorkload || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("The desktop pattern requires a synthetic sender or loopback.") + } + copyCaptureSurface = flags.contains("--copy-capture-surface") + if copyCaptureSurface, + (!capturePicker && displayID == nil) || listDisplays || capabilities || checkCodecs || showWorkload + || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Copying capture surfaces requires desktop capture.") + } + captureQueueDepth = try integer("--capture-queue-depth", fallback: 3) + if values["--capture-queue-depth"] != nil, + !(3...8).contains(captureQueueDepth) || (!capturePicker && displayID == nil) + || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Capture queue depth requires desktop capture and 3...8 surfaces.") + } + if let raw = values["--capture-interval-fps"] { + guard let requested = Int(raw), requested >= configuration.framesPerSecond, + requested <= ScreenSharingCaptureIntervalRequest.maximumFramesPerSecond + else { + throw ScreenSharingError.invalid( + "Capture interval request must be an integer from the video rate up to 120 fps.") + } + guard captureOwnedWindow || capturePicker || displayID != nil, mode != .receive, + !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil, !flags.contains("--observe-window"), !flags.contains("--clock-sync") + else { + throw ScreenSharingError.invalid( + "Capture interval request requires real ScreenCaptureKit capture (display, picker or owned window).") + } + captureIntervalFPS = requested + } else { + captureIntervalFPS = nil + } + if capturePicker, + mode == .receive || displayID != nil || listDisplays || capabilities || checkCodecs || showWorkload + || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("The capture picker requires a sender or loopback, without --display.") + } + if values["--jitter-window-frames"] != nil { + let window = try integer("--jitter-window-frames", fallback: 60) + guard (30...600).contains(window), mode != .send, !listDisplays, !capabilities, !checkCodecs, + !showWorkload, !requestScreenRecording, values["--check-host"] == nil + else { + throw ScreenSharingError.invalid("Jitter window requires receiving media or loopback and 30...600 frames.") + } + jitterWindowFrames = window + } else { + jitterWindowFrames = nil + } + lowLatencyPlayout = flags.contains("--low-latency-playout") + if lowLatencyPlayout, + mode == .send || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Low-latency playout requires receiving media or loopback.") + } + let minRaw = values["--playout-delay-min-ms"], maxRaw = values["--playout-delay-max-ms"] + if minRaw != nil || maxRaw != nil { + guard let minRaw, let maxRaw, let minMs = Int(minRaw), let maxMs = Int(maxRaw), 0 <= minMs, minMs <= maxMs, + maxMs <= 500 + else { + throw ScreenSharingError.invalid( + "Playout delay bounds need both --playout-delay-min-ms and --playout-delay-max-ms with 0 <= min <= max <= 500." + ) + } + guard !lowLatencyPlayout, + mode != .send, !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil, !flags.contains("--observe-window"), !flags.contains("--clock-sync") + else { + throw ScreenSharingError.invalid( + "Playout delay bounds require receiving media or loopback and exclude --low-latency-playout.") + } + playoutDelayBoundsMs = (min: minMs, max: maxMs) + } else { + playoutDelayBoundsMs = nil + } + let auditBeginRaw = values["--delivery-audit-begin"], auditDurationRaw = values["--delivery-audit-duration"] + if auditBeginRaw != nil || auditDurationRaw != nil { + guard let auditBeginRaw, let auditDurationRaw, let begin = Double(auditBeginRaw), + let auditDuration = Double(auditDurationRaw), + begin.isFinite, auditDuration.isFinite, begin >= 0, (1...120).contains(auditDuration), + begin + auditDuration <= Double(duration) + else { + throw ScreenSharingError.invalid( + "The delivery audit needs both --delivery-audit-begin (>= 0) and --delivery-audit-duration (1...120) with the window inside --duration." + ) + } + guard mode != .send, !headless, !captureOwnedWindow, !listDisplays, !capabilities, !checkCodecs, !showWorkload, + !requestScreenRecording, values["--check-host"] == nil, !flags.contains("--observe-window"), + !flags.contains("--clock-sync"), flags.contains("--render-on-arrival"), !flags.contains("--metal-display-link") + else { + throw ScreenSharingError.invalid( + "The delivery audit requires an arrival-driven rendering viewer or loopback (--render-on-arrival; no headless, sender, owned capture, observer, clock sync or --metal-display-link)." + ) + } + deliveryAuditWindow = (beginSeconds: begin, durationSeconds: auditDuration) + } else { + deliveryAuditWindow = nil + } + let logBeginRaw = values["--rtc-event-log-begin"], logDurationRaw = values["--rtc-event-log-duration"] + if logBeginRaw != nil || logDurationRaw != nil { + guard let logBeginRaw, let logDurationRaw, let begin = Double(logBeginRaw), + let logDuration = Double(logDurationRaw), begin.isFinite, logDuration.isFinite, begin >= 0, + (1...ScreenSharingRtcEventLogDiagnostic.Window.maximumDurationSeconds).contains(logDuration), + begin + logDuration <= Double(duration) + else { + throw ScreenSharingError.invalid( + "The RTC event log needs both --rtc-event-log-begin (>= 0) and --rtc-event-log-duration (1...120) with the window inside --duration." + ) + } + guard mode != .send, !captureOwnedWindow, !listDisplays, !capabilities, !checkCodecs, !showWorkload, + !requestScreenRecording, values["--check-host"] == nil, !flags.contains("--observe-window"), + !flags.contains("--clock-sync"), let reportURL + else { + throw ScreenSharingError.invalid( + "The RTC event log requires a receiving media probe (--receive or --loopback) with --report; no sender, owned capture, observer, clock sync or host check." + ) + } + let path = reportURL.path + ".rtc-event-log.binarypb" + try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(path) + rtcEventLogWindow = (beginSeconds: begin, durationSeconds: logDuration) + rtcEventLogPath = path + } else { + rtcEventLogWindow = nil + rtcEventLogPath = nil + } + let sendLogBeginRaw = values["--sender-rtc-event-log-begin"] + let sendLogDurationRaw = values["--sender-rtc-event-log-duration"] + if sendLogBeginRaw != nil || sendLogDurationRaw != nil { + guard let sendLogBeginRaw, let sendLogDurationRaw, let begin = Double(sendLogBeginRaw), + let logDuration = Double(sendLogDurationRaw), begin.isFinite, logDuration.isFinite, begin >= 0, + (1...ScreenSharingRtcEventLogDiagnostic.Window.maximumDurationSeconds).contains(logDuration), + begin + logDuration <= Double(duration) + else { + throw ScreenSharingError.invalid( + "The sender RTC event log needs both --sender-rtc-event-log-begin (>= 0) and --sender-rtc-event-log-duration (1...120) with the window inside --duration." + ) + } + guard mode != .receive, !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil, !flags.contains("--observe-window"), !flags.contains("--clock-sync"), + let reportURL + else { + throw ScreenSharingError.invalid( + "The sender RTC event log requires a sending media probe (--send or --loopback) with --report; no receiver-only, observer, clock sync or host check." + ) + } + let path = reportURL.path + ".sender-rtc-event-log.binarypb" + try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(path) + senderRtcEventLogWindow = (beginSeconds: begin, durationSeconds: logDuration) + senderRtcEventLogPath = path + } else { + senderRtcEventLogWindow = nil + senderRtcEventLogPath = nil + } + guard let pixelFormat = SyntheticPixelFormat(rawValue: values["--synthetic-format"] ?? "bgra") else { + throw ScreenSharingError.invalid("Synthetic format must be bgra or nv12.") + } + syntheticPixelFormat = pixelFormat + guard let videoCodec = ScreenSharingVideoCodec(rawValue: values["--codec"] ?? "h264") else { + throw ScreenSharingError.invalid("Codec must be h264, hevc or hevc444.") + } + self.videoCodec = videoCodec + if let captureFormat = values["--capture-format"] { + guard ["nv12", "bgra"].contains(captureFormat), displayID != nil || capturePicker, + !listDisplays, !capabilities, !checkCodecs, !showWorkload, !requestScreenRecording, + values["--check-host"] == nil, videoCodec != .hevc444 || captureFormat == "bgra" + else { throw ScreenSharingError.invalid("Capture format requires desktop capture; HEVC 4:4:4 requires BGRA.") } + capturePixelFormat = + captureFormat == "bgra" ? kCVPixelFormatType_32BGRA : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + } else { + capturePixelFormat = videoCodec.capturePixelFormat + } + if values["--codec"] != nil, + listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("--codec requires a media probe.") + } + if videoCodec == .hevc444, mode != .receive, + !standardRateControl || syntheticPixelFormat == .nv12 + { + throw ScreenSharingError.invalid("HEVC 4:4:4 requires standard rate control and full-chroma source input.") + } + if values["--synthetic-format"] != nil, + mode == .receive || displayID != nil || capturePicker || listDisplays || capabilities || checkCodecs + || showWorkload + || requestScreenRecording || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Synthetic format requires a synthetic sender or loopback.") + } + if standardRateControl, + mode == .receive || listDisplays || capabilities || checkCodecs || showWorkload || requestScreenRecording + || values["--check-host"] != nil + { + throw ScreenSharingError.invalid("Standard rate control requires sending media or loopback.") + } + codecCase = values["--codec-case"] + if codecCase != nil, !checkCodecs { throw ScreenSharingError.invalid("--codec-case requires --check-codecs.") } + if checkCodecs, mode != .loopback || checkQuality || displayID != nil { + throw ScreenSharingError.invalid("Codec checks require local synthetic mode without quality transitions.") + } + if let value = values["--check-host"] { + guard let url = URL(string: value), url.user == nil, url.password == nil, + url.scheme == "https" + || (url.scheme == "http" && ["localhost", "127.0.0.1", "[::1]"].contains(url.host ?? "")), + let workspace = UUID(uuidString: values["--workspace"] ?? ""), + let pane = UUID(uuidString: values["--pane"] ?? ""), mode == .loopback, !checkQuality + else { throw ScreenSharingError.invalid("Host checks require HTTPS (or loopback HTTP), a workspace and a pane.") } + hostCheck = (url, workspace, pane) + } else { + guard values["--workspace"] == nil, values["--pane"] == nil else { + throw ScreenSharingError.invalid("Workspace and pane require --check-host.") + } + hostCheck = nil + } + if checkQuality, mode != .loopback { throw ScreenSharingError.invalid("Quality checks require loopback mode.") } + if dropRecoveryKeyframe, !checkRecovery { + throw ScreenSharingError.invalid("Dropping recovery output requires --check-recovery.") + } + if headlessRecovery { + guard checkRecovery, !keepFront, !renderOnArrival, renderFPS == nil, !metalDisplayLink, + viewerDisplayID == nil, values["--drawable-count"] == nil, !unsyncedPresentation, !renderOffMain, + !capturePicker, displayID == nil + else { + throw ScreenSharingError.invalid( + "Headless recovery requires --check-recovery, synthetic input and no viewer options.") + } + } + if headless { + guard !keepFront, !renderOnArrival, renderFPS == nil, !metalDisplayLink, + viewerDisplayID == nil, values["--drawable-count"] == nil, !unsyncedPresentation, !renderOffMain, + !capturePicker, displayID == nil, !checkQuality, !checkCodecs, !showWorkload, + !listDisplays, !capabilities, !requestScreenRecording, hostCheck == nil + else { + throw ScreenSharingError.invalid("Headless media requires synthetic input and no viewer or other checks.") + } + } + if checkRecovery { + guard mode == .loopback, !checkQuality, !checkCodecs, !showWorkload, !listDisplays, !capabilities, + !requestScreenRecording, hostCheck == nil, keyframeIntervalSeconds >= 10, duration >= 10 + else { + throw ScreenSharingError.invalid( + "Recovery checks require media loopback, no other checks, a keyframe interval >=10 and duration >=10.") + } + } + if mode != .loopback, offerURL == nil || answerURL == nil { + throw ScreenSharingError.invalid("Send and receive require --offer and --answer paths.") + } + if showWorkload, + mode != .loopback || displayID != nil || checkCodecs || checkQuality || capabilities || listDisplays + || hostCheck != nil || offerURL != nil || answerURL != nil || renderOnArrival + { + throw ScreenSharingError.invalid("The desktop workload runs alone, without capture, peers or codec checks.") + } + if requestScreenRecording, + mode != .loopback || showWorkload || displayID != nil || checkCodecs || checkQuality || capabilities + || listDisplays || hostCheck != nil || offerURL != nil || answerURL != nil || reportURL != nil + { + throw ScreenSharingError.invalid("Request Screen Recording permission separately from other probe operations.") + } + let paths = [offerURL, answerURL, reportURL].compactMap { $0?.path } + guard Set(paths).count == paths.count else { throw ScreenSharingError.invalid("Output paths must be distinct.") } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeOwnedWorkloadWindow.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeOwnedWorkloadWindow.swift new file mode 100644 index 000000000..18a440a10 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeOwnedWorkloadWindow.swift @@ -0,0 +1,279 @@ +import AppKit +import CodevisorScreenSharing +import CoreGraphics + +/// The diagnostic's one owned workload window, the only thing its +/// ScreenCaptureKit stream may capture: exact backing-pixel size at the main +/// display's top-left corner, normal level, never activated, never key, ignores +/// input, no presentation-option or Space changes, never touches other windows. +/// Its lifecycle is finite and recorded: show → ready (first draw completed) +/// → capturing → optional animation pause (static content stays captured) +/// → capture stopped → closed. +/// The owned window can never become key or main, so showing it steals +/// nothing from the console user's session. +@MainActor +final class OwnedWorkloadWindow: NSWindow { + override var canBecomeKey: Bool { false } + override var canBecomeMain: Bool { false } + override func constrainFrameRect(_ frameRect: NSRect, to screen: NSScreen?) -> NSRect { frameRect } +} + +@MainActor +final class ProbeOwnedWorkloadWindow { + let window: OwnedWorkloadWindow + let view: WorkloadView + let rasterWidth: Int + let rasterHeight: Int + let recordDrawTimes: Bool + private let gate = ScreenSharingFirstDrawGate() + private(set) var session: ScreenSharingOwnedWindowSession! + /// Bounded named snapshots: each label is stored once (overwritten if taken + /// again) as soon as it is taken, so pre-start facts survive a later throw. + private(set) var snapshots: [String: [String: Any]] = [:] + + /// `stopCapture` stops the stream this window feeds; the session awaits it on + /// every path where the stream may have started before hiding the window. + /// `recordDrawTimes` is the explicit opt-in for the bounded first-draw-start + /// record (off by default; nothing is retained otherwise). + init( + configuration: ScreenSharingVideoConfiguration, recordDrawTimes: Bool = false, + stopCapture: @escaping @MainActor () async throws -> Void + ) + throws + { + guard let screen = NSScreen.main else { + throw ScreenSharingError.unavailable("No desktop display for the owned workload window.") + } + let painter = try ProbeDesktopPainter.make( + width: configuration.width, height: configuration.height, fps: configuration.framesPerSecond) + rasterWidth = configuration.width + rasterHeight = configuration.height + let view = WorkloadView(painter: painter, recordDrawTimes: recordDrawTimes) + self.recordDrawTimes = recordDrawTimes + self.view = view + let size = NSSize( + width: Double(configuration.width) / screen.backingScaleFactor, + height: Double(configuration.height) / screen.backingScaleFactor) + let frame = NSRect( + x: screen.frame.minX, y: screen.frame.maxY - size.height, width: size.width, height: size.height) + let window = OwnedWorkloadWindow(contentRect: frame, styleMask: [.borderless], backing: .buffered, defer: false) + window.hasShadow = false + window.isReleasedWhenClosed = false + window.title = "Codevisor · Screen Sharing Probe Owned Workload" + window.level = .normal + window.ignoresMouseEvents = true + window.isExcludedFromWindowsMenu = true + window.contentView = view + self.window = window + let gate = self.gate + view.onFirstDraw = { gate.signal() } + session = ScreenSharingOwnedWindowSession( + show: { + // On screen without activating the app or taking key/main. + window.orderFrontRegardless() + view.start() + }, + hide: { + view.stop() + window.orderOut(nil) + }, + stopCapture: stopCapture) + } + + var windowID: UInt32 { UInt32(max(0, window.windowNumber)) } + var lifecycle: ScreenSharingOwnedWorkloadLifecycle { session.lifecycle } + + /// Shows the window, waits for its first completed draw CALL — not a display + /// presentation — (deadline = deadlock guard; cancellation, timeout or + /// teardown resumes the wait once and blocks capture), confirms visibility, + /// then starts capture through `startCapture`. Any failure hides this window + /// once and rethrows; no lifecycle evidence is invented. Returns the + /// readiness record. + func start(timeoutSeconds: Double, startCapture: () async throws -> Void) async throws -> [String: Any] { + var ready: [String: Any] = [:] + try await session.start( + ready: { [self] in + try await gate.wait(timeout: .seconds(timeoutSeconds)) { try await Task.sleep(for: $0) } + guard window.isVisible, window.windowNumber > 0, let firstDrawStartedAt = view.firstDrawStartedAt, + let firstDrawCompletedAt = view.firstDrawCompletedAt + else { + throw ScreenSharingError.unavailable("Owned workload window is not visible.") + } + ready = [ + "windowID": window.windowNumber, "processID": ProcessInfo.processInfo.processIdentifier, + "framePoints": [window.frame.minX, window.frame.minY, window.frame.width, window.frame.height], + "contentWidthPoints": view.bounds.width, "contentHeightPoints": view.bounds.height, + "backingScaleFactor": window.backingScaleFactor, + "rasterWidth": rasterWidth, "rasterHeight": rasterHeight, + "displayID": (window.screen?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber) ?? 0, + "level": window.level.rawValue, "ignoresMouseEvents": window.ignoresMouseEvents, + "activated": NSApplication.shared.isActive, "isKeyWindow": window.isKeyWindow, + "startedAtSeconds": view.startedAt, + "firstDrawStartedAtSeconds": firstDrawStartedAt, "firstDrawCompletedAtSeconds": firstDrawCompletedAt, + "readinessMeaning": "first draw(_:) call completed on a visible window; not a display presentation time", + ] + ready["observationAtReadiness"] = observation( + "readiness (after first draw and visibility check, before capture start)") + }, + startCapture: startCapture) + ready["readyAtUptimeNs"] = session.lifecycle.timestampsNs[.ready] ?? 0 + return ready + } + + /// Stops the animation and freezes the frame code; the window and its last + /// frame stay on screen and stay captured. Returns the boundary record. + func pauseAnimation() throws -> [String: Any] { + try session.pauseWorkload { view.pause() } + return [ + "pausedAtSeconds": CACurrentMediaTime(), "pausedAtUptimeNs": session.lifecycle.timestampsNs[.pauseWorkload] ?? 0, + "observation": observation("workload pause"), + "drawCalls": view.drawCalls, "frozenCode": view.frozenCode ?? -1, "lastDrawnCode": view.sequence, + "frozen": view.isPaused, + "frozenCodeMeaning": + "the last DRAWN frame code, frozen as drawn (not re-derived from time) and held for every later redraw; not evidence of what the display showed", + "windowStillVisible": window.isVisible, + ] + } + + /// Stops the stream through the session so success or failure is preserved. + @discardableResult + func stopCapture() async -> Bool { await session.stopCapture() } + + /// The single cleanup used on every exit path; hides only this window, once. + @discardableResult + func cleanUp() -> ScreenSharingOwnedWorkloadLifecycle.CleanupOutcome { + // Resolve a still-pending readiness wait by teardown (not task cancellation). + gate.teardown() + return session.finish() + } + + /// One timestamped, read-only observation of the facts the missing-delivery + /// investigation needs. Nothing here prompts, activates, or changes state: + /// `CGPreflightScreenCaptureAccess()` is the GLOBAL non-prompting preflight + /// (false does not prove this owned-window filter must be refused); the CG + /// window record is restricted to this exact window number and pid; the + /// display predicates are for the window's own screen (unknown if none). + /// Coordinate conventions are kept raw and the single conversion is labelled. + @discardableResult + func observation(_ label: String) -> [String: Any] { + let record = takeObservation(label) + snapshots[label] = record + return record + } + + private func takeObservation(_ label: String) -> [String: Any] { + var record: [String: Any] = [ + "label": label, "atUptimeNs": ScreenSharingMetrics.nowNs, "atMediaTimeSeconds": CACurrentMediaTime(), + "screenCapturePreflight": CGPreflightScreenCaptureAccess(), + "screenCapturePreflightMeaning": + "global non-prompting CGPreflightScreenCaptureAccess(); false is not proof that this owned-window filter must be refused", + "activationPolicy": NSApplication.shared.activationPolicy().rawValue, + "appIsActive": NSApplication.shared.isActive, "appIsHidden": NSApplication.shared.isHidden, + "windowNumber": window.windowNumber, "processID": ProcessInfo.processInfo.processIdentifier, + "windowIsVisible": window.isVisible, "windowOcclusionVisible": window.occlusionState.contains(.visible), + "windowIsOnActiveSpace": window.isOnActiveSpace, "windowLevel": window.level.rawValue, + "windowFrameCocoaPoints": [window.frame.minX, window.frame.minY, window.frame.width, window.frame.height], + "windowFrameCocoaConvention": "NSWindow.frame: points, origin bottom-left of the main display (raw)", + "backingScaleFactor": window.backingScaleFactor, + ] + let mainBounds = CGDisplayBounds(CGMainDisplayID()) + let converted = ScreenSharingOwnedWindowGeometry.cocoaToTopLeft( + .init(x: window.frame.minX, y: window.frame.minY, width: window.frame.width, height: window.frame.height), + mainDisplayHeight: mainBounds.height) + record["windowFrameTopLeftConverted"] = [converted.x, converted.y, converted.width, converted.height] + record["windowFrameTopLeftConversion"] = + "y' = mainDisplayHeight - (y + height) with CGDisplayBounds(CGMainDisplayID()).height = \(mainBounds.height)" + // Own-window CG record: exact window number AND pid, or an explicit non-result. + let number = CGWindowID(max(0, window.windowNumber)) + var entries: [ScreenSharingOwnedWindowGeometry.WindowListEntry]? + if let list = CGWindowListCopyWindowInfo([.optionIncludingWindow], number) as? [[String: Any]] { + entries = [] + for item in list { + guard let itemNumber = item[kCGWindowNumber as String] as? UInt32 else { continue } + let owner = item[kCGWindowOwnerPID as String] as? Int32 // may be unreported + var bounds: ScreenSharingOwnedWindowGeometry.Rect? + if let dictionary = item[kCGWindowBounds as String] as? [String: Double], let x = dictionary["X"], + let y = dictionary["Y"], let width = dictionary["Width"], let height = dictionary["Height"] + { + bounds = .init(x: x, y: y, width: width, height: height) + } + entries?.append( + .init( + number: itemNumber, ownerPID: owner, bounds: bounds, layer: item[kCGWindowLayer as String] as? Int, + isOnscreen: item[kCGWindowIsOnscreen as String] as? Bool, alpha: item[kCGWindowAlpha as String] as? Double)) + } + } + switch ScreenSharingOwnedWindowGeometry.ownWindow( + in: entries, number: number, pid: ProcessInfo.processInfo.processIdentifier) + { + case .found(let entry): + record["cgWindowRecord"] = [ + "number": entry.number, "ownerPID": entry.ownerPID, + "boundsTopLeftPoints": entry.bounds.map { [$0.x, $0.y, $0.width, $0.height] } ?? "unreported", + "boundsConvention": "kCGWindowBounds: points, origin top-left of the main display (raw)", + "layer": entry.layer ?? "unreported", "isOnscreen": entry.isOnscreen ?? "unreported", + "alpha": entry.alpha ?? "unreported", + ] + case .queryUnavailable: record["cgWindowRecord"] = "query unavailable (CGWindowListCopyWindowInfo returned nil)" + case .absent: record["cgWindowRecord"] = "absent (query succeeded; no entry with this window number)" + case .ownerUnreported: + record["cgWindowRecord"] = "exact window number present but owner pid unreported; fields not persisted" + case .ownerMismatch(let pid): record["cgWindowRecord"] = "owner mismatch: reported pid \(pid); fields not persisted" + case .duplicate(let count): record["cgWindowRecord"] = "duplicate: \(count) entries; fields not persisted" + } + // Display predicates for the window's own screen; a missing screen is unknown. + if let screen = window.screen, + let displayID = (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value + { + let bounds = CGDisplayBounds(displayID) + record["display"] = [ + "displayID": displayID, "isActive": CGDisplayIsActive(displayID) != 0, + "isAsleep": CGDisplayIsAsleep(displayID) != 0, + "isOnline": CGDisplayIsOnline(displayID) != 0, "isMain": CGDisplayIsMain(displayID) != 0, + "screenFrameCocoaPoints": [screen.frame.minX, screen.frame.minY, screen.frame.width, screen.frame.height], + "displayBoundsTopLeftPoints": [bounds.minX, bounds.minY, bounds.width, bounds.height], + ] + } else { + record["display"] = "unknown (window has no screen)" + } + return record + } + + /// The owned workload's draw-timestamp report in the SAME schema as the + /// desktop workload report the image-age analyzer consumes (`drawSamples` + /// of `{code, startedAtSeconds}` + `drawSamplesTruncated`), plus the + /// owned-window facts. Nil unless recording was opted in. + /// `mediaMeasuredSeconds` is the probe's media measurement (its origin is the + /// media-ready boundary); the workload's own interval is computed here from + /// its start (`WorkloadView.start()` sets `startedAt` before scheduling the + /// first draw; the first draw start is recorded separately in the readiness + /// record) to now, so the two origins are never mixed. + func workloadTimesReport(mediaMeasuredSeconds: Double) -> [String: Any]? { + guard recordDrawTimes else { return nil } + let endedAt = CACurrentMediaTime() + return [ + "kind": "owned-workload", + "source": "owned window captured through ScreenCaptureKit (currentProcess selection)", + "timingScope": "AppKit draw-call starts and frozen-code pause; not physical presentation or input-to-photon", + "rasterWidth": rasterWidth, "rasterHeight": rasterHeight, + "windowWidthPoints": view.bounds.width, "windowHeightPoints": view.bounds.height, + "backingScaleFactor": window.backingScaleFactor, "requestedFPS": view.fps, + "startedAtSeconds": view.startedAt, "endedAtSeconds": endedAt, "elapsedSeconds": endedAt - view.startedAt, + "mediaMeasuredSeconds": mediaMeasuredSeconds, + "origins": + "startedAtSeconds/endedAtSeconds/elapsedSeconds: this workload's start (set before its first draw is scheduled) to report time (CACurrentMediaTime); firstDrawStartedAtSeconds lives in the readiness record; mediaMeasuredSeconds: the probe's media measurement from its media-ready boundary", + "drawCalls": view.drawCalls, + "lastSequence": view.sequence, "frozenCode": view.frozenCode ?? -1, "frozen": view.isPaused, + "responses": 0, "eventSamples": [] as [Any], "eventSamplesTruncated": false, + "drawSamples": view.drawSamples.map { ["code": $0.code, "startedAtSeconds": $0.startedAtSeconds] }, + "drawSamplesTruncated": view.drawSamplesTruncated, + "drawSampleLimit": ScreenSharingDrawTimestampRecord.defaultLimit, + ] + } + + var lifecycleRecord: [String: Any] { + var record = session.record + record["workloadFrozen"] = view.isPaused + return record + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeTimeline.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeTimeline.swift new file mode 100644 index 000000000..7d90c66da --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeTimeline.swift @@ -0,0 +1,57 @@ +import CodevisorScreenSharing +import Darwin +import Foundation + +/// At most 361 samples (the CLI bounds duration / interval to 360, plus the +/// final sample); each sample copies counters, labels and bounded timing +/// summaries, not raw histories. CPU is whole-process user+system time (100% +/// is one core); GPU utilization is not measured here. +@MainActor +struct ProbeTimeline { + struct Sample: Encodable { + let elapsedSeconds: Double + let residentBytes: UInt64? + let physicalFootprintBytes: UInt64? + let cpuPercentOfOneCore: Double? + let presentedFramesPerSecond: Double? + let sender: ScreenSharingMetrics.Snapshot + let receiver: ScreenSharingMetrics.Snapshot + let rendererMailboxDrops: Int + } + private(set) var samples: [Sample] = [] + private var previous: (time: Double, cpu: Double, frames: Int)? + + mutating func append( + elapsed: Double, sender: ScreenSharingMetrics.Snapshot, receiver: ScreenSharingMetrics.Snapshot, drops: Int + ) { + var memory = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let memoryStatus = withUnsafeMutablePointer(to: &memory) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + var usage = rusage() + let cpuStatus = getrusage(RUSAGE_SELF, &usage) + let cpu = + Double(usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) + + Double(usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1_000_000 + let frames = receiver.counters["presentedFrames", default: 0] + let interval = previous.map { elapsed - $0.time } ?? 0 + samples.append( + Sample( + elapsedSeconds: elapsed, + residentBytes: memoryStatus == KERN_SUCCESS ? memory.resident_size : nil, + physicalFootprintBytes: memoryStatus == KERN_SUCCESS ? memory.phys_footprint : nil, + cpuPercentOfOneCore: cpuStatus == 0 && interval > 0 ? (cpu - (previous?.cpu ?? cpu)) / interval * 100 : nil, + presentedFramesPerSecond: interval > 0 ? Double(frames - (previous?.frames ?? frames)) / interval : nil, + sender: sender, receiver: receiver, rendererMailboxDrops: drops)) + previous = cpuStatus == 0 ? (elapsed, cpu, frames) : nil + } + func writeProgress(to report: URL?) throws { + guard let report else { return } + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(samples).write(to: URL(fileURLWithPath: report.path + ".progress.json"), options: .atomic) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeTransportTimeline.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeTransportTimeline.swift new file mode 100644 index 000000000..72e983aea --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeTransportTimeline.swift @@ -0,0 +1,23 @@ +import CodevisorScreenSharing + +/// Per-second interval means; never present percentiles of these as per-frame p95. +struct ProbeTransportTimeline { + struct Sample: Encodable { + let elapsedSeconds: Double + let senderIntervalMeans: [String: Double] + let receiverIntervalMeans: [String: Double] + let senderRTC: [String: String] + let receiverRTC: [String: String] + } + private var sender = ScreenSharingRTCIntervalMetrics() + private var receiver = ScreenSharingRTCIntervalMetrics() + private(set) var samples: [Sample] = [] + + mutating func append(elapsed: Double, sender sent: [String: String], receiver received: [String: String]) { + guard samples.count < 3602 else { return } + samples.append( + Sample( + elapsedSeconds: elapsed, senderIntervalMeans: sender.update(sent), + receiverIntervalMeans: receiver.update(received), senderRTC: sent, receiverRTC: received)) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeWindowObservation.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeWindowObservation.swift new file mode 100644 index 000000000..5813e4ded --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeWindowObservation.swift @@ -0,0 +1,271 @@ +import AppKit +import CodevisorScreenSharing +import CoreMedia +import QuartzCore +@preconcurrency import ScreenCaptureKit + +/// Samples only the benchmark marker from a user-selected window. No frames, +/// audio or input are retained. An external analysis combines these local host +/// timestamps with the workload's clock and measured clock-offset bounds. +@MainActor +enum ProbeWindowObservation { + struct Options { + let sourceWidth: Int + let sourceHeight: Int + let contentTopPoints: Double + let duration: Double + let report: URL + let windowID: UInt32? + let captureKind: String + + init(arguments: [String]) throws { + let allowed = Set([ + "--width", "--height", "--content-top-points", "--duration", "--report", "--window-id", "--capture-kind", + ]) + guard arguments.count.isMultiple(of: 2) else { + throw ScreenSharingError.invalid("Observation options require values.") + } + var values: [String: String] = [:] + for index in stride(from: 0, to: arguments.count, by: 2) { + let key = arguments[index] + guard allowed.contains(key), values[key] == nil else { + throw ScreenSharingError.invalid("Unsupported or repeated observation option: \(key)") + } + values[key] = arguments[index + 1] + } + // Below 900 source pixels the workload response panel overlaps the marker. + guard let width = Int(values["--width"] ?? "3840"), (900...3840).contains(width), + let height = Int(values["--height"] ?? "2160"), (360...2160).contains(height), + let top = Double(values["--content-top-points"] ?? "0"), top.isFinite, (0...300).contains(top), + let duration = Double(values["--duration"] ?? "30"), duration.isFinite, (1...120).contains(duration), + let path = values["--report"], !path.isEmpty + else { throw ScreenSharingError.invalid("Invalid observation geometry, duration or report path.") } + sourceWidth = width; sourceHeight = height; contentTopPoints = top + self.duration = duration + report = URL(fileURLWithPath: path).standardizedFileURL + if let value = values["--window-id"] { + guard let id = UInt32(value), id > 0 else { throw ScreenSharingError.invalid("Invalid window ID.") } + windowID = id + } else { + windowID = nil + } + captureKind = values["--capture-kind"] ?? "window" + guard ["window", "display-crop"].contains(captureKind), captureKind != "display-crop" || windowID != nil else { + throw ScreenSharingError.invalid("Choose window or display-crop capture; display-crop requires a window ID.") + } + } + } + + static func run(options: Options) async throws { + let picker = ProbeCapturePicker() + defer { picker.stop() } + let filter: SCContentFilter + var sourceRect: CGRect? + var displayID: UInt32? + if let windowID = options.windowID { + guard CGPreflightScreenCaptureAccess() else { + throw ScreenSharingError.unavailable("Direct window observation requires existing Screen Recording permission.") + } + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) + guard let window = content.windows.first(where: { $0.windowID == windowID }) else { + throw ScreenSharingError.unavailable("The requested window is no longer available.") + } + if options.captureKind == "display-crop" { + guard window.isOnScreen, + let display = content.displays.first(where: { $0.frame.contains(window.frame) }) + else { + throw ScreenSharingError.unavailable( + "Display observation requires the entire window on one attached display. " + + "Window \(windowID): onScreen=\(window.isOnScreen), frame=\(window.frame); " + + "displays=\(content.displays.map { "\($0.displayID):\($0.frame)" })") + } + filter = SCContentFilter(display: display, excludingWindows: []) + sourceRect = window.frame.offsetBy(dx: -display.frame.minX, dy: -display.frame.minY) + displayID = display.displayID + } else { + filter = SCContentFilter(desktopIndependentWindow: window) + } + } else { + filter = try await picker.choose(window: true) + } + let bounds = sourceRect ?? filter.contentRect + let contentHeight = bounds.height - options.contentTopPoints + guard bounds.width >= 300, contentHeight >= 180, + abs(bounds.width / contentHeight - Double(options.sourceWidth) / Double(options.sourceHeight)) < 0.02 + else { throw ScreenSharingError.invalid("Selected window/content inset does not match the source aspect ratio.") } + let scale = min(Double(filter.pointPixelScale), 1920 / bounds.width) + let width = Int((bounds.width * scale).rounded()) + let height = Int((bounds.height * scale).rounded()) + let config = SCStreamConfiguration() + config.width = width; config.height = height + config.pixelFormat = kCVPixelFormatType_32BGRA + config.minimumFrameInterval = CMTime(value: 1, timescale: 120) + config.queueDepth = 3 + config.showsCursor = false + config.capturesAudio = false + config.ignoreShadowsSingleWindow = true + config.scalesToFit = true + if let sourceRect { config.sourceRect = sourceRect } + let output = ObservationOutput( + sourceWidth: options.sourceWidth, sourceHeight: options.sourceHeight, + topFraction: options.contentTopPoints / bounds.height) + let stream = SCStream(filter: filter, configuration: config, delegate: output) + try stream.addStreamOutput(output, type: .screen, sampleHandlerQueue: output.queue) + let started = CACurrentMediaTime() + let clockDifference = CMTimeGetSeconds(CMClockGetTime(CMClockGetHostTimeClock())) - CACurrentMediaTime() + try await stream.startCapture() + var ready: [String: Any] = [ + "captureKind": options.captureKind, + "widthPoints": bounds.width, "heightPoints": bounds.height, + "contentTopPoints": options.contentTopPoints, "captureWidth": width, "captureHeight": height, + "startedAtSeconds": started, "hostClockMinusCoreAnimationSeconds": clockDifference, + ] + if let displayID { ready["displayID"] = displayID } + if let sourceRect { + ready["sourceRectPoints"] = [sourceRect.minX, sourceRect.minY, sourceRect.width, sourceRect.height] + } + do { + try JSONSerialization.data(withJSONObject: ready, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: options.report.path + ".ready.json"), options: .atomic) + try await Task.sleep(for: .seconds(options.duration)) + try await stream.stopCapture() + } catch { + try? await stream.stopCapture() + throw error + } + let snapshot = await output.finish() + let report = Report( + captureKind: options.captureKind, displayID: displayID, + sourceRectPoints: sourceRect.map { [$0.minX, $0.minY, $0.width, $0.height] }, + sourceWidth: options.sourceWidth, sourceHeight: options.sourceHeight, + contentTopPoints: options.contentTopPoints, captureWidth: width, captureHeight: height, + elapsedSeconds: CACurrentMediaTime() - started, + hostClockMinusCoreAnimationSeconds: clockDifference, observation: snapshot) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(report).write(to: options.report, options: .atomic) + guard snapshot.failure == nil, !snapshot.truncated, !snapshot.samples.isEmpty, + snapshot.samples.contains(where: { $0.code != nil }) + else { throw ScreenSharingError.unavailable("Observation did not produce valid bounded marker samples.") } + print( + "Observed \(snapshot.samples.count) \(options.captureKind) samples; image-age analysis requires clock calibration." + ) + } + + private struct Report: Encodable { + let kind = "window-marker-observation" + let timingScope = "Local ScreenCaptureKit sample timestamps; not physical presentation or input-to-photon" + let captureKind: String + let displayID: UInt32? + let sourceRectPoints: [Double]? + let sourceWidth: Int + let sourceHeight: Int + let contentTopPoints: Double + let captureWidth: Int + let captureHeight: Int + let elapsedSeconds: Double + let hostClockMinusCoreAnimationSeconds: Double + let observation: ObservationOutput.Snapshot + } +} + +private final class ObservationOutput: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable { + let queue = DispatchQueue(label: "codevisor.probe.window-observation", qos: .userInteractive) + private let sourceWidth: Int + private let sourceHeight: Int + private let topFraction: Double + private let lock = NSLock() + private var samples: [Sample] = [] + private var incompleteSamples = 0 + private var truncated = false + private var failure: String? + + struct Sample: Encodable, Sendable { + let samplePTSSeconds: Double + let callbackSeconds: Double + let code: Int? + let width: Int + let height: Int + } + + struct Snapshot: Encodable, Sendable { + let samples: [Sample] + let incompleteSamples: Int + let truncated: Bool + let failure: String? + } + + init(sourceWidth: Int, sourceHeight: Int, topFraction: Double) { + self.sourceWidth = sourceWidth; self.sourceHeight = sourceHeight; self.topFraction = topFraction + } + + func stream(_ stream: SCStream, didOutputSampleBuffer buffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard type == .screen else { return } + let callback = CACurrentMediaTime() + guard buffer.isValid, + let attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, createIfNecessary: false) + as? [[SCStreamFrameInfo: Any]], + let status = attachments.first?[.status] as? Int, status == SCFrameStatus.complete.rawValue, + let pixel = buffer.imageBuffer, buffer.presentationTimeStamp.isNumeric + else { lock.withLock { incompleteSamples += 1 }; return } + let sample = Sample( + samplePTSSeconds: CMTimeGetSeconds(buffer.presentationTimeStamp), callbackSeconds: callback, + code: decode(pixel), width: CVPixelBufferGetWidth(pixel), height: CVPixelBufferGetHeight(pixel)) + lock.withLock { + guard samples.count < 20_000 else { truncated = true; return } + samples.append(sample) + } + } + + func stream(_ stream: SCStream, didStopWithError error: any Error) { + lock.withLock { failure = error.localizedDescription } + } + + func finish() async -> Snapshot { + await withCheckedContinuation { continuation in + queue.async { + continuation.resume( + returning: self.lock.withLock { + Snapshot( + samples: self.samples, incompleteSamples: self.incompleteSamples, + truncated: self.truncated, failure: self.failure) + }) + } + } + } + + private func decode(_ pixel: CVPixelBuffer) -> Int? { + guard CVPixelBufferGetPixelFormatType(pixel) == kCVPixelFormatType_32BGRA, + CVPixelBufferLockBaseAddress(pixel, .readOnly) == kCVReturnSuccess + else { return nil } + defer { CVPixelBufferUnlockBaseAddress(pixel, .readOnly) } + guard let raw = CVPixelBufferGetBaseAddress(pixel) else { return nil } + let width = CVPixelBufferGetWidth(pixel), height = CVPixelBufferGetHeight(pixel) + let stride = CVPixelBufferGetBytesPerRow(pixel) + let bytes = raw.assumingMemoryBound(to: UInt8.self) + let top = Double(height) * topFraction + let cy = Int((top + 105 * (Double(height) - top) / Double(sourceHeight)).rounded()) + guard cy >= 1, cy + 1 < height else { return nil } + var cells: [Double] = [] + for index in 0..<20 { + let cx = Int((Double(36 + 28 * index) * Double(width) / Double(sourceWidth)).rounded()) + guard cx >= 1, cx + 1 < width else { return nil } + var values: [Double] = [] + for y in (cy - 1)...(cy + 1) { + for x in (cx - 1)...(cx + 1) { + let offset = y * stride + x * 4 + values.append((Double(bytes[offset]) + Double(bytes[offset + 1]) + Double(bytes[offset + 2])) / 3) + } + } + cells.append(values.sorted()[4]) + } + guard let white = cells.first, let black = cells.last, white - black >= 120 else { return nil } + var code = 0 + for cell in cells.dropFirst().dropLast() { + let normalized = (cell - black) / (white - black) + guard normalized <= 0.2 || normalized >= 0.8 else { return nil } + code = (code << 1) | (normalized >= 0.5 ? 1 : 0) + } + return code + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ProbeWorkloadView.swift b/packages/swift/CodevisorScreenSharing/Probe/ProbeWorkloadView.swift new file mode 100644 index 000000000..7063a80f8 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ProbeWorkloadView.swift @@ -0,0 +1,123 @@ +import AppKit +import CodevisorScreenSharing +import QuartzCore + +/// Drawing window and view shared by the visible desktop workload and the +/// owned-window capture diagnostic. Counters describe AppKit draw calls and +/// received events, never claimed source/display presentation times. +@MainActor +final class WorkloadWindow: NSWindow { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } + // The diagnostic may need a 4K backing surface on a smaller physical display. + override func constrainFrameRect(_ frameRect: NSRect, to screen: NSScreen?) -> NSRect { frameRect } +} + +struct WorkloadDraw: Encodable { + let code: Int + let startedAtSeconds: Double +} + +struct WorkloadEvent: Encodable { + let kind: String + let response: Int + let elapsedSeconds: Double +} + +@MainActor +final class WorkloadView: NSView { + let painter: ProbeDesktopPainter + let fps: Int + private var timer: Timer? + private(set) var startedAt = 0.0 + private(set) var drawCalls = 0 + private(set) var sequence = 0 + private(set) var responses = 0 + private(set) var events: [WorkloadEvent] = [] + private let recordDrawTimes: Bool + /// Bounded first-draw-start record (shared semantics with the image-age analyzer). + private var drawRecord = ScreenSharingDrawTimestampRecord() + var drawSamples: [WorkloadDraw] { + drawRecord.samples.map { WorkloadDraw(code: $0.code, startedAtSeconds: $0.startedAtSeconds) } + } + var drawSamplesTruncated: Bool { drawRecord.truncated } + /// Core Animation time at which the first `draw(_:)` call started, and the + /// time at which that call returned. Neither is a presentation time: the + /// window server displays the drawn content later. + private(set) var firstDrawStartedAt: Double? + private(set) var firstDrawCompletedAt: Double? + /// Called once, on the main actor, right after the first draw call returns. + var onFirstDraw: (() -> Void)? + /// Frozen at a pause: later redraws keep the same code (see the sequence type). + private(set) var codes: ScreenSharingWorkloadSequence? + var isPaused: Bool { codes?.isFrozen ?? false } + override var acceptsFirstResponder: Bool { true } + override var isOpaque: Bool { true } + + init(painter: ProbeDesktopPainter, recordDrawTimes: Bool) { + self.painter = painter + self.fps = painter.fps + self.recordDrawTimes = recordDrawTimes + super.init(frame: .zero) + } + + required init?(coder: NSCoder) { nil } + + func start() { + startedAt = CACurrentMediaTime() + codes = ScreenSharingWorkloadSequence(framesPerSecond: fps, startedAtSeconds: startedAt) + let timer = Timer( + timeInterval: 1 / Double(fps), target: self, selector: #selector(tick), userInfo: nil, repeats: true) + self.timer = timer + RunLoop.main.add(timer, forMode: .common) + needsDisplay = true + } + + func stop() { timer?.invalidate(); timer = nil } + + /// Stops the animation and freezes the frame code at the LAST DRAWN code — + /// not a newly time-derived one — so a pause between frames holds what was + /// actually rendered last, through any later redraw. Distinct from stopping + /// a capture stream. + func pause() { + stop() + codes?.freezeAtLastDrawn(atSeconds: CACurrentMediaTime()) + } + + /// The frozen code after a pause; equals `sequence` (the last drawn code). + var frozenCode: Int? { codes?.frozenCode } + + @objc private func tick() { needsDisplay = true } + + override func mouseDown(with event: NSEvent) { respond(kind: "mouseDown") } + override func keyDown(with event: NSEvent) { + if !event.isARepeat { respond(kind: "keyDown") } + } + + private func respond(kind: String) { + responses += 1 + if events.count < 128 { + events.append(WorkloadEvent(kind: kind, response: responses, elapsedSeconds: CACurrentMediaTime() - startedAt)) + } + needsDisplay = true + } + + override func draw(_ dirtyRect: NSRect) { + guard startedAt > 0, let context = NSGraphicsContext.current?.cgContext else { return } + drawCalls += 1 + let drawStarted = CACurrentMediaTime() + // A paused view redraws its frozen code unchanged (window server + // requests only); it never advances the content. `sequence` is the last + // DRAWN code, not evidence of what the display showed. + sequence = codes?.drawn(atSeconds: drawStarted) ?? 0 + if recordDrawTimes { drawRecord.record(code: sequence, startedAtSeconds: drawStarted) } + painter.draw(in: context, bounds: bounds, sequence: sequence, responses: responses) + if firstDrawStartedAt == nil { + firstDrawStartedAt = drawStarted + firstDrawCompletedAt = CACurrentMediaTime() + let callback = onFirstDraw + onFirstDraw = nil + callback?() + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Probe/ScreenSharingProbe.swift b/packages/swift/CodevisorScreenSharing/Probe/ScreenSharingProbe.swift new file mode 100644 index 000000000..42a246245 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/ScreenSharingProbe.swift @@ -0,0 +1,1083 @@ +#if os(macOS) + import AppKit + import CodevisorScreenSharing + import Foundation + import QuartzCore + import ScreenCaptureKit + @preconcurrency import WebRTC + + @main + @MainActor + struct ScreenSharingProbe { + static func main() { + if CommandLine.arguments.contains("--help") { print(ProbeOptions.usage); return } + do { + if CommandLine.arguments.dropFirst().first == "--clock-sync" { + try ProbeClockSync.run() + return + } + if CommandLine.arguments.dropFirst().first == "--observe-window" { + let options = try ProbeWindowObservation.Options(arguments: Array(CommandLine.arguments.dropFirst(2))) + let app = NSApplication.shared + app.setActivationPolicy(.regular) + Task { @MainActor in + do { + try await ProbeWindowObservation.run(options: options) + exit(EXIT_SUCCESS) + } catch { + FileHandle.standardError.write(Data("Window observation: \(error.localizedDescription)\n".utf8)) + exit(EXIT_FAILURE) + } + } + app.run() + return + } + let options = try ProbeOptions(arguments: Array(CommandLine.arguments.dropFirst())) + // This standalone executable owns its process. M152 exposes these experiments through process-wide trials; + // they are installed once, through the single boundary, before any RTC call. + try ScreenSharingFieldTrials.process.install(options.fieldTrialSelection) + let app = NSApplication.shared + // Headless stays prohibited (no windows); only the owned-window capture + // mode runs as an accessory so it may own one window without activation. + app.setActivationPolicy(options.captureOwnedWindow ? .accessory : options.headless ? .prohibited : .regular) + let runner = ProbeRunner(options: options) + Task { @MainActor in + do { try await runner.run(); exit(EXIT_SUCCESS) } catch { + await runner.stop() + runner.writeFailureRecord(error) + FileHandle.standardError.write(Data("Screen Sharing probe: \(error.localizedDescription)\n".utf8)) + exit(EXIT_FAILURE) + } + } + withExtendedLifetime(runner) { app.run() } + } catch { + FileHandle.standardError.write(Data("\(error.localizedDescription)\n".utf8)) + exit(EXIT_FAILURE) + } + } + } + + @MainActor + private final class ProbeRunner: NSObject, NSWindowDelegate { + let options: ProbeOptions + let senderMetrics = ScreenSharingMetrics() + let receiverMetrics = ScreenSharingMetrics() + var sender: ScreenSharingPeer? + var receiver: ScreenSharingPeer? + var synthetic: SyntheticSource? + var capture: ScreenSharingCapture? + var ownedWorkload: ProbeOwnedWorkloadWindow? + /// Owned-window mode: bounded first-observed delivery state, nil until + /// measurement begins; kept on the runner so failure evidence keeps it. + var firstObservation: ScreenSharingFirstObservation? + var measurementStartedNs: Int64? + /// Media start (every mode) on the metrics uptime clock; the event-log diagnostic's measured seconds use it. + var mediaStartedNs: Int64? + var picker: ProbeCapturePicker? + var window: NSWindow? + var metalView: ScreenSharingMetalView? + var deliveryAudit: ScreenSharingFrameDeliveryAudit? + /// Receiver-only diagnostic RTC event-log lifecycle (nil = nothing allocated or scheduled). + var rtcEventLog: ScreenSharingRtcEventLogDiagnostic? + /// Sender-only diagnostic RTC event-log lifecycle on the sending peer (nil = nothing allocated or scheduled). + var senderRtcEventLog: ScreenSharingRtcEventLogDiagnostic? + /// Outcome of each role's RTC event-log sidecar write made in stop() ("written …" or "write failed: …"); absent = not requested. + var rtcEventLogSidecarOutcomes: [ScreenSharingRtcEventLogDiagnostic.Role: String] = [:] + var displayLink: ProbeMetalDisplayLink? + var encoderLogger: RTCCallbackLogger? + + init(options: ProbeOptions) { self.options = options } + + func run() async throws { + let build = + Bundle.main.object(forInfoDictionaryKey: "CodevisorProbeBuildConfiguration") as? String ?? "unspecified" + senderMetrics.label("probeBuildConfiguration", build) + receiverMetrics.label("probeBuildConfiguration", build) + if let window = options.jitterWindowFrames { + receiverMetrics.label("jitterEstimatorExperiment", "frame-size p95 over \(window) frames") + } + // `playoutExperiment` is published by the peer from the INSTALLED trial selection (identical native string), + // so the label can never describe an option that failed to install. + if options.requestScreenRecording { + NSApplication.shared.activate() + if !CGPreflightScreenCaptureAccess() { + _ = CGRequestScreenCaptureAccess() + // Keep the app/run loop alive while the system presents its prompt. + // Exiting immediately can dismiss the asynchronous permission UI. + let deadline = ContinuousClock.now.advanced(by: .seconds(options.duration)) + while !CGPreflightScreenCaptureAccess(), ContinuousClock.now < deadline { + try await Task.sleep(for: .seconds(1)) + } + } + guard CGPreflightScreenCaptureAccess() else { + throw ScreenSharingError.unavailable( + "Allow Screen Sharing Probe in System Settings > Privacy & Security > Screen & System Audio Recording, then relaunch." + ) + } + print("Screen Recording permission is granted. No capture started.") + return + } + if options.showWorkload { + try await ProbeDesktopWorkload.run(options: options) + return + } + if options.checkCodecs { + let configuration = options.configuration + let report = options.reportURL + let codecCase = options.codecCase + try await Task.detached { + try ProbeCodecCheck.run(configuration: configuration, report: report, caseName: codecCase) + }.value + return + } + if let host = options.hostCheck { + try await ProbeHostRecovery(url: host.url, workspace: host.workspace, pane: host.pane).run( + report: options.reportURL) + return + } + if options.capabilities { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + let data = try encoder.encode(ProbeCapabilities.read()) + if let url = options.reportURL { try data.write(to: url, options: .atomic) } + print(String(decoding: data, as: UTF8.self)) + return + } + if options.listDisplays { + for display in try await ScreenSharingCapture.displays() { + print("\(display.id): \(display.width) × \(display.height)") + } + return + } + var pickedFilter: SCContentFilter? + if options.capturePicker { + let picker = ProbeCapturePicker() + self.picker = picker + pickedFilter = try await picker.choose(window: options.capturePickerWindow) + senderMetrics.label( + "captureSelection", options.capturePickerWindow ? "macOS system window picker" : "macOS system display picker" + ) + } + // Keep the activity scoped to this finite media run, including error exits. + // This isolates background scheduling without changing system sleep policy. + let activity = + options.userInitiatedActivity + ? ProcessInfo.processInfo.beginActivity( + options: .userInitiatedAllowingIdleSystemSleep, reason: "Screen sharing benchmark") + : nil + defer { if let activity { ProcessInfo.processInfo.endActivity(activity) } } + for metrics in [senderMetrics, receiverMetrics] { + metrics.label("processActivity", activity == nil ? "none" : "user initiated; idle sleep allowed") + } + if options.traceBoundary { + for metrics in [senderMetrics, receiverMetrics] { + metrics.enableTracing() + metrics.label("boundaryTracing", "enabled: bounded refresh/encoder/decoder traces") + } + } + if options.mode != .receive { + let logger = RTCCallbackLogger() + if options.traceBoundary { logger.severity = .verbose } + let metrics = senderMetrics + let tracing = options.traceBoundary + logger.start { @Sendable message in + for (name, count) in ScreenSharingEncoderDropLog.counters(in: message) { + metrics.increment(name, by: count) + } + // Bounded boundary diagnostic: which WebRTC path dropped or held a + // submitted frame before it reached the native encoder. Addresses are + // stripped; only the pinned drop/pause messages are retained. + if tracing, let sanitized = ScreenSharingEncoderDropLog.dropDiagnostic(in: message) { + metrics.trace("webrtcDropLog", "\(ScreenSharingMetrics.nowNs) \(sanitized)") + } + } + encoderLogger = logger + sender = try ScreenSharingPeer( + sending: true, configuration: options.configuration, metrics: senderMetrics, + useLowLatencyRateControl: !options.standardRateControl, codec: options.videoCodec, + disableLookAhead: options.disableLookAhead, maximumPendingFrames: options.encoderInFlight, + maintainSourceRate: options.maintainSourceRate, staticCodecRate: options.staticCodecRate, + completeEachFrame: options.completeEachFrame, prioritizeSpeed: options.prioritizeSpeed, + keyframeIntervalSeconds: options.keyframeIntervalSeconds, + sourceIdleThresholdNs: options.idleThresholdMs.map { Int64($0) * 1_000_000 }) + if let threshold = options.idleThresholdMs { + senderMetrics.label("sourceIdleThresholdExperiment", "\(threshold) ms idle threshold") + } + sender?.onConnectionChanged = { print("Sender: \($0)") } + if let window = options.senderRtcEventLogWindow, let path = options.senderRtcEventLogPath, let peer = sender { + // Sender-only diagnostic: the same lifecycle on the sending peer (bracket A on its own start/stop calls). + let log = ScreenSharingRtcEventLogDiagnostic( + window: try .init(beginSeconds: window.beginSeconds, durationSeconds: window.durationSeconds), path: path, + role: .sender, + boundaries: .init( + clock: { ScreenSharingRtcEventLogDiagnostic.defaultClock() }, + start: { path, maxSizeBytes in peer.startRtcEventLog(path: path, maxSizeBytes: maxSizeBytes) }, + stop: { peer.stopRtcEventLog() })) + senderRtcEventLog = log + senderMetrics.label( + "rtcEventLog", + "sender window begin \(log.window.beginSeconds) s duration \(log.window.durationSeconds) s, cap \(log.maxSizeBytes) B, bracket A (CA before/after start and stop); outgoing events at the post-pacer transport hand-off" + ) + } + } + if options.mode != .send { + // Receiver-only diagnostic frame-delivery audit: allocated only when requested; nil means no work anywhere. + let deliveryAudit = try options.deliveryAuditWindow.map { window in + ScreenSharingFrameDeliveryAudit( + window: try .init(beginSeconds: window.beginSeconds, durationSeconds: window.durationSeconds)) + } + self.deliveryAudit = deliveryAudit + if let deliveryAudit { + receiverMetrics.label( + "deliveryAudit", + "window begin \(deliveryAudit.window.beginSeconds) s duration \(deliveryAudit.window.durationSeconds) s, capacity \(deliveryAudit.capacity) records × \(ScreenSharingFrameDeliveryAudit.recordByteStride) B" + ) + } + let receiver = try ScreenSharingPeer( + sending: false, configuration: options.configuration, metrics: receiverMetrics, codec: options.videoCodec, + deliveryGrace: options.idleGraceMs.map { .milliseconds($0) }, + deliveryGraceExtensions: options.idleGraceExtensions, frameDeliveryAudit: deliveryAudit) + if let grace = options.idleGraceMs { + receiverMetrics.label("sourceIdleGraceExperiment", "\(grace) ms delivery grace") + } + if let extensions = options.idleGraceExtensions { + receiverMetrics.label("sourceIdleGraceExtensionExperiment", "up to \(extensions) progress extensions") + } + self.receiver = receiver + if let window = options.rtcEventLogWindow, let path = options.rtcEventLogPath { + // Bracket A only: the two synchronous shipped API calls through the peer's narrow boundary, timed on + // CACurrentMediaTime by the diagnostic itself; driven from the measurement ticks below. + let log = ScreenSharingRtcEventLogDiagnostic( + window: try .init(beginSeconds: window.beginSeconds, durationSeconds: window.durationSeconds), path: path, + boundaries: .init( + clock: { ScreenSharingRtcEventLogDiagnostic.defaultClock() }, + start: { path, maxSizeBytes in receiver.startRtcEventLog(path: path, maxSizeBytes: maxSizeBytes) }, + stop: { receiver.stopRtcEventLog() })) // Bool: true only when the native stop API ran + rtcEventLog = log + receiverMetrics.label( + "rtcEventLog", + "window begin \(log.window.beginSeconds) s duration \(log.window.durationSeconds) s, cap \(log.maxSizeBytes) B, bracket A (CA before/after start and stop)" + ) + } + if options.checkRecovery { + receiver.simulateDecoderLoss( + afterFrames: 120, droppingRecoveryKeyframeFrom: options.dropRecoveryKeyframe ? sender : nil, + idlingCaptureFrom: options.idleOnDecoderReset ? sender : nil) + } + receiver.onConnectionChanged = { print("Receiver: \($0)") } + if options.headless { + receiverMetrics.label("presentationTelemetry", "disabled: headless media check") + } else { + let view = try ScreenSharingMetalView( + mailbox: receiver.mailbox, metrics: receiverMetrics, renderOnArrival: options.renderOnArrival, + maximumDrawableCount: options.drawableCount, unsyncedPresentation: options.unsyncedPresentation, + offMainPreparation: options.renderOffMain, deliveryAudit: deliveryAudit) + receiverMetrics.label("renderScheduling", options.renderOnArrival ? "frame arrival" : "display link") + if let renderFPS = options.renderFPS { view.preferredFramesPerSecond = renderFPS } + receiverMetrics.label("renderRequestedFPS", String(view.preferredFramesPerSecond)) + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 960, height: 540), + styleMask: [.titled, .closable, .resizable, .miniaturizable], backing: .buffered, defer: false) + window.isReleasedWhenClosed = false + window.delegate = self + window.title = "Codevisor · Native Screen Sharing Probe" + if options.keepFront { + window.level = .floating + window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + } + window.contentView = view + window.center() + if let displayID = options.viewerDisplayID { + guard + let screen = NSScreen.screens.first(where: { + ($0.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value == displayID + }) + else { throw ScreenSharingError.invalid("The requested viewer display is no longer attached.") } + window.setFrameOrigin( + NSPoint( + x: screen.visibleFrame.midX - window.frame.width / 2, + y: screen.visibleFrame.midY - window.frame.height / 2)) + } + window.makeKeyAndOrderFront(nil) + NSApplication.shared.activate() + if let report = options.reportURL { + let ready: [String: Any] = [ + "windowID": window.windowNumber, + "displayID": (window.screen?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber) + ?? 0, + "widthPoints": window.frame.width, "heightPoints": window.frame.height, + "contentWidthPoints": view.bounds.width, "contentHeightPoints": view.bounds.height, + "contentTopPoints": window.frame.height - view.bounds.height, + "backingScaleFactor": window.backingScaleFactor, + ] + try JSONSerialization.data(withJSONObject: ready, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".viewer-ready.json"), options: .atomic) + } + self.window = window + metalView = view + if options.metalDisplayLink { + displayLink = try ProbeMetalDisplayLink( + view: view, framesPerSecond: options.renderFPS ?? options.configuration.framesPerSecond) + } + } + } + switch options.mode { + case .loopback: + guard let sender, let receiver else { throw ScreenSharingError.invalid("Missing loopback peers.") } + // Exercise the product flow: the viewer initiates negotiation. + try await sender.accept(receiver.makeDescription(offer: true)) + try await receiver.accept(sender.makeDescription(offer: false)) + case .send: + guard let sender, let offer = options.offerURL, let answer = options.answerURL else { return } + guard !FileManager.default.fileExists(atPath: answer.path) else { + throw ScreenSharingError.invalid("Answer file already exists; use fresh signaling paths for each session.") + } + try writeDescription(await sender.makeDescription(offer: true), to: offer) + print("Offer written to \(offer.path). Exchange files over a trusted channel. Waiting for \(answer.path).") + try await waitUntil(seconds: 120) { FileManager.default.fileExists(atPath: answer.path) } + try await sender.accept(readDescription(answer)) + case .receive: + guard let receiver, let offer = options.offerURL, let answer = options.answerURL else { return } + try await receiver.accept(readDescription(offer)) + try writeDescription(await receiver.makeDescription(offer: false), to: answer) + print("Answer written to \(answer.path). Send it to the sender over a trusted channel.") + } + try await waitUntil(seconds: 120) { [self] in + (sender == nil || senderMetrics.snapshot().labels["connection"] == "connected") + && (receiver == nil || receiverMetrics.snapshot().labels["connection"] == "connected") + } + if let sender { + if options.captureOwnedWindow { + // One owned window, shown without activation; capture starts only + // after its first draw completed and it is confirmed visible. + let capture = ScreenSharingCapture( + queueDepth: options.captureQueueDepth, pixelFormat: options.capturePixelFormat, + copySurface: options.copyCaptureSurface, captureIntervalFPS: options.captureIntervalFPS) + self.capture = capture + let workload = try ProbeOwnedWorkloadWindow( + configuration: options.configuration, recordDrawTimes: options.recordOwnedWorkloadTimes + ) { + try await capture.stop() + } + ownedWorkload = workload + // show → first draw (cancellation-safe gate) → visible → capture start; + // any failure hides only this window once and surfaces the error. + var beforeStart: [String: Any] = [:] + var ready = try await workload.start(timeoutSeconds: 10) { + // Immediately before OUR capture.start call; the framework's own + // SCStream.start moment is not observable without shared changes. + beforeStart = workload.observation("immediately before capture.start (probe call, not the framework start)") + try await capture.start( + ownedWindowID: workload.windowID, configuration: options.configuration, + sender: sender.frameSender, metrics: senderMetrics) + } + ready["observationBeforeCaptureStart"] = beforeStart // also retained in workload.snapshots + var afterStart = workload.observation("after capture.start returned") + let startLabels = senderMetrics.snapshot().labels + afterStart["sckSelectionAndStartLabels"] = Dictionary( + uniqueKeysWithValues: startLabels.filter { $0.key.hasPrefix("capture") }.map { ($0.key, $0.value) }) + ready["observationAfterCaptureStart"] = afterStart + if let report = options.reportURL { + try JSONSerialization.data(withJSONObject: ready, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".workload-ready.json"), options: .atomic) + } + senderMetrics.label("ownedWorkloadWindowID", String(workload.windowID)) + senderMetrics.label("ownedWorkloadReadyAtNs", String(workload.lifecycle.timestampsNs[.ready] ?? 0)) + print( + "Owned workload window \(workload.windowID) is being captured through ScreenCaptureKit; no viewer rendering." + ) + } else if let pickedFilter { + let capture = ScreenSharingCapture( + queueDepth: options.captureQueueDepth, pixelFormat: options.capturePixelFormat, + copySurface: options.copyCaptureSurface, captureIntervalFPS: options.captureIntervalFPS) + self.capture = capture + try await capture.start( + pickedFilter: pickedFilter, configuration: options.configuration, + sender: sender.frameSender, metrics: senderMetrics) + } else if let display = options.displayID { + let capture = ScreenSharingCapture( + queueDepth: options.captureQueueDepth, pixelFormat: options.capturePixelFormat, + copySurface: options.copyCaptureSurface, captureIntervalFPS: options.captureIntervalFPS) + self.capture = capture + try await capture.start( + displayID: display, configuration: options.configuration, + sender: sender.frameSender, metrics: senderMetrics) + } else { + let source = try SyntheticSource( + configuration: options.configuration, + sender: sender.frameSender, metrics: senderMetrics, pixelFormat: options.syntheticPixelFormat, + desktopPattern: options.desktopPattern, gapMilliseconds: options.syntheticGapMs) + synthetic = source + source.start() + } + } + if let sender, let receiver { + try await ProbeControlCheck(host: sender.control, viewer: receiver.control).run() + senderMetrics.label("controlChannel", "ordered input and release verified") + print("Control channel: eight input events and release acknowledged; no OS input posted.") + try await ProbeClipboardCheck(host: sender.clipboard, viewer: receiver.clipboard).run() + senderMetrics.label("clipboardChannel", "bidirectional chunked Unicode transfer verified") + print("Clipboard channel: bidirectional Unicode transfer verified; no system clipboard accessed.") + } + if options.checkQuality { try await checkQuality() } + print( + "Measuring \(options.duration) seconds of \(options.configuration.width)×\(options.configuration.height) video." + ) + let started = ScreenSharingMetrics.nowNs + mediaStartedNs = started + let startedAtSeconds = CACurrentMediaTime() + for metrics in [senderMetrics, receiverMetrics] { metrics.label("mediaStartUptimeNs", String(started)) } + // The audit window is relative to this media start on the audit's own clock (CACurrentMediaTime). + deliveryAudit?.start(originNs: Int64(startedAtSeconds * 1_000_000_000)) + // The event-log diagnostics' first tick is the media-start boundary itself (a begin of 0 starts here). + rtcEventLog?.tick(measuredSeconds: 0) + senderRtcEventLog?.tick(measuredSeconds: 0) + // Relay checks require this exact capability; absent counters are never zero. + receiverMetrics.label("idleAuditInstrumentation", "demand-attribution-1") + if let report = options.reportURL { + // Both clocks are read back to back so a relay can verify that the + // Core Animation and Dispatch uptime clocks agree within this process. + let ready: [String: Any] = [ + "startedAtSeconds": startedAtSeconds, "startedAtUptimeNs": started, + "startedAtMediaTimeNs": Int64(CACurrentMediaTime() * 1_000_000_000), + // Backward compatible: existing keys unchanged; the meaning is now explicit. + "meaning": + "SETUP readiness: peers connected and the source (synthetic, display, picked or owned-window stream) started; not a first captured, encoded or decoded frame", + ] + try JSONSerialization.data(withJSONObject: ready, options: [.sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".media-ready.json"), options: .atomic) + } + var transport = ProbeTransportTimeline() + transport.append( + elapsed: 0, sender: await sender?.statistics() ?? [:], receiver: await receiver?.statistics() ?? [:]) + var timeline = ProbeTimeline() + timeline.append( + elapsed: 0, sender: senderMetrics.snapshot(), receiver: receiverMetrics.snapshot(), + drops: receiver?.mailbox.droppedFrames ?? 0) + try timeline.writeProgress(to: options.reportURL) + var measured = 0.0 + var sourcePausedAt: Double? + var workloadPausedAt: Double? + // Owned-window mode: first-observed delivery over measurement ticks (bounded, one entry per metric), + // kept on the runner so failure evidence retains what was already seen. + if options.captureOwnedWindow { + firstObservation = ScreenSharingFirstObservation(names: [ + "anyCallbackIncludingInvalidOrMissingStatus", "completeStatusCallbacks", "acceptedCapturedFrames", + "decodedFrames", + ]) + measurementStartedNs = started + } + try recordFirstObservation(elapsedSeconds: measured) // initial counter observation at tick 0 + var idleCheckpointTaken = false + var settlingCheckpointAtSeconds: Double? + let settlingCounters = [ + ( + senderMetrics, + ["capturedFrames", "encodedFrames", "refreshFrames", "videoRefreshRequestsReceived", "sourceIdleEvaluations"] + ), + (receiverMetrics, ["videoRefreshRequestsSent"]), + ] + while measured < options.duration { + // The event-log diagnostic borrows these ticks: the sleep is shortened to its next requested boundary so + // the start/stop calls happen at the first tick at or after it (requested and actual are both recorded). + let boundaries = [rtcEventLog?.nextBoundarySeconds(), senderRtcEventLog?.nextBoundarySeconds()] + .compactMap { $0 } + try await Task.sleep( + for: .seconds( + ScreenSharingRtcEventLogDiagnostic.sleepSeconds( + measured: measured, remaining: options.duration - measured, nextBoundary: boundaries.min()))) + measured = Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000 + rtcEventLog?.tick(measuredSeconds: measured) + senderRtcEventLog?.tick(measuredSeconds: measured) + if let pauseAfter = options.pauseSourceAfterSeconds, sourcePausedAt == nil, measured >= pauseAfter { + if options.finalBurstSignal, let report = options.reportURL { + // Announce that the final changing content follows, keep producing it + // briefly, then stop. A loss relay coordinates its drop window on this. + let signal: [String: Any] = [ + "signalAtSeconds": CACurrentMediaTime(), "measuredSeconds": measured, "continueMilliseconds": 250, + ] + try JSONSerialization.data(withJSONObject: signal, options: [.sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".final-burst.json"), options: .atomic) + try await Task.sleep(for: .milliseconds(250)) + } + synthetic?.stop() + senderMetrics.label("sourcePausedAtMediaTimeSeconds", String(CACurrentMediaTime())) + measured = Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000 + sourcePausedAt = measured + if let report = options.reportURL { + // The source has drained (stop waits for its queue); a relay may end + // a coordinated drop window shortly after observing this file. + let paused: [String: Any] = [ + "pausedAtSeconds": CACurrentMediaTime(), "pausedAtUptimeNs": ScreenSharingMetrics.nowNs, + "measuredSeconds": measured, + "latestCapturedTimestampNs": senderMetrics.snapshot().labels["latestCapturedTimestampNs"] ?? "none", + ] + try JSONSerialization.data(withJSONObject: paused, options: [.sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".source-paused.json"), options: .atomic) + } + senderMetrics.label("sourceIdleExperiment", "synthetic input stopped; observe automatic idle output") + senderMetrics.label("sourcePausedAtSeconds", String(measured)) + senderMetrics.increment("sourcePauseEvents") + } + if let pauseAfter = options.pauseWorkloadAfterSeconds, workloadPausedAt == nil, measured >= pauseAfter, + let workload = ownedWorkload + { + // Only the animation stops; the window and its last frame remain on + // screen and remain the captured content. The stream is not touched. + var record = try workload.pauseAnimation() + measured = Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000 + workloadPausedAt = measured + sourcePausedAt = measured // idle-checkpoint accounting only; not a pass criterion here + let snapshot = senderMetrics.snapshot() + record["measuredSeconds"] = measured + record["capturedFramesAtPause"] = snapshot.counters["capturedFrames", default: 0] + record["captureCallbacksCompleteAtPause"] = snapshot.counters["captureCallbacksComplete", default: 0] + record["latestCapturedTimestampNs"] = snapshot.labels["latestCapturedTimestampNs"] ?? "none" + if let report = options.reportURL { + try JSONSerialization.data(withJSONObject: record, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".workload-paused.json"), options: .atomic) + } + senderMetrics.label("sourceIdleExperiment", "owned workload animation paused; static window remains captured") + senderMetrics.label("sourcePausedAtSeconds", String(measured)) + senderMetrics.label("workloadPausedAtSeconds", String(measured)) + senderMetrics.label("workloadPausedAtMediaTimeSeconds", String(CACurrentMediaTime())) + senderMetrics.increment("workloadPauseEvents") + } + if options.idleOnDecoderReset, sourcePausedAt == nil, + receiverMetrics.snapshot().counters["recoveryKeyframesReceived", default: 0] == 1 + { + sourcePausedAt = measured + senderMetrics.label("idleRecoveryObservedAtSeconds", String(measured)) + } + if let sourcePausedAt, !idleCheckpointTaken, measured >= sourcePausedAt + 2 { + let snapshot = senderMetrics.snapshot() + senderMetrics.increment("capturedFramesAtIdleCheckpoint", by: snapshot.counters["capturedFrames", default: 0]) + senderMetrics.increment("encodedFramesAtIdleCheckpoint", by: snapshot.counters["encodedFrames", default: 0]) + senderMetrics.increment("refreshFramesAtIdleCheckpoint", by: snapshot.counters["refreshFrames", default: 0]) + senderMetrics.increment( + "videoRefreshRequestsReceivedAtIdleCheckpoint", + by: snapshot.counters["videoRefreshRequestsReceived", default: 0]) + receiverMetrics.increment( + "videoRefreshRequestsSentAtIdleCheckpoint", + by: receiverMetrics.snapshot().counters["videoRefreshRequestsSent", default: 0]) + senderMetrics.label("idleCheckpointAtSeconds", String(measured)) + idleCheckpointTaken = true + } + // Recovery may legitimately continue after the idle checkpoint (a + // blackout); the final three seconds must then be genuinely quiet. + // Every headless peer records the window so a separate-process + // experiment can verify both ends. + if options.headless, settlingCheckpointAtSeconds == nil, measured >= options.duration - 3 { + for (metrics, names) in settlingCounters { + let snapshot = metrics.snapshot() + for name in names { + metrics.increment(name + "AtSettlingCheckpoint", by: snapshot.counters[name, default: 0]) + } + metrics.label("settlingCheckpointAtSeconds", String(measured)) + } + settlingCheckpointAtSeconds = measured + } + try recordFirstObservation(elapsedSeconds: measured) + transport.append( + elapsed: measured, sender: await sender?.statistics() ?? [:], receiver: await receiver?.statistics() ?? [:]) + if measured - (timeline.samples.last?.elapsedSeconds ?? 0) >= Double(options.sampleIntervalSeconds) + || measured >= options.duration + { + timeline.append( + elapsed: measured, sender: senderMetrics.snapshot(), receiver: receiverMetrics.snapshot(), + drops: receiver?.mailbox.droppedFrames ?? 0) + try timeline.writeProgress(to: options.reportURL) + print("Measurement progress: \(Int(measured)) / \(Int(options.duration)) seconds") + } + } + // Normal completion: each event log is stopped exactly once here, before any peer teardown. + rtcEventLog?.finish(measuredSeconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000) + senderRtcEventLog?.finish(measuredSeconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000) + synthetic?.stop() + if let workload = ownedWorkload { + // Boundary order: stream stop through the session (success or failure + // preserved; the lifecycle advances only on success), then the window + // closes during stop(); both are in the report and the after-close file. + let stopped = await workload.stopCapture() + guard stopped else { + throw ScreenSharingError.unavailable( + "Capture stop failed: \(senderMetrics.snapshot().labels["captureStopFailed"] ?? "unknown error").") + } + } else { + try await capture?.stop() + } + if let workload = ownedWorkload { + measured = Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000 + try recordFirstObservation(elapsedSeconds: measured) // final post-stop observation + publishFirstObservationLabels() + let snapshot = senderMetrics.snapshot() + var stopped: [String: Any] = [ + "captureStopRequestedAtNs": snapshot.labels["captureStopRequestedAtNs"] ?? "none", + "captureStopCompletedAtNs": snapshot.labels["captureStopCompletedAtNs"] ?? "none", + "capturedFrames": snapshot.counters["capturedFrames", default: 0], + "lifecycle": workload.lifecycleRecord, + "observationAfterStop": workload.observation("after capture stop completed"), + "firstObservations": firstObservation?.summary ?? "measurement never began", + ] + for name in ScreenSharingCaptureCallbackAccounting.Status.allCases.map(\.counterName) + + [ + ScreenSharingCaptureCallbackAccounting.otherStatusCounter, + ScreenSharingCaptureCallbackAccounting.invalidSampleCounter, + ScreenSharingCaptureCallbackAccounting.missingStatusCounter, + ScreenSharingCaptureCallbackAccounting.missingImageCounter, + ] + { + stopped[name] = snapshot.counters[name, default: 0] + } + if let report = options.reportURL { + try JSONSerialization.data(withJSONObject: stopped, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".capture-stopped.json"), options: .atomic) + if let times = workload.workloadTimesReport(mediaMeasuredSeconds: measured) { + try JSONSerialization.data(withJSONObject: times, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".workload-times.json"), options: .atomic) + } + } + } + let elapsed = Double(ScreenSharingMetrics.nowNs - started) / 1_000_000_000 + let senderStats = await sender?.statistics() ?? [:] + let receiverStats = await receiver?.statistics() ?? [:] + if options.headless, let latest = receiver?.mailbox.take() { + receiverMetrics.label("latestReceivedRtpTimestamp", String(latest.rtpTimestamp)) + } + let sent = senderMetrics.snapshot() + let received = receiverMetrics.snapshot() + // After the checkpoint the host may encode only cache refreshes, each + // answering a viewer request received after the checkpoint; a viewer in + // this process may request one only after a verified delivery shortfall + // (transport loss before idle). Without loss nothing continues. + func sinceCheckpoint(_ snapshot: ScreenSharingMetrics.Snapshot, _ name: String) -> Int { + snapshot.counters[name, default: 0] - snapshot.counters[name + "AtIdleCheckpoint", default: 0] + } + func settled(_ snapshot: ScreenSharingMetrics.Snapshot, _ name: String) -> Bool { + snapshot.counters[name, default: 0] == snapshot.counters[name + "AtSettlingCheckpoint", default: 0] + } + // A late checkpoint (delayed loop) must not pass with an unobserved window. + let settlingObserved = settlingCheckpointAtSeconds.map { elapsed - $0 >= 2 } ?? false + let idleObservationPassed = + (options.pauseSourceAfterSeconds == nil && !options.idleOnDecoderReset) + || (idleCheckpointTaken && sinceCheckpoint(sent, "capturedFrames") == 0 + && sinceCheckpoint(sent, "encodedFrames") <= sinceCheckpoint(sent, "refreshFrames") + && sinceCheckpoint(sent, "refreshFrames") <= sinceCheckpoint(sent, "videoRefreshRequestsReceived") + && (received.counters["sourceIdleRefreshRequests", default: 0] > 0 + || sinceCheckpoint(received, "videoRefreshRequestsSent") == 0) + && settlingObserved + && settlingCounters.allSatisfy { metrics, names in + let snapshot = metrics === senderMetrics ? sent : received + return names.allSatisfy { settled(snapshot, $0) } + }) + let idleResetPassed = + !options.idleOnDecoderReset || sent.counters["captureDeliveryStoppedAtDecoderReset", default: 0] == 1 + let encoderRetryPassed = + !options.dropRecoveryKeyframe + || (sent.counters["injectedEncoderKeyframeDrops", default: 0] == 1 + && sent.counters["encoderRetriedKeyframes", default: 0] >= 1 + // The idle retry intentionally creates another request. Retention + // without a new request is covered by the encoder state tests. + && sent.counters["encoderForcedKeyframesSubmitted", default: 0] >= 3) + let recoveryPassed = + !options.checkRecovery + || (received.counters["injectedDecoderResets", default: 0] == 1 + && received.counters["recoveryKeyframesReceived", default: 0] == 1 + && received.counters["decodedFrames", default: 0] >= (options.idleOnDecoderReset ? 120 : 126) + && (options.headless || received.counters["presentedFrames", default: 0] >= 126) + && (received.timings["decoderResetToRecoveryKeyframe"]?.maximumMs ?? .infinity) < 2000) + // Owned-window mode passes on delivered media, no errors, a recorded + // pause when one was requested, and a completed capture stop. Idle + // behaviour after the pause is recorded, not judged, in this mode. + let ownedWindowPassed = + !options.captureOwnedWindow + || (sent.labels["captureStartedAtNs"] != nil && sent.labels["captureStopCompletedAtNs"] != nil + && sent.counters["captureCallbacksComplete", default: 0] > 0 + && (options.pauseWorkloadAfterSeconds == nil || workloadPausedAt != nil)) + let passed = + ownedWindowPassed + && (sender == nil || sent.counters["encodedFrames", default: 0] > 0) + && (receiver == nil + || received.counters[options.headless ? "decodedFrames" : "presentedFrames", default: 0] > 0) + && sent.labels["captureError"] == nil && sent.labels["encoderError"] == nil + && received.labels["decoderError"] == nil + && sent.counters["encodeErrors", default: 0] == 0 && received.counters["decodeErrors", default: 0] == 0 + && sent.counters["syntheticDrawErrors", default: 0] == 0 + && sent.counters["syntheticConversionErrors", default: 0] == 0 + && received.counters["renderErrors", default: 0] == 0 + && recoveryPassed + && encoderRetryPassed + && idleObservationPassed + && idleResetPassed + let report = ProbeReport( + mode: options.mode, configuration: options.configuration, + source: options.mode == .receive + ? "remote" + : options.captureOwnedWindow + ? "ScreenCaptureKit owned window" + : (options.displayID != nil || options.capturePicker) ? "ScreenCaptureKit" : "synthetic", + startedAtSeconds: startedAtSeconds, elapsedSeconds: elapsed, + passed: passed, renderingEnabled: receiver != nil && !options.headless, sender: sent, + receiver: received, + senderRTC: senderStats, receiverRTC: receiverStats, + rendererMailboxDrops: receiver?.mailbox.droppedFrames ?? 0, + presentedFramesPerSecond: Double( + received.counters["presentedFrames", default: 0] + - (timeline.samples.first?.receiver.counters["presentedFrames"] ?? 0)) / elapsed, + timeline: timeline.samples, transportTimeline: transport.samples, + receiverDeliveryAudit: deliveryAudit?.snapshot(), receiverRtcEventLog: rtcEventLog?.record, + senderRtcEventLog: senderRtcEventLog?.record) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + let data = try encoder.encode(report) + if let url = options.reportURL { try data.write(to: url, options: .atomic) } + print(String(decoding: data, as: UTF8.self)) + let cacheHeldBeforeClose = sender?.frameSender.isHoldingCachedFrame + await stop() + // Close check in three observations: immediately after close (tasks + // cancelled, not yet awaited), after each peer's owned work completed, + // and after a bounded 500 ms observation window. It fails the run when + // the cache or mailbox is still held or a tracked counter moved during + // that window. Only owned main-actor tasks are awaited; WebRTC and + // VideoToolbox threads are outside this boundary. + func observation() throws -> [String: Any] { + let sent = senderMetrics.snapshot() + let received = receiverMetrics.snapshot() + return [ + "cacheHeld": sender?.frameSender.isHoldingCachedFrame ?? false, + "mailboxHeld": receiver?.mailbox.isHolding ?? false, + "encoderPendingAtStop": sent.labels["encoderPendingAtStop"] ?? "not observed", + "sender": try JSONSerialization.jsonObject(with: encoder.encode(sent)), + "receiver": try JSONSerialization.jsonObject(with: encoder.encode(received)), + ] + } + let immediate = try observation() + let senderTasks = await sender?.awaitClosed() + let receiverTasks = await receiver?.awaitClosed() + let drained = try observation() + try await Task.sleep(for: .milliseconds(500)) + let settled = try observation() + let tracked = [ + "capturedFrames", "encodedFrames", "refreshFrames", "sourceIdleEvaluations", "decodedFrames", + "videoRefreshRequestsSent", "videoRefreshRequestsReceived", + ] + func counters(_ observation: [String: Any]) -> [String: Int] { + var result: [String: Int] = [:] + for side in ["sender", "receiver"] { + let values = (observation[side] as? [String: Any])?["counters"] as? [String: Int] ?? [:] + for name in tracked { result[side + "." + name] = values[name] ?? 0 } + } + return result + } + let movedCounters = counters(drained).filter { counters(settled)[$0.key] != $0.value }.keys.sorted() + // An owned-window run must also have closed its window in order after a + // successful stream stop; an abandoned lifecycle never passes this check. + let closeCheckPassed = + movedCounters.isEmpty && settled["cacheHeld"] as? Bool == false && settled["mailboxHeld"] as? Bool == false + && (ownedWorkload?.session.completedInOrder ?? true) + let closed: [String: Any] = [ + "applicable": true, "passed": closeCheckPassed, + "cacheHeldBeforeClose": cacheHeldBeforeClose ?? false, + "ownedTasksAwaited": ["sender": senderTasks ?? -1, "receiver": receiverTasks ?? -1], + "immediate": immediate, "afterOwnedWorkDrained": drained, "afterObservationWindow": settled, + "observationWindowMilliseconds": 500, "trackedCounters": tracked, "movedCounters": movedCounters, + "rtcEventLogSidecar": rtcEventLogSidecarOutcomes[.receiver] ?? "not requested", + "senderRtcEventLogSidecar": rtcEventLogSidecarOutcomes[.sender] ?? "not requested", + "quiescentOverObservationWindow": closeCheckPassed, + "scope": "owned main-actor tasks awaited; WebRTC and VideoToolbox threads not covered", + "ownedWorkload": ownedWorkload?.lifecycleRecord ?? "not used", + ] + if let url = options.reportURL { + try JSONSerialization.data(withJSONObject: closed, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: url.path + ".after-close.json"), options: .atomic) + } + guard passed else { throw ScreenSharingError.unavailable("Media did not pass the probe. See the metrics above.") } + guard closeCheckPassed else { + throw ScreenSharingError.unavailable( + "Close check failed: cache/mailbox retained, counters moved (\(movedCounters.joined(separator: ", ")))" + + " or the owned window did not close in order.") + } + } + + func deliveryValues() -> [String: Int] { + let sent = senderMetrics.snapshot().counters + let received = receiverMetrics.snapshot().counters + return [ + // Every callback bucket once; captureSamplesWithoutImage is a sub-count of complete and is NOT added. + "anyCallbackIncludingInvalidOrMissingStatus": ScreenSharingCaptureCallbackAccounting.callbackTotal( + counters: sent), + "completeStatusCallbacks": sent["captureCallbacksComplete", default: 0], + "acceptedCapturedFrames": sent["capturedFrames", default: 0], + "decodedFrames": received["decodedFrames", default: 0], + ] + } + + /// One tick of first-observed delivery; writes the first-callback / + /// first-frame sidecars the first time their metric becomes non-zero. + func recordFirstObservation(elapsedSeconds: Double) throws { + guard firstObservation != nil else { return } + let values = deliveryValues() + let newly = firstObservation!.record(elapsedSeconds: elapsedSeconds, values: values) + guard let report = options.reportURL else { return } + for name in newly { + let sidecar: String? = + name == "anyCallbackIncludingInvalidOrMissingStatus" + ? "first-callback" : name == "acceptedCapturedFrames" ? "first-frame" : nil + guard let sidecar else { continue } + let record: [String: Any] = [ + "metric": name, "observedAtSeconds": elapsedSeconds, "tick": (firstObservation?.ticks ?? 1) - 1, + "values": values, + "resolution": ScreenSharingFirstObservation.resolution, + "definition": sidecar == "first-frame" + ? "first measurement tick with capturedFrames > 0 (frames ACCEPTED by the sender), not merely a complete-status callback" + : "first measurement tick with any SCK callback counter > 0, including invalid/missing-status samples", + ] + try JSONSerialization.data(withJSONObject: record, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: report.path + ".\(sidecar).json"), options: .atomic) + } + } + + func publishFirstObservationLabels() { + guard let firstObservation else { return } + for (name, metric) in firstObservation.summary { + senderMetrics.label( + "firstObserved." + name, metric["firstObservedAtSeconds"] ?? ScreenSharingFirstObservation.neverObserved) + } + senderMetrics.label("firstObservedResolution", ScreenSharingFirstObservation.resolution) + } + + /// The final event-log record: the diagnostic's own when it was constructed, otherwise (options requested a + /// log but the probe failed earlier) an explicit not-initialized record; nil when no log was requested. + func rtcEventLogRecord( + role: ScreenSharingRtcEventLogDiagnostic.Role, reason: String + ) -> ScreenSharingRtcEventLogDiagnostic.Record? { + let log = role == .receiver ? rtcEventLog : senderRtcEventLog + if let log { return log.record } + let window = role == .receiver ? options.rtcEventLogWindow : options.senderRtcEventLogWindow + let path = role == .receiver ? options.rtcEventLogPath : options.senderRtcEventLogPath + guard let window, let path else { return nil } + return .notInitialized( + beginSeconds: window.beginSeconds, durationSeconds: window.durationSeconds, path: path, + maxSizeBytes: ScreenSharingRtcEventLogDiagnostic.fixedMaxSizeBytes, reason: reason, role: role) + } + + /// Shared finalization (normal completion, failure, window close): after the actual stop, publish the record + /// atomically as REPORT.rtc-event-log.json so the brackets and close reason survive even without a report. + /// A write error is recorded and printed, never a silent success. Idempotent per process. + func publishRtcEventLogSidecar(role: ScreenSharingRtcEventLogDiagnostic.Role) { + guard rtcEventLogSidecarOutcomes[role] == nil, let url = options.reportURL, + let record = rtcEventLogRecord(role: role, reason: "probe stopped before the peer and the log were constructed") + else { return } + let suffix = role == .receiver ? ".rtc-event-log.json" : ".sender-rtc-event-log.json" + let sidecar = URL(fileURLWithPath: url.path + suffix) + do { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + try encoder.encode(record).write(to: sidecar, options: .atomic) + rtcEventLogSidecarOutcomes[role] = "written \(sidecar.path)" + } catch { + rtcEventLogSidecarOutcomes[role] = "write failed: \(error.localizedDescription)" + FileHandle.standardError.write( + Data("RTC event log sidecar (\(role.rawValue)): \(rtcEventLogSidecarOutcomes[role] ?? "")\n".utf8)) + } + } + + /// The role's final event-log record as a JSON object for the failure record; nil when no log was requested. + func failureRecordJSON(role: ScreenSharingRtcEventLogDiagnostic.Role) -> Any? { + let reason = "probe failed before the peer and the log were constructed" + guard let record = rtcEventLogRecord(role: role, reason: reason) else { return nil } + return (try? JSONEncoder().encode(record)).flatMap { try? JSONSerialization.jsonObject(with: $0) } + } + + /// Durable failure evidence: written after cleanup on any failed run so the + /// original error, a stop failure, the owned window's real lifecycle, every + /// snapshot already taken (readiness / before-start / after-start / pause / + /// stop) and the first-observed delivery state survive even when the normal + /// report was never written. A final observation is taken only if + /// measurement had begun; elapsed times are never invented. + func writeFailureRecord(_ error: any Error) { + guard let url = options.reportURL else { return } + if firstObservation != nil, let startedNs = measurementStartedNs { + try? recordFirstObservation(elapsedSeconds: Double(ScreenSharingMetrics.nowNs - startedNs) / 1_000_000_000) + publishFirstObservationLabels() + } + let sent = senderMetrics.snapshot() + let record: [String: Any] = [ + "failure": error.localizedDescription, "failedAtUptimeNs": ScreenSharingMetrics.nowNs, + "reportWritten": FileManager.default.fileExists(atPath: url.path), + "captureStopFailed": sent.labels["captureStopFailed"] ?? "none", + "captureStopCompletedAtNs": sent.labels["captureStopCompletedAtNs"] ?? "none", + "captureError": sent.labels["captureError"] ?? "none", + "ownedWorkload": ownedWorkload?.lifecycleRecord ?? "not used", + "ownedWorkloadCleanup": sent.labels["ownedWorkloadCleanup"] ?? "none", + "firstObservations": firstObservation?.summary ?? "measurement never began", + "firstObservationTicks": firstObservation?.ticks ?? 0, + "snapshots": ownedWorkload?.snapshots ?? "not used", + "observationAtFailure": ownedWorkload?.observation("failure record (after cleanup)") ?? "not used", + // The event-log diagnostic record survives a failure (stop() already closed it before peer teardown and + // published the sidecar); a request that never reached construction is stated as not initialized. + "rtcEventLog": failureRecordJSON(role: .receiver) ?? "not requested", + "rtcEventLogSidecar": rtcEventLogSidecarOutcomes[.receiver] ?? "not requested", + "senderRtcEventLog": failureRecordJSON(role: .sender) ?? "not requested", + "senderRtcEventLogSidecar": rtcEventLogSidecarOutcomes[.sender] ?? "not requested", + ] + try? JSONSerialization.data(withJSONObject: record, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: url.path + ".failure.json"), options: .atomic) + // Draw timestamps already recorded survive a failure too (same schema, marked partial). + if let workload = ownedWorkload, + var times = workload.workloadTimesReport( + mediaMeasuredSeconds: measurementStartedNs.map { Double(ScreenSharingMetrics.nowNs - $0) / 1_000_000_000 } + ?? 0) + { + times["partial"] = true + try? JSONSerialization.data(withJSONObject: times, options: [.prettyPrinted, .sortedKeys]) + .write(to: URL(fileURLWithPath: url.path + ".workload-times.json"), options: .atomic) + } + } + + func windowShouldClose(_ sender: NSWindow) -> Bool { + Task { @MainActor in + await stop(); exit(EXIT_SUCCESS) + } + return false + } + + func stop() async { + synthetic?.stop() + synthetic = nil + if ownedWorkload == nil { try? await capture?.stop() } + if let workload = ownedWorkload { + // Every exit path (normal, error, cancel) goes through one boundary: + // the stop is credited only when the session recorded a successful + // stream stop; the window closes in order or is abandoned; only this + // window is ever hidden, at most once. A stop never attempted (error + // before the normal end) is attempted here and its result preserved. + if workload.lifecycle.state == .capturing || workload.lifecycle.state == .workloadPaused { + await workload.stopCapture() + } + let outcome = workload.cleanUp() + senderMetrics.label("ownedWorkloadCleanup", String(describing: outcome)) + senderMetrics.label("ownedWorkloadFinalState", workload.lifecycle.state.rawValue) + if let closed = workload.lifecycle.timestampsNs[.close] { + senderMetrics.label("ownedWorkloadClosedAtNs", String(closed)) + } + if let abandoned = workload.lifecycle.timestampsNs[.abandon] { + senderMetrics.label("ownedWorkloadAbandonedAtNs", String(abandoned)) + senderMetrics.label("ownedWorkloadAbandonedFrom", workload.lifecycle.abandonedFrom?.rawValue ?? "unknown") + } + } + capture = nil + picker?.stop() + picker = nil + // The standalone display link is invalidated before the renderer's + // terminal stop so no supplied-drawable draw can follow it. + displayLink?.stop() + displayLink = nil + metalView?.stop() + // Failure, cancellation or early close: stop a started event log exactly once before the peer closes; + // after a normal completion this is already final and does nothing. + let measuredAtStop = mediaStartedNs.map { Double(ScreenSharingMetrics.nowNs - $0) / 1_000_000_000 } + rtcEventLog?.closeEarly(measuredSeconds: measuredAtStop, reason: "runner stop before peer teardown") + senderRtcEventLog?.closeEarly(measuredSeconds: measuredAtStop, reason: "runner stop before peer teardown") + publishRtcEventLogSidecar(role: .receiver) + publishRtcEventLogSidecar(role: .sender) + sender?.close() + receiver?.close() + encoderLogger?.stop() + encoderLogger = nil + window?.orderOut(nil) + } + + private func checkQuality() async throws { + guard let sender else { return } + let original = options.configuration + for (scale, fps) in [(1.0, 30), (0.75, 30), (0.5, 20), (1.0, 60)] { + let video = try ScreenSharingVideoConfiguration( + width: max(64, Int(Double(original.width) * scale) / 2 * 2), + height: max(64, Int(Double(original.height) * scale) / 2 * 2), + framesPerSecond: min(fps, original.framesPerSecond), bitrate: original.bitrate) + synthetic?.stop() + sender.updateVideoConfiguration(video) + if let capture { + try await capture.update(configuration: video) + } else { + synthetic = try SyntheticSource( + configuration: video, sender: sender.frameSender, metrics: senderMetrics, + pixelFormat: options.syntheticPixelFormat, desktopPattern: options.desktopPattern) + synthetic?.start() + } + let frames = receiverMetrics.snapshot().counters["presentedFrames", default: 0] + try await waitUntil(seconds: 15) { [self] in + let snapshot = receiverMetrics.snapshot() + return snapshot.labels["videoSize"] == "\(video.width) × \(video.height)" + && snapshot.counters["presentedFrames", default: 0] >= frames + 6 + } + print("Format transition: \(video.width) × \(video.height) at requested \(video.framesPerSecond) fps") + } + senderMetrics.label("qualityTransitions", "full30, 75%30, 50%20, full60 verified") + } + + private func readDescription(_ url: URL) throws -> ScreenSharingDescription { + let file = try FileHandle(forReadingFrom: url) + defer { try? file.close() } + let data = try file.read(upToCount: 256 * 1024 + 1) ?? Data() + guard data.count <= 256 * 1024 else { throw ScreenSharingError.invalid("Signaling file is too large.") } + return try JSONDecoder().decode(ScreenSharingDescription.self, from: data) + } + + private func writeDescription(_ description: ScreenSharingDescription, to url: URL) throws { + // Publish a complete private file atomically, refusing to replace a prior + // session. The receiver can never observe a partially written description. + let data = try JSONEncoder().encode(description) + let temporary = url.deletingLastPathComponent().appendingPathComponent(".screen-sharing-" + UUID().uuidString) + let descriptor = open(temporary.path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR) + guard descriptor >= 0 else { throw ScreenSharingError.unavailable("Cannot create private signaling file.") } + defer { unlink(temporary.path) } + let file = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true) + try file.write(contentsOf: data) + try file.close() + guard link(temporary.path, url.path) == 0 else { + throw ScreenSharingError.unavailable("Cannot publish signaling file; use fresh paths for each session.") + } + } + + /// Real diagnostic workflow, not a test synchronization primitive. Bounded + /// polling allows a human to transfer the signaling file without a server. + private func waitUntil(seconds: Double, condition: () -> Bool) async throws { + let deadline = ContinuousClock.now + .seconds(seconds) + while !condition() { + guard ContinuousClock.now < deadline else { throw ScreenSharingError.unavailable("Probe setup timed out.") } + try await Task.sleep(for: .milliseconds(100)) + } + } + } + + private struct ProbeReport: Encodable { + let mode: ProbeOptions.Mode + let configuration: ScreenSharingVideoConfiguration + let source: String + let startedAtSeconds: Double + let elapsedSeconds: Double + let passed: Bool + let renderingEnabled: Bool + let sender: ScreenSharingMetrics.Snapshot + let receiver: ScreenSharingMetrics.Snapshot + let senderRTC: [String: String] + let receiverRTC: [String: String] + let rendererMailboxDrops: Int + let presentedFramesPerSecond: Double + let timeline: [ProbeTimeline.Sample] + let transportTimeline: [ProbeTransportTimeline.Sample] + /// Receiver-only diagnostic audit snapshot taken at report time (before close); absent when not requested. + let receiverDeliveryAudit: ScreenSharingFrameDeliveryAudit.Snapshot? + /// Receiver-only diagnostic RTC event-log lifecycle record (final at report time); absent when not requested. + let receiverRtcEventLog: ScreenSharingRtcEventLogDiagnostic.Record? + /// Sender-only diagnostic RTC event-log lifecycle record on the sending peer; absent when not requested. + let senderRtcEventLog: ScreenSharingRtcEventLogDiagnostic.Record? + } +#else + @main + struct ScreenSharingProbe { + static func main() { print("The diagnostic executable requires macOS; the media library also supports iOS.") } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/Probe/SyntheticSource.swift b/packages/swift/CodevisorScreenSharing/Probe/SyntheticSource.swift new file mode 100644 index 000000000..3469f0b77 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Probe/SyntheticSource.swift @@ -0,0 +1,181 @@ +#if os(macOS) + import CodevisorScreenSharing + import CoreGraphics + import CoreVideo + import Foundation + import VideoToolbox + + /// Diagnostic workload only. A bounded pool feeds a serial capture timer; + /// each surface becomes immutable before it is handed to WebRTC. + final class SyntheticSource: @unchecked Sendable { + private let queue = DispatchQueue(label: "codevisor.screen-sharing.synthetic", qos: .userInteractive) + private let pool: CVPixelBufferPool + private let convertedPool: CVPixelBufferPool? + private let transfer: VTPixelTransferSession? + private let configuration: ScreenSharingVideoConfiguration + private let sender: ScreenSharingFrameSender + private let metrics: ScreenSharingMetrics + private var timer: DispatchSourceTimer? + private var sequence = 0 + private var firstFrameNs: Int64? + private let desktopPainter: ProbeDesktopPainter? + private let burstNs: Int64 = 250_000_000 + private let gapNs: Int64? + + init( + configuration: ScreenSharingVideoConfiguration, sender: ScreenSharingFrameSender, + metrics: ScreenSharingMetrics, pixelFormat: ProbeOptions.SyntheticPixelFormat = .bgra, + desktopPattern: Bool = false, gapMilliseconds: Int? = nil + ) throws { + self.configuration = configuration + self.sender = sender + self.metrics = metrics + gapNs = gapMilliseconds.map { Int64($0) * 1_000_000 } + metrics.label( + "syntheticCadence", + gapMilliseconds.map { "250 ms bursts with \($0) ms gaps" } ?? "continuous at the configured rate") + desktopPainter = + desktopPattern + ? try ProbeDesktopPainter.make( + width: configuration.width, height: configuration.height, fps: configuration.framesPerSecond) + : nil + metrics.label("syntheticWorkload", desktopPattern ? "desktop text and motion" : "color bars and motion") + var pool: CVPixelBufferPool? + let attributes: [CFString: Any] = [ + kCVPixelBufferWidthKey: configuration.width, + kCVPixelBufferHeightKey: configuration.height, + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, + kCVPixelBufferIOSurfacePropertiesKey: [:], + ] + let status = CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary, &pool) + guard status == kCVReturnSuccess, let pool else { throw ScreenSharingError.codec("Create source pool", status) } + self.pool = pool + if pixelFormat == .nv12 { + var convertedPool: CVPixelBufferPool? + var convertedAttributes = attributes + convertedAttributes[kCVPixelBufferPixelFormatTypeKey] = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + let poolStatus = CVPixelBufferPoolCreate(nil, nil, convertedAttributes as CFDictionary, &convertedPool) + guard poolStatus == kCVReturnSuccess, let convertedPool else { + throw ScreenSharingError.codec("Create NV12 source pool", poolStatus) + } + var transfer: VTPixelTransferSession? + let transferStatus = VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer) + guard transferStatus == noErr, let transfer else { + throw ScreenSharingError.codec("Create NV12 source transfer", transferStatus) + } + self.convertedPool = convertedPool + self.transfer = transfer + } else { + convertedPool = nil + transfer = nil + } + metrics.label("syntheticInputFormat", pixelFormat.rawValue) + metrics.label("syntheticConversion", pixelFormat == .nv12 ? "BGRA to NV12 before WebRTC" : "none before WebRTC") + } + + deinit { if let transfer { VTPixelTransferSessionInvalidate(transfer) } } + + func start() { + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule( + deadline: .now(), repeating: .nanoseconds(1_000_000_000 / configuration.framesPerSecond), + leeway: .milliseconds(1)) + timer.setEventHandler { [weak self] in self?.draw() } + self.timer = timer + timer.resume() + } + + func stop() { + timer?.cancel() + timer = nil + queue.sync {} + } + + private func draw() { + autoreleasepool { + let started = ScreenSharingMetrics.nowNs + // Bursty input: a healthy desktop that changes in short bursts must not + // trigger refreshes; the timer keeps running so cadence stays exact. + if let gapNs { + let origin = firstFrameNs ?? started + if firstFrameNs == nil { firstFrameNs = started } + if (started - origin) % (burstNs + gapNs) >= burstNs { return } + } + var pixel: CVPixelBuffer? + let status = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( + nil, pool, + [kCVPixelBufferPoolAllocationThresholdKey: 6] as CFDictionary, &pixel) + guard status == kCVReturnSuccess, let pixel else { metrics.increment("syntheticPoolDrops"); return } + CVPixelBufferLockBaseAddress(pixel, []) + guard + let context = CGContext( + data: CVPixelBufferGetBaseAddress(pixel), width: configuration.width, height: configuration.height, + bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixel), + space: CGColorSpace(name: CGColorSpace.itur_709)!, + bitmapInfo: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue) + else { CVPixelBufferUnlockBaseAddress(pixel, []); metrics.increment("syntheticDrawErrors"); return } + if let desktopPainter { + if firstFrameNs == nil { firstFrameNs = started } + let frameCode = Int( + Double(started - (firstFrameNs ?? started)) / 1_000_000_000 * Double(configuration.framesPerSecond)) + desktopPainter.draw( + in: context, + bounds: CGRect(x: 0, y: 0, width: configuration.width, height: configuration.height), + sequence: frameCode, responses: 0) + } else { + let width = CGFloat(configuration.width) + let height = CGFloat(configuration.height) + context.setFillColor(CGColor(gray: 0.08, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + for index in 0..<8 { + context.setFillColor( + CGColor( + red: index & 1 == 0 ? 0.85 : 0.1, + green: index & 2 == 0 ? 0.85 : 0.1, blue: index & 4 == 0 ? 0.85 : 0.1, alpha: 1)) + context.fill(CGRect(x: CGFloat(index) * width / 8, y: height / 2, width: width / 8, height: height / 2)) + } + context.setFillColor(CGColor(gray: 0.95, alpha: 1)) + for row in 0..<16 { + let offset = CGFloat((sequence * 5 + row * 43) % configuration.width) + context.fill(CGRect(x: offset, y: CGFloat(row) * height / 36, width: width / 5, height: 2)) + } + context.setFillColor(CGColor(red: 1, green: 0.35, blue: 0.1, alpha: 1)) + context.fill(CGRect(x: CGFloat(sequence * 7 % configuration.width), y: height / 8, width: 48, height: 96)) + } + sequence += 1 + CVPixelBufferUnlockBaseAddress(pixel, []) + metrics.observe("syntheticDraw", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + guard let input = prepare(pixel) else { return } + metrics.label("syntheticInputPixelFormatCode", String(CVPixelBufferGetPixelFormatType(input))) + metrics.observe( + "syntheticFramePreparation", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + sender.push(ScreenSharingVideoFrame(pixelBuffer: input, timestampNs: started)) + } + } + + private func prepare(_ pixel: CVPixelBuffer) -> CVPixelBuffer? { + guard let convertedPool, let transfer else { return pixel } + let started = ScreenSharingMetrics.nowNs + var converted: CVPixelBuffer? + let poolStatus = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( + nil, convertedPool, [kCVPixelBufferPoolAllocationThresholdKey: 6] as CFDictionary, &converted) + guard poolStatus == kCVReturnSuccess, let converted else { + metrics.increment("syntheticPoolDrops"); return nil + } + CVBufferSetAttachment( + converted, kCVImageBufferColorPrimariesKey, kCVImageBufferColorPrimaries_ITU_R_709_2, .shouldPropagate) + CVBufferSetAttachment( + converted, kCVImageBufferTransferFunctionKey, kCVImageBufferTransferFunction_ITU_R_709_2, .shouldPropagate) + CVBufferSetAttachment( + converted, kCVImageBufferYCbCrMatrixKey, kCVImageBufferYCbCrMatrix_ITU_R_709_2, .shouldPropagate) + let status = VTPixelTransferSessionTransferImage(transfer, from: pixel, to: converted) + metrics.observe("syntheticConversion", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + guard status == noErr else { + metrics.increment("syntheticConversionErrors") + metrics.label("syntheticConversionError", String(status)) + return nil + } + return converted + } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/README.md b/packages/swift/CodevisorScreenSharing/README.md new file mode 100644 index 000000000..a7de0838e --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/README.md @@ -0,0 +1,294 @@ +# Native Screen Sharing + +Implementation of [the Screen Sharing plan](../../../docs/plans/native-screen-sharing.md): a native media library, a diagnostic app, and a native macOS workspace pane with authenticated signaling and explicit remote control. No HTTP plugin server or webview participates in video delivery. + +The working path is ScreenCaptureKit (or a synthetic motion source) → native WebRTC video source → custom VideoToolbox H.264 encoder → encrypted WebRTC media → custom VideoToolbox decoder → CVPixelBuffer-backed Metal textures. The custom codec factories are used by WebRTC itself; frames are not encoded twice. + +## Native workspace pane + +Run `bun run dev:macos`. In a workspace hosted by a Mac, open New Tab → Screen Sharing, select a display and Connect. The host must run this version of the native app and allow Screen Recording. The native viewer supports Fit and Actual Size. iOS preserves the pane and shows an unavailable state until its viewer milestone. + +The viewer sends a receive-only offer through the existing machine-authenticated, relay-aware client to `POST /v1/screen-sharing`. The server validates the active workspace and native pane, then forwards the request over Codevisor’s authenticated Unix bridge to the app’s native host service. SDP stays transient; only the stable display UUID and fit preference enter pane metadata. Synced size preferences update the existing native surface; a display change from another client requires a fresh Connect. Websites are rejected even on loopback. + +One viewer owns the host at a time. Capture begins after its peer connects. The host shows a menu-bar Sharing indicator with Stop Sharing. An eight-second heartbeat renews a 25-second lease; loss of the viewer stops capture. Sleep, session deactivation, display changes and SCK errors also stop it. A connection without video ends after three heartbeat intervals, and decoder failures end viewing on the next heartbeat. Hiding or closing the viewer tears down media; returning to a previously connected tab renegotiates. A newer negotiation waits for the old stop request, and generation checks reject late frames and signaling responses. Reopening the app requires an explicit Connect. + +The current host scales to fit 1920×1080, preserving aspect ratio, at up to 60 fps / 12 Mbps. Direct ICE is the default. Optional host-configured STUN/TURN supports other routes; the existing cloud channel still carries signaling only. See connectivity configuration below. + +## Native control + +After video connects, choose **Control** to send mouse, keyboard and app shortcuts to the selected Mac display. The host also needs Accessibility permission (called Device Control and Data Access in macOS 27). Control is explicit for every new connection. **Control–Option–Escape** always returns to viewing; Stop Control, clicking outside the video, losing keyboard/window focus, hiding the pane, disconnecting and the host’s Stop Sharing action also release it. The host indicator changes to “Controlled”. + +A fresh host-issued control lease binds input to the current authenticated WebRTC peer and selected display. One-second heartbeats renew a separate three-second control deadline, checked every 250 ms. Expiry, permission loss, malformed messages and channel closure revoke the lease and release tracked keys/buttons. Delayed grants are released after viewer cancellation; input from an expired lease cannot revive control. No control lease, input or key content enters pane metadata or logs. + +The native ordered data channel carries key/button transitions, scroll deltas and motion. A 16 ms viewer tick coalesces waiting motion; transitions flush it first to preserve ordering. Each button event contains its own normalized coordinates. The send queue admits at most 16 KiB, and the receive queue at most 256 messages / 64 KiB; congestion or invalid input closes control instead of accumulating actions. This first implementation deliberately uses one reliable ordered channel. A separate unreliable motion channel needs button-generation coordination before it can replace this ordering contract. + +The native surface maps through Fit/Actual Size letterboxing, scroll-document position and backing-pixel scale. The host maps normalized coordinates into Quartz display bounds, including negative origins. It uses public CGEvents from a private event source, with no automation delays or app activation. Tagged injected events are excluded from capture by a viewer on the same Mac. Fractional trackpad deltas accumulate before conversion to integer pixels. + +Physical Mac key codes use the host’s keyboard layout and input method. OS-reserved shortcuts may remain local. Basic physical typing, modifiers, repeat, mouse buttons, dragging and scrolling are implemented; automatic clipboard synchronization, a local cursor overlay and dedicated Unicode composition UI remain follow-up work. Explicit plain-text clipboard transfer is available from the toolbar. The protocol has a bounded, separate text-insertion message exercised by the diagnostic sink. Rotated/external-display mapping, non-US layouts, IME behavior and real interactions on a second Mac still need hardware validation. + +## Clipboard, cursor and connection details + +The clipboard menu explicitly sends or retrieves plain text, up to 64 KiB of UTF-8. A separate ordered SCTP channel carries 2 KiB chunks with stop-and-wait acknowledgments; it cannot fill the keyboard/mouse channel. Transfers expire after ten seconds, reject invalid or incomplete UTF-8, and discard partial data on cancellation. A download cannot replace a newer local copy made while it was in flight. There is no automatic polling, file transfer, rich text, or clipboard persistence in signaling, logs or the workspace database. + +ScreenCaptureKit includes the host cursor in video. AppKit cursor rectangles suppress the viewer cursor only inside the active control surface and restore the local arrow on release/exit. Apple's current SDK deprecates `NSCursor.currentSystem`, so this implementation uses the supported captured-cursor fallback rather than a cursor-shape stream. Same-Mac loopback cannot establish independent host/viewer cursor fidelity. + +Connection Details shows the selected direct/relay route, transport, decoded resolution, interval presentation rate, receive bitrate, round-trip time and decode p95. It omits SDP, addresses and credentials. Statistics run separately from the control heartbeat. + +## Connectivity and recovery + +The native host reads optional deployment configuration from its launch environment: + +| Variable | Meaning | +| --- | --- | +| `CODEVISOR_SCREEN_SHARING_STUN_URLS` | Comma-separated `stun:` or `stuns:` URLs | +| `CODEVISOR_SCREEN_SHARING_TURN_URLS` | Comma-separated `turn:` or `turns:` URLs, optionally `?transport=udp` or `?transport=tcp` | +| `CODEVISOR_SCREEN_SHARING_TURN_SECRET` | Host-only TURN REST shared secret; never shipped in the viewer | +| `CODEVISOR_SCREEN_SHARING_RELAY_ONLY` | `1` forces a TURN route; default `0` allows direct connections | + +With no configuration, neither peer contacts a third-party ICE server. The host mints five-minute, viewer-specific TURN REST credentials using HMAC-SHA1. The viewer requests fresh credentials immediately before negotiation through the authenticated, uncached machine API. This matches [coturn's TURN REST mechanism](https://github.com/coturn/coturn/blob/master/README.turnserver#turn-rest-api). A deployed TURN service and forced-relay/UDP-restricted network verification remain required; no relay infrastructure or fixed client secret is included here. + +After an established connection drops, the viewer releases input, discards the old native peer and its frame mailbox, and renegotiates using `restart`. The host accepts replacement only for the same still-live viewer lease and display. Host stop, sleep, display change, cancellation and lease expiry prevent renewal. Replacement uses fresh ICE/DTLS and codec state rather than restarting ICE inside the old peer; control must be requested again. Three automatic replacements are allowed before explicit reconnect is required. This avoids replaying queued input or decoder state across routes. + +The host polls WebRTC's selected candidate-pair bandwidth estimate once per second. Two seconds of insufficient bandwidth lowers one quality step: full resolution at 60 fps → full resolution at 30 fps → 75% dimensions at 30 fps → 50% dimensions at 20 fps. Fifteen seconds of sustained headroom restores one step. Capture and the native source change together; old-size raw frames are discarded before encoding. Coordinates remain normalized to the original display. Missing estimates leave the current quality unchanged. This policy has live format-transition checks; tuning against real congested networks is still pending. + +## Latency and codec investigation + +The [tuftlord report](../../../docs/measurements/native-screen-sharing-tuftlord-2026-09-10/README.md) records hardware codec experiments and native WebRTC runs in both directions between an M1 Pro and M4 Max. Hardware HEVC 4:4:4 works through the public VideoToolbox APIs, including direct BGRA input when selecting the Main444 value returned by the supported-property query. The low-latency rate-control flag changed tested NV24 input to 4:2:0, so output format must be verified. + +The probe now samples receive-buffer/processing/codec/send-delay interval means from WebRTC cumulative counters and records receiver-local callback-to-submission/presentation timings. It never subtracts clocks on different Macs. Only positive Metal `presentedTime` values count as new on-screen frames; skipped drawables and redraws are excluded. A historical callback-counting error is corrected in the earlier report. + +`--render-on-arrival` enables an experimental bounded renderer driven by mailbox availability. It improved frame rate and median renderer delay in the two-Mac runs but worsened p95, so the native app retains its display-link default. `--keep-front` makes diagnostic visibility explicit. These results do not establish input-to-photon latency or Apple High Performance parity. + +`--standard-rate-control` compares standard VideoToolbox rate control with the default low-latency rate controller in a sending or loopback probe. The default codec is hardware H.264 without frame reordering and with at most two admitted frames. Explicit probe flags can select HEVC or change this admission bound. The native app keeps the low-latency default. The [timing investigation](../../../docs/measurements/native-screen-sharing-timing-2026-09-11/README.md) records local comparisons and the blocked tuftlord desktop follow-up. No additional encoder-internal frame-delay cap is claimed: the tested encoder rejects `MaxFrameDelayCount`. + +Cadence diagnostics now distinguish capture/source timestamps, arrival at the native source, encoder input, the duration of the VideoToolbox submit call, encoded output, decoder input and the receiver callback. Capture delivery age uses SCK's timestamp and the Core Media host clock; it is absent from synthetic runs. These bounded per-stage measurements supplement the existing RTC buffer/send-delay statistics. The probe retains only numeric counts from WebRTC's periodic encoder drop log. Decoder input occurs after WebRTC buffering and is not a packet-arrival timestamp. + +```sh +bun run screen-sharing:probe --instance codecs --check-codecs --report tmp/screen-sharing/codecs.json +bun run screen-sharing:probe --instance codecs --check-codecs --codec-case hevc-advertised444-bgra --width 3840 --height 2160 --bitrate 75000000 --report tmp/screen-sharing/codecs-4k.json +``` + +`--check-codecs` is a standalone VT experiment with matching source/decoded images in `.images`; it does not enable HEVC in the product. The expanded measurements passed 1,788 Swift tests, strict media lint/format checks, and both native app builds. + +The follow-up [real-desktop runs on tuftlord](../../../docs/measurements/native-screen-sharing-tuftlord-2026-09-10/README.md#real-desktop-native-media-runs) verify ScreenCaptureKit capture at 1080p and 4K and expose more WebRTC receive buffering than the synthetic workload. They also record the shared Apple High Performance workload, a CoreText correction for the diagnostic header, and the limits of the exploratory renderer comparison. Probe packaging now isolates identities by worktree and instance; the approved remote capture copy remains usable alongside the separately updated workload app. + +The [September 11 Apple High Performance comparison](../../../docs/measurements/native-screen-sharing-apple-2026-09-11/README.md) uses the same recording method and viewer content size for both 4K streams. Two retained recordings per viewer observed 53.9 updates/s for Apple and 38.9–39.7 for Codevisor; a local recording control observed 57.4 at a requested 60 fps. These are sampled updates, not physical FPS or input-to-photon latency. Codevisor's approximately 26 ms encoder p95 and 115–145 ms receive-buffer weighted means identify further profiling work. The shared network and Apple's concurrent virtual-display stream limit the comparison. + +## Run the diagnostic app locally + +From the repository root, on macOS 26 or later with Xcode and Bun installed: + +```sh +bun run screen-sharing:probe --loopback --duration 10 --report tmp/screen-sharing/loopback.json +bun run screen-sharing:probe --capabilities +bun run screen-sharing:probe --list-displays +bun run screen-sharing:probe --loopback --display 1 --duration 10 +``` + +Additional checks: + +```sh +# Live resolution/frame-rate changes; add --display ID to exercise SCK. +bun run screen-sharing:probe --instance quality --loopback --check-quality --duration 10 +# Resource timeline: also writes .progress.json every 30 seconds. +bun run screen-sharing:probe --instance endurance --loopback --duration 1800 --report tmp/screen-sharing/endurance.json +# Actual native host API; use an existing, disconnected Screen Sharing test pane. +bun run screen-sharing:probe --instance host --check-host http://127.0.0.1:59372 --workspace WORKSPACE_UUID --pane PANE_UUID +``` + +Loopback checks both ordered input/release and bidirectional chunked Unicode clipboard delivery with recording/in-memory sinks; they do not post OS input or access system clipboards. `--check-host` verifies video, rejects replacement from another viewer, replaces the authorized media peer, and verifies that a stopped lease cannot restart. It requires the running development app. Supply an optional API token only through `CODEVISOR_SCREEN_SHARING_PROBE_TOKEN`; remote host checks require HTTPS. + +Use the display ID returned by `--list-displays`. Display capture requires Screen Recording permission. The probe fails with an explanation when preflight permission is absent. Synthetic mode needs no screen capture permission. Local Network permission and firewall settings must allow the probe to connect. + +Defaults are 1920×1080, 60 fps, and a 12 Mbps bitrate ceiling. Options include `--width`, `--height`, `--fps`, `--bitrate`, `--duration` (1–3600 seconds), and `--report`. Use `--help` for details. The default synthetic source animates lines and a rectangle below color bars; it is a lightweight transport workload, not a text-fidelity or worst-case bitrate benchmark. + +Synthetic input defaults to **BGRA**, while ScreenCaptureKit supplies video-range **NV12**. Use `--synthetic-format nv12` to convert the same generated image with a bounded NV12 pool before passing it to WebRTC. Reports record the actual source pixel-format code, conversion time and total source preparation time; conversion work is moved outside the encoder measurement, not eliminated. `--synthetic-format bgra` reproduces the original input path. These options apply only to generated sender/loopback traffic and also work with quality transitions. Codec-only comparisons use `h264-low-delay-nv12` and `h264-realtime-nv12`, with conversion reported separately and included in the decoded-frame fidelity result. + +The [pixel-format investigation](../../../docs/measurements/native-screen-sharing-pixel-format-2026-09-11/README.md) records two trials for each input/controller combination. Standard mode improves the lightweight NV12 media workload, while low-latency mode wins the isolated NV12 colored-text codec test. Real-desktop measurements remain necessary before selecting a different product default. + +The build command creates `tmp/screen-sharing/ScreenSharingProbe.app`, embeds WebRTC and its resource bundle, fixes the framework search path, and signs the diagnostic app ad hoc. `--build-only` builds without launching. The resulting binary is at `ScreenSharingProbe.app/Contents/MacOS/screen-sharing-probe`; invoke it directly to rerun without rebuilding. Close the window to stop a viewer early. + +Use `--instance NAME` to build a separately named diagnostic app without replacing another running probe. The bundle identifier is stable for each worktree/instance pair and differs between pairs, so Launch Services and permission entries cannot confuse distinct diagnostic instances. Grant capture permission separately for each identity. Previously built probes retain their old identity until rebuilt. + +The helper builds for the current Mac's architecture. Build on the other Mac too, or copy the app to another Mac of the same architecture. The bundle is a local diagnostic artifact, not a notarized release. The default and file-signaling modes have no dependency on the Codevisor server or a running development app. + +### Capture permission and a shared desktop workload + +Finish building before granting Screen Recording permission. Ad-hoc signing can tie the grant to the executable's code hash, so rebuilding can invalidate it. Keep the final copy at a stable location such as `~/Applications/CodevisorDiagnostics/ScreenSharingProbe.app`, launch that copy through Launch Services, and grant it access under **Privacy & Security → Screen & System Audio Recording**. If an enabled entry still refers to a previous build's signature, remove that diagnostic entry with the minus button before adding the final copy; adding over the existing entry may retain its old code requirement. + +```sh +open -n -W ~/Applications/CodevisorDiagnostics/ScreenSharingProbe.app \ + --args --request-screen-recording --duration 120 +open -n -W --stdout /tmp/screen-displays.log --stderr /tmp/screen-displays.err \ + ~/Applications/CodevisorDiagnostics/ScreenSharingProbe.app --args --list-displays +``` + +The permission command requests access without starting capture and keeps the app alive for up to `--duration` seconds. Relaunch after macOS requests it, then inspect the display-list output and error log; `open` exiting successfully does not establish that the probe passed. When invoking a remote probe over SSH, use this Launch Services form for capture too: directly executing its binary can attribute the request to the SSH daemon instead of the diagnostic app. + +`--show-workload` provides a common native desktop target for Codevisor and Apple's viewer. It displays scrolling colored text, motion, an 18-bit time-driven frame code and a response counter that changes when the window receives a click or a non-repeating key press. It closes itself after the requested duration, needs no capture permission and injects no input: + +```sh +open -n -W ~/Applications/CodevisorDiagnostics/ScreenSharingProbe.app \ + --args --show-workload --width 3840 --height 2160 --fps 60 --duration 60 \ + --report /tmp/screen-workload.json +``` + +Run the workload on the sending Mac and view its desktop with the transport under test. Match the raster to the display's backing pixels, and use the same capture resolution, refresh rate and viewer scale in both runs. A Retina display at 1920×1080 points can have a 3840×2160 backing raster. The report records raster dimensions, window points, backing scale, AppKit draw calls and received-event times. Draw calls and the frame code are not physical presentation measurements; input-to-photon timing still needs an external recording or a calibrated method with stated error. + +## Optimized pipeline experiments + +Use `--release` on the build wrapper for performance runs. The report records the build configuration. `--codec hevc` and `--codec hevc444` select the codec on both peers; Main444 sending requires `--standard-rate-control` and BGRA source input. HEVC profile, bit depth and chroma are checked from the encoded format, and Main444 decoding must produce full chroma. + +`--capture-picker` selects a display through macOS's scoped sharing picker. It is separate from the persistent Screen Recording permission used by `--display`. New ad-hoc probe identities can use that picker without replacing the previously approved diagnostic app. + +`--desktop-pattern` draws the same text, scrolling pattern and time-driven marker as `--show-workload`, directly into the synthetic source. It isolates capture from the rest of the pipeline; it does not reproduce WindowServer or ScreenCaptureKit scheduling. Inspect source preparation and arrival timings before treating a run as a steady 60 fps source. `--user-initiated-activity` holds a Foundation user-initiated activity only during the finite media probe and allows idle system sleep; reports label whether it was requested. + +`--prioritize-encoding-speed` requests the public speed-over-quality property with standard rate control. `--complete-each-frame` drains VideoToolbox through each submitted timestamp. Both are diagnostic options, and the synchronous-drain experiment reduced throughput in the current Main444 trial. + +`--keyframe-interval N` changes periodic keyframes from the default two nominal seconds to 1–60. It sets a frame count using the configured FPS, so lower actual encoding throughput can lengthen the wall-clock interval. Requested keyframes still use the existing force-keyframe path. Reports split encoded frame/byte counters into keyframes and delta frames. Longer intervals need recovery validation before product use. + +The [pipeline report](../../../docs/measurements/native-screen-sharing-pipeline-2026-09-11/README.md) records frame-arrival rendering, receive-buffer, capture-format, encoder-admission, codec and rate-control investigations. These remain explicit diagnostic options; successful HEVC transport is not a 4K60 or Apple-parity claim. `--help` lists experiment bounds and incompatible combinations. Keep both peers' codec selections equal. + +## Two-Mac LAN workflow + +Build the probe on each Mac. Choose fresh signaling filenames for every session. On the sending Mac: + +```sh +ScreenSharingProbe.app/Contents/MacOS/screen-sharing-probe \ + --send --offer /tmp/screen-offer.json --answer /tmp/screen-answer.json \ + --duration 30 --report /tmp/screen-sender-report.json +``` + +Add `--display ID` for a real display. Otherwise the sender generates synthetic motion. Transfer the offer to the receiving Mac over a trusted channel such as authenticated SSH, then run there: + +```sh +ScreenSharingProbe.app/Contents/MacOS/screen-sharing-probe \ + --receive --offer /tmp/screen-offer.json --answer /tmp/screen-answer.json \ + --duration 30 --report /tmp/screen-receiver-report.json +``` + +Return the answer to the sender. When transferring into a filename the probe is waiting for, copy to a temporary filename and rename after the transfer completes. The sender waits up to 120 seconds for the answer. Both peers begin their measurement interval after connecting. Setup failures, absent media, and codec/render errors produce a nonzero exit code. A passing report confirms media delivery; it does not enforce performance targets. + +The probe writes complete signaling files atomically with mode `0600` and refuses to overwrite an existing session. SDP includes connection credentials and network addresses: exchange it privately and remove it afterward. JSON reports exclude SDP, IP addresses, and credentials. File exchange is a diagnostic replacement for the authenticated Codevisor signaling channel; it is not an identity or authorization system. + +The file-signaling probe configures only host ICE candidates. There are no STUN/TURN services, cloud relay credentials, automatic discovery, or VNC interoperability. Check the nominated candidate's protocol in the reports to verify the actual route. Two peers in one process, or two processes on one Mac, do not establish LAN performance. + +## Ownership and limits + +- Capture callbacks feed WebRTC on the capture queue, without per-frame main-actor tasks. SCK uses a queue depth of three; the synthetic pool allows six outstanding surfaces. +- Hardware acceleration is required at VideoToolbox session creation. Software fallback is disabled. On this Mac, the low-latency encoder omits the optional hardware-status query; reports distinguish that from a confirmed query. The decoder's query returns true. +- H.264 constrained baseline, no B-frame reordering, 8-bit NV12/4:2:0, BT.709 SDR. Input dimensions are even and bounded to 3840×2160; the target frame rate is at most 60. These are supported configuration limits, not measured performance guarantees. +- WebRTC handles packetization, congestion feedback, pacing and keyframe requests. Bitrate updates reach VT; the native host adapts capture resolution/frame rate as described above. HEVC negotiation and 4:4:4 are future work. +- The encoder admits at most two pending raw frames. VT owns each output callback's immutable metadata. The decoder runs synchronously on WebRTC's decoder queue. These bounds describe Codevisor-owned stages; WebRTC also has internal queues and jitter buffering. +- Rendering has a one-frame mailbox, one command buffer in flight, and one retained last frame for resize redraw. Slow consumers replace stale waiting frames. CVPixelBuffers and their Metal textures survive until the GPU completes its reads. +- The peer and capture owner must call `close()` / `stop()`. Closing the peer stops capture-frame admission and remote rendering before disconnecting. The native viewer implements session generations, reconnect and hidden-pane teardown; the standalone probe owns its own lifecycle. +- The captured cursor is included in video. There is no cursor overlay, audio, automatic clipboard synchronization, virtual display, or saved screenshot. Native control and its separate lease are described above. + +## Measurements and validation + +The 30-minute baseline loopback completed at **57.83 presentation callbacks/second** (the original counter included skipped drawables; see the correction in the report), with no codec/render errors and a 293.52–298.88 MiB physical footprint after warm-up. See the [endurance report and raw timeline](../../../docs/measurements/native-screen-sharing-2026-09-10/README.md) for the workload, chart and limits. Adaptive-quality and recovery additions were checked in separate live probes. + +The historical `presented fps` values in this section used that same callback counter and must not be interpreted as verified on-screen rates. The corrected probe records `unpresentedDrawables` separately. + +Initial measurements on an Apple M4 Max, macOS 27.0 beta (26A428), debug build: + +| Workload | Duration | Presented fps | Encode p95 | Decode p95 | GPU submission-to-completion p95 | +| --- | ---: | ---: | ---: | ---: | ---: | +| Synthetic 1920×1080, requested 60 fps, same-process WebRTC loopback | 40.0 s | 59.1 | 6.78 ms | 1.11 ms | 0.72 ms | +| ScreenCaptureKit 1512×982 desktop, same-process WebRTC loopback | 10.0 s | 49.0 | 11.32 ms | 2.03 ms | 1.66 ms | + +Viewer-initiated negotiation, matching the native tab, passed an eight-second synthetic 1080p check at 57.0 presented fps (encode p95 8.70 ms, decode p95 1.56 ms). The sender uses `addTrack` so a remote offer can associate it with the offered video m-line; an explicit unassociated transceiver had connected ICE without sending video. Loopback now exercises that negotiation direction. + +The final callback/cleanup revision also passed a 10-second 1080p60 check at 56.2 presented fps (encode p95 8.24 ms, decode p95 1.45 ms). A 3840×2160 synthetic run at requested 60 fps / 24 Mbps passed at 45.2 presented fps, with encode p95 40.27 ms and decode p95 3.01 ms. That 4K run recorded 108 encoder admission drops: the encoder stage is the first bottleneck to investigate, and 4K60 has not been achieved. + +These runs passed without codec errors. The synthetic run replaced 17 waiting renderer frames and had one encoder admission drop; VT reported four dropped frames during startup. SCK emits frames as content changes, and the desktop workload was not controlled. Builds were also running on the machine. These runs establish functionality, not a controlled performance comparison. Reports are kept locally under `tmp/screen-sharing/`. + +Timing samples use a bounded rolling history of 1,800 observations. The fps calculation includes startup within the measured interval. GPU completion is measured separately from drawable presentation; neither is input-to-photon latency. The Simulator cannot report drawable presentation and only exposes GPU completion. No unrelated machine clocks are subtracted. + +A separate sender/receiver process smoke test on this Mac passed the file-signaling flow, hardware codec path, and native rendering; both signaling files had mode `0600`. The native viewer was also inspected visually. The integrated development app successfully opened Screen Sharing from New Tab, enumerated the Retina display, and displayed live SCK video through the authenticated HTTP/Unix bridge. Switching away released the host immediately, returning renegotiated successfully, and explicit disconnect released it again. Actual Size rendered at one video pixel per backing pixel; a registry update to Fit resized the existing surface without reconnecting. Closing a connected tab released the host, and reopening restored the display and size preferences in the ready state. These are same-Mac functional checks, not two-Mac measurements. `--capabilities` reports advertised hardware H.264 and HEVC encode/decode on this M4 Max. The subsequent [tuftlord investigation](../../../docs/measurements/native-screen-sharing-tuftlord-2026-09-10/README.md) verifies hardware HEVC 4:4:4 round trips on M1 Pro and M4 Max, including 4K. The subsequent pipeline experiments add explicit HEVC/Main444 transport in the probe; product codec selection and performance promotion remain pending. + +The control revision passed an eight-second synthetic 1080p loopback at 56.5 presented fps, including a bidirectional ordered data-channel check of eight input events and release while video ran. The probe’s input sink records these events without posting OS input. In the development app, missing Accessibility permission denied control; enabling it allowed explicit control, a physical key down/up crossed the native path exactly once, the local release shortcut worked, and switching tabs ended control. Returning resumed viewing without reacquiring control. In a split workspace, requesting control activated the viewer pane, and clicking the neighboring pane released input while video stayed visible. These checks do not establish two-Mac interaction fidelity or input-to-photon latency. + +The clipboard/quality revision passed bidirectional large Unicode transfers on its independent data channel and all four live format transitions. After restoring 1080p60, the synthetic probe presented 59.8 fps over ten seconds; the SCK probe presented 56.3 fps. Both reported zero codec/render errors. The native-host API integration check also passed initial video, rejection of a different viewer without stopping video, replacement of the current peer, and rejection of renewal after host stop. No OS input or system clipboard changes were used by these probes. + +Validation commands: + +```sh +bun run swift:test +bun run --cwd apps/server test:coverage +bun run --cwd packages/automation test:coverage +swiftlint lint --strict --quiet packages/swift/CodevisorScreenSharing +bun run build:macos +bun run build:ios +``` + +Deterministic tests cover NAL framing and malformed input, configuration validation, newest-frame delivery, bounded metrics, pane metadata, synced preferences, control-message bounds, pointer geometry, fractional scroll accumulation, Quartz event construction, held-input release, late control grants, lease ownership/expiry, late signaling callbacks, hidden-tab cleanup and heartbeat failure. Focused HTTP and Unix-socket tests cover authenticated signaling, body limits, stale panes and socket cleanup. Neither permission prompts nor hardware/network benchmarks run in those tests. The full Swift package suite passes 1,788 tests. The server suite passes 449 tests and the automation suite passes 144 tests; both retain 100% measured coverage. Both existing native apps build, and the media module was separately typechecked against iOS device and Simulator SDKs. Device playback remains untested. + +Milestone 0 now includes exploratory two-Mac runs and a matched-scale 4K recording comparison with Apple's High Performance mode. It remains open for isolated wired-LAN measurements, a matched 1080p Apple comparison, quantitative text fidelity and input-to-photon measurements, the controlled resource/latency comparison, and the dependency build/notice work below. Those are gates before claiming Apple-like performance or shipping the native pane. + +## WebRTC dependency record + +| Item | Pin | +| --- | --- | +| Swift package | `https://github.com/stasel/WebRTC.git`, exact `152.0.0` | +| Package/reference commit | `1d04692697cb642bfebf6ad2dd99fe52649c3d6d` | +| Reported upstream WebRTC source commit | `6f37672d358475cd17544121a12494da454d85fb` (`branch-heads/7977`) | +| Binary | [WebRTC-M152.xcframework.zip](https://github.com/stasel/WebRTC/releases/download/152.0.0/WebRTC-M152.xcframework.zip) | +| SHA-256 | `115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8` | +| Required slices present | macOS arm64/x86_64; iOS arm64; iOS Simulator arm64/x86_64 | +| Upstream workflow Xcode pin | `26.5` | + +SwiftPM validates the artifact checksum. The package and both Xcode workspaces pin the package revision. The reference checkout is [.repos/WebRTC](../../../.repos/WebRTC), added as a submodule. The downloaded archive checksum and required slices were inspected locally; this change uses the published binary rather than claiming a local WebRTC source build. + +The reference build script and release workflow are available at `.repos/WebRTC/scripts/build.sh` and `.repos/WebRTC/.github/workflows/webrtc-release.yml`. They build native Objective-C codec hooks and enable H.265 packetization. The distribution provides no Apple HEVC encoder/decoder implementation. Our custom VideoToolbox adapters provide H.264 by default and explicit HEVC/Main444 selection for probe experiments. + +The Codevisor-owned recipe is [scripts/build-webrtc.mjs](../../../scripts/build-webrtc.mjs), with source, depot_tools, Python and Xcode pins in [webrtc-build.lock.json](../../../scripts/webrtc-build.lock.json). `node scripts/build-webrtc.mjs --plan` reviews its five architecture builds without changing files. The build disables depot_tools auto-update, records actual dependency revisions and GN arguments, generates notices from each platform's GN target graph, preserves dSYMs/privacy manifests, signs the local frameworks and writes an XCFramework archive with a SHA-256 manifest. Source and artifacts stay under a recipe-specific `tmp/webrtc-source/` directory; SwiftPM pins are unchanged. + +A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Local preflight correctly rejected Xcode 27: the pinned upstream recipe requires Xcode 26.5, which is not installed here. The complete source build, notices review, binary smoke tests and candidate promotion remain unverified. Pinning inputs alone is not evidence of a byte-reproducible build. + +The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The archive does not include a generated aggregate third-party license file: generate and audit the complete notices from the actual source-build dependency graph before product distribution. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. + +Control API references: [WebRTC data channels](https://webrtc.org/getting-started/data-channels), [Apple’s private event-source state](https://developer.apple.com/documentation/coregraphics/cgeventsourcestateid), and [event-source user data](https://developer.apple.com/documentation/coregraphics/cgeventsource/userdata). Native channel signatures were checked against the pinned M152 framework headers. + + +`--check-recovery --keyframe-interval 60 --duration 15` performs a local loopback recovery check. After 120 decoder inputs, a package-only diagnostic hook destroys the VideoToolbox decoder state and rejects deltas until a fresh keyframe arrives. The check requires one reset, a replacement keyframe within two seconds and subsequent decoded/presented frames. A sixty-second nominal periodic interval prevents an ordinary scheduled keyframe from satisfying this short check. This is fault injection at the decoder, not a packet-loss emulator or a physical latency measurement. It cannot be combined with quality transitions or nonmedia checks. + +Transport keyframe requests remain pending through bounded-pool admission drops and VideoToolbox completion. Only usable keyframe output delivered to the encoder callback fulfills a request. One forced attempt is allowed in flight; dropped, failed or unusable output retries on a later input frame. Request generations prevent an older keyframe from clearing a newer request, and attempt identifiers reject stale completions. Local H.264 and HEVC Main444 checks recovered in 22.78 and 17.96 ms; a separate 4K Main444 check with one admitted frame recovered in 18.68 ms. These runs validate the recovery path but do not establish loss resilience on a real congested network, or constitute a hardware fault-injection test of VideoToolbox dropping a submitted keyframe. Six deterministic request-state tests cover that retry decision and completion ordering. See the [recovery results](../../../docs/measurements/native-screen-sharing-pipeline-2026-09-11/recovery-checks.json). + + +`--capture-picker-window` authorizes one window through macOS's native content picker. For capture isolation, `--show-workload --workload-window` creates a borderless window whose backing pixels match the requested raster, including portions outside a smaller physical display. The workload writes `.ready.json` with its window ID, actual point size and backing scale. SCK metrics independently record the selected content size/scale. This diagnostic supports comparing real 4K window capture without a competing Apple stream; it is explicitly distinct from full-display capture. The app's display chooser is unchanged. + +### Display-link cadence diagnostic + +The standalone probe accepts `--render-fps N` (30...240) to request a Metal display-link cadence independently of the stream FPS. It excludes `--render-on-arrival` and requires a viewer or loopback. This allows a fixed 60 fps stream to be measured on, for example, a verified 72 Hz display without changing application renderer defaults. The report labels `renderRequestedFPS`; actual scheduling must be checked through `renderDriveInterval` and native presentation counters. The fresh [separate-stream Apple comparison](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/README.md) records the current candidate and the remaining smoothness/latency limits. + +`--metal-display-link` selects a separate probe-only `CAMetalDisplayLink` driver. It uses the drawable supplied by each update, requests preferred frame latency 1 and the selected render cadence, and records callback intervals and target presentation lead. It requires a receiver or loopback and excludes frame-arrival rendering. Apple's requested latency is best effort, and the local control still measured approximately 50 ms from submission to presentation. The normal app continues using MTKView scheduling; this experiment makes no claim about Apple's Screen Sharing internals. + +### Calibrated image-age diagnostic + +`--show-workload --record-workload-times` records the first monotonic AppKit draw-start timestamp for each visible marker code. It limits workload duration to 240 seconds and retains at most 20,000 entries. `--clock-sync` runs alone as a bounded stdin/stdout timestamp responder for request-bracketed clock-offset calibration; it does not adjust system clocks. + +`--observe-window --window-id ID --content-top-points N --duration 30 --report /path.json` uses existing Screen Recording permission to sample one known viewer window. Omitting `--window-id` uses the system sharing picker. It retains numeric marker codes and local SCK sample/callback timestamps, with no recording, pixel retention, audio or input injection. Source raster defaults to 3840×2160; capture preserves aspect ratio up to 1920 pixels wide. A before/after clock calibration and the workload draw report are required to calculate cross-Mac image age. The [analysis and local control](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/image-age/README.md) document the first Codevisor/Apple comparison and its uncertainty. This metric is not physical presentation or input-to-photon latency. + +Add `--capture-kind display-crop` to sample the display region containing that window instead of an independent window surface. It requires a window ID, existing capture permission, and the whole window visible on one attached display. Readiness and final reports include the display ID and crop rectangle in display-local points. It preserves ordinary occlusion; moving or covering the marker can invalidate the observation. Independent-window and display-crop samples are distinct measurement modes and must not be mixed in a parity claim. The known workload marker requires a source width of at least 900 pixels because its response panel overlaps the marker below that width. + +`--viewer-display ID` places the probe viewer on an explicitly verified, attached local display. It requires a viewer or loopback. Viewer runs write `.viewer-ready.json` with the actual window/display ID, content size, chrome inset and backing scale, so observation can target the correct window without changing focus. A Release loopback check verified placement on display 2 at 960×540 content points and 2× backing scale; invalid IDs and incompatible modes were rejected. Display IDs are local, transient diagnostics and are not persisted as product preferences. + +`--low-latency-playout` is a separate receiver/loopback experiment. Before any RTC initialization, the standalone process sets the pinned M152 `WebRTC-ForcePlayoutDelay` trial to `min_ms:0,max_ms:0`. The upstream receiver accepts these values and the timing implementation selects rendering as soon as possible. This is a request to remove playout waiting, not a guarantee of zero latency or a bound on network delay; it may expose more stuttering under variable delivery. It retains encryption, codec recovery and transport feedback. Application initialization and defaults are unchanged. + +The trial was verified in the pinned [receiver parser](https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/rtp_video_stream_receiver2.cc) and [timing implementation](https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/timing/timing.cc); its identifier is present in the downloaded M152 binary. A ten-second local H.264 recovery check passed with a 0.18 ms cumulative receive-buffer mean and recovery in 16.45 ms after injected reference loss. Those local results are functional evidence, not a remote performance result. Probe media reports now include their monotonic measurement start and write `.media-ready.json` so later observations can be aligned with transport timelines. + +`--check-recovery --headless-recovery --keyframe-interval 60 --duration 10` runs real local WebRTC and hardware codec recovery without a window or capture. Its report sets `renderingEnabled: false`, labels presentation telemetry disabled, and requires decoded recovery instead of rendered frames. It rejects viewer/capture options and cannot establish presentation performance. The updated encoder passed local H.264 recovery in 16.13 ms and 4K Main444 recovery in 28.01 ms; [saved reports](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/recovery/README.md) retain the counters and limits. + +Add `--drop-recovery-keyframe` to deliberately discard the next forced encoder output after the decoder reset. The request must survive that loss: the probe requires one injected drop, a retry, at least three forced submissions (initial keyframe, discarded recovery attempt, replacement), and successful decoder recovery. Local H.264 and 4K Main444 checks passed with two requests, three submissions and one retry, recovering in 32.39 and 61.94 ms. The fault discards real VT output at the application callback; it does not induce an internal codec failure or emulate network packet loss. It is unarmed in the application. + +### Idle delivery audit + +Trailing packet loss before the desktop goes idle produces no decoder error, no later sequence gap and, in the pinned M152 sources, the same single PLI that a healthy idle transition produces. Every encoded frame therefore carries its content identity, the capture timestamp, in a zero-byte-free user-data SEI NAL unit (30 bytes for H.264, 31 for HEVC). The identity is attached to the pixel buffer by `ScreenSharingFrameSender`, because WebRTC's native source truncates and translates the submitted timestamp before the encoder sees it; the transport encoder refuses input without it through the `encoderError` lifecycle, and the decoder reads and strips the marker before VideoToolbox. + +After 100 ms without a capture submission the host re-offers a latest capture that was never encoded (four times at the threshold, then every 2 s until it is encoded, replaced or the peer closes) and then sends one `sourceIdle` notice over the video-refresh channel. The viewer compares the announced identity with its newest decoded one; if older it waits a 100 ms grace, extended by up to four more windows while strictly newer content keeps arriving (at most 500 ms after the notice), then requests a keyframe through the recovery path, keeps the target until that content is decoded, and retries stale completions with a 100 ms to 5 s backoff. These defaults were promoted after the [brief-loss measurements](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/brief-loss/README.md); the former 500 ms threshold and fixed 500 ms grace remain explicit probe overrides. Healthy idle costs one wake per 100 ms of activity, one nine-byte message per idle transition and no encode. [Measured results](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/network-loss/README.md) cover a local encrypted UDP relay with a five-second video blackout, plus the decoder-reset and idle checks on both Macs; the [brief-loss report](../../../docs/measurements/native-screen-sharing-independent-2026-09-11/brief-loss/README.md) covers brief trailing loss, healthy delayed streams and the promotion of the current defaults. + + +The receiver also requests a refresh over the bounded negotiated `codevisor.video-refresh.v1` SCTP stream (ID 4) before its first decoded keyframe and after decoder failure. Both endpoints rate-limit refreshes to at most ten per second. The sender retains at most one capture buffer and resubmits it with a strictly increasing host-clock timestamp, forcing a keyframe, so an idle desktop can recover without new ScreenCaptureKit output. Pending recovery retries stop after a successfully decoded keyframe; normal idle screens have no refresh timer. Decoder generations reject stale completions, and VT callback errors schedule session reset on the decoder queue. Close and format changes release the cached buffer. This adds one retained capture surface; sustained capture-pool and network-loss checks remain necessary. + +`--idle-on-decoder-reset` requires headless recovery and suspends fresh capture forwarding exactly at the reset edge, while leaving the last frame available for explicit refresh. It can combine with `--drop-recovery-keyframe`. The probe requires decoded recovery and an unchanged capture/encode/request count after a two-second settling checkpoint. `--pause-source-after N` instead stops the synthetic source at a specified time, with at least three active and three remaining seconds; it excludes idle-at-reset. These checks use hardware media with synthetic input and do not measure visible presentation or Apple parity. diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/NALUnitBitstream.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/NALUnitBitstream.swift new file mode 100644 index 000000000..2d1a4fa4d --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/NALUnitBitstream.swift @@ -0,0 +1,73 @@ +import Foundation + +/// WebRTC uses Annex B; VideoToolbox uses big-endian length-prefixed NALs. +/// Keep framing validation independent of the hardware codec. +enum NALUnitBitstream { + static let maximumFrameBytes = 16 * 1024 * 1024 + + static func nalUnits(_ annexB: Data) throws -> [Data] { + guard !annexB.isEmpty, annexB.count <= maximumFrameBytes else { + throw ScreenSharingError.invalid("Invalid NAL frame size.") + } + let bytes = [UInt8](annexB) + var starts: [(offset: Int, size: Int)] = [] + var index = 0 + while index + 2 < bytes.count { + if bytes[index] == 0, bytes[index + 1] == 0 { + if bytes[index + 2] == 1 { + guard starts.count < 1024 else { throw ScreenSharingError.invalid("Too many NAL units.") } + starts.append((index, 3)); index += 3; continue + } + if index + 3 < bytes.count, bytes[index + 2] == 0, bytes[index + 3] == 1 { + guard starts.count < 1024 else { throw ScreenSharingError.invalid("Too many NAL units.") } + starts.append((index, 4)); index += 4; continue + } + } + index += 1 + } + guard let first = starts.first, first.offset == 0 else { + throw ScreenSharingError.invalid("NAL frame has no Annex B prefix.") + } + return try starts.enumerated().map { position, start in + let begin = start.offset + start.size + let end = position + 1 < starts.count ? starts[position + 1].offset : bytes.count + guard begin < end else { throw ScreenSharingError.invalid("Empty NAL unit.") } + return Data(bytes[begin.. Data { + var result = Data() + for unit in units { + guard !unit.isEmpty, unit.count <= maximumFrameBytes - result.count - 4 else { + throw ScreenSharingError.invalid("Invalid NAL unit size.") + } + var length = UInt32(unit.count).bigEndian + withUnsafeBytes(of: &length) { result.append(contentsOf: $0) } + result.append(unit) + } + return result + } + + static func annexB(_ avcc: Data, lengthSize: Int = 4) throws -> Data { + guard !avcc.isEmpty, (1...4).contains(lengthSize), avcc.count <= maximumFrameBytes else { + throw ScreenSharingError.invalid("Invalid NAL length field.") + } + let bytes = [UInt8](avcc) + var result = Data() + var offset = 0 + while offset < bytes.count { + guard bytes.count - offset >= lengthSize else { throw ScreenSharingError.invalid("Truncated NAL length.") } + var length = 0 + for byte in bytes[offset..<(offset + lengthSize)] { length = (length << 8) | Int(byte) } + offset += lengthSize + guard length > 0, length <= bytes.count - offset, + length <= maximumFrameBytes - result.count - 4 + else { throw ScreenSharingError.invalid("Truncated NAL unit.") } + result.append(contentsOf: [0, 0, 0, 1]) + result.append(contentsOf: bytes[offset..<(offset + length)]) + offset += length + } + return result + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-LICENSE.txt b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-LICENSE.txt new file mode 100644 index 000000000..4c41b7b25 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-LICENSE.txt @@ -0,0 +1,29 @@ +Copyright (c) 2011, The WebRTC project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingAdaptiveQuality.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingAdaptiveQuality.swift new file mode 100644 index 000000000..2e9bd0721 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingAdaptiveQuality.swift @@ -0,0 +1,41 @@ +import Foundation + +/// Text-first congestion policy. A sustained two-second shortage lowers one +/// step; fifteen seconds of headroom restores one step. Missing estimates do +/// not change quality, and no decision depends on a wall clock. +public struct ScreenSharingAdaptiveQuality { + public private(set) var level = 0 + private let original: ScreenSharingVideoConfiguration + private var candidate: Int? + private var candidateSince: TimeInterval = 0 + private static let scale = [1.0, 1.0, 0.75, 0.5] + private static let rate = [1.0, 0.5, 0.3, 0.15] + + public init(configuration: ScreenSharingVideoConfiguration) { original = configuration } + + public mutating func update(availableBitrate: Double?, now: TimeInterval) -> ScreenSharingVideoConfiguration? { + guard let availableBitrate, availableBitrate.isFinite, availableBitrate > 0, now.isFinite else { + candidate = nil; return nil + } + let fraction = availableBitrate / Double(original.bitrate) + let next: Int + if level < 3, fraction < Self.rate[level] * 0.65 { + next = level + 1 + } else if level > 0, fraction > Self.rate[level - 1] * 0.9 { + next = level - 1 + } else { + candidate = nil; return nil + } + if candidate != next || now < candidateSince { + candidate = next; candidateSince = now; return nil + } + guard now - candidateSince >= (next > level ? 2 : 15) else { return nil } + level = next; candidate = nil + let scale = Self.scale[level] + return try? ScreenSharingVideoConfiguration( + width: max(64, Int(Double(original.width) * scale) / 2 * 2), + height: max(64, Int(Double(original.height) * scale) / 2 * 2), + framesPerSecond: min(original.framesPerSecond, level == 0 ? 60 : level == 3 ? 20 : 30), + bitrate: original.bitrate) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCapture.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCapture.swift new file mode 100644 index 000000000..64c07605c --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCapture.swift @@ -0,0 +1,327 @@ +#if os(macOS) + import CoreMedia + import ScreenCaptureKit + + /// One display, SDR, no audio. Lifecycle stays on the main actor; sample + /// delivery goes straight from SCK's serial queue into the native sender. + @MainActor + public final class ScreenSharingCapture { + public var onStopped: ((String) -> Void)? + private var generation = 0 + private var starting = false + private var stream: SCStream? + private var output: CaptureOutput? + private let queueDepth: Int + private let pixelFormat: OSType + private let copySurface: Bool + /// Optional SCK minimum-frame-interval request in frames per second, isolated + /// from the negotiated video/encoder rate. `nil` keeps the existing behaviour + /// (interval = 1 / video fps). Validated by `ScreenSharingCaptureIntervalRequest`. + /// The interval request in force. It changes only through `update(configuration:captureIntervalFPS:)`, and only + /// AFTER a successful stream update in the same generation (see `ScreenSharingCaptureRequestState`). + private(set) var requestState: ScreenSharingCaptureRequestState + private var captureIntervalFPS: Int? { requestState.overrideFramesPerSecond } + + public init( + queueDepth: Int = 3, pixelFormat: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + copySurface: Bool = false, captureIntervalFPS: Int? = nil + ) { + self.queueDepth = queueDepth; self.pixelFormat = pixelFormat + self.copySurface = copySurface + self.requestState = ScreenSharingCaptureRequestState(overrideFramesPerSecond: captureIntervalFPS) + } + + public static func displays() async throws -> [(id: UInt32, width: Int, height: Int)] { + try requirePermission() + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + return content.displays.map { ($0.displayID, $0.width, $0.height) } + } + + public func start( + displayID: UInt32, configuration: ScreenSharingVideoConfiguration, + sender: ScreenSharingFrameSender, metrics: ScreenSharingMetrics + ) async throws { + try Self.requirePermission() + guard stream == nil, !starting else { throw ScreenSharingError.invalid("Capture is already running.") } + generation += 1 + let generation = generation + starting = true + defer { if self.generation == generation { starting = false } } + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + try Task.checkCancellation() + guard self.generation == generation else { throw CancellationError() } + guard let display = content.displays.first(where: { $0.displayID == displayID }) else { + throw ScreenSharingError.unavailable("Selected display is unavailable.") + } + try await startResolved( + filter: SCContentFilter(display: display, excludingWindows: []), configuration: configuration, + sender: sender, metrics: metrics, generation: generation) + } + + /// Diagnostic mode: capture one window this process owns, selected from + /// current-process shareable content (available without TCC consent per the + /// SDK) by exact window ID and owning process ID. There is no display or + /// other-window fallback; any mismatch fails closed. SCK validates the + /// filter when the stream starts. + public func start( + ownedWindowID: UInt32, configuration: ScreenSharingVideoConfiguration, + sender: ScreenSharingFrameSender, metrics: ScreenSharingMetrics + ) async throws { + guard stream == nil, !starting else { throw ScreenSharingError.invalid("Capture is already running.") } + generation += 1 + let generation = generation + starting = true + defer { if self.generation == generation { starting = false } } + let content = try await SCShareableContent.currentProcess + try Task.checkCancellation() + guard self.generation == generation else { throw CancellationError() } + let processID = ProcessInfo.processInfo.processIdentifier + let candidates = content.windows.map { + ScreenSharingOwnedWindowSelection.Candidate( + windowID: $0.windowID, owningProcessID: $0.owningApplication?.processID) + } + metrics.label("captureOwnedWindowCandidates", String(candidates.count)) + let window: SCWindow + switch ScreenSharingOwnedWindowSelection.select(windowID: ownedWindowID, processID: processID, from: candidates) { + case .success(let selected): + guard let match = content.windows.first(where: { $0.windowID == selected.windowID }) else { + throw ScreenSharingError.unavailable("Owned window \(ownedWindowID) disappeared before capture.") + } + window = match + case .failure(let failure): + throw ScreenSharingError.unavailable("Owned window \(ownedWindowID) is not capturable: \(failure).") + } + metrics.label( + "captureSelection", "owned window \(ownedWindowID) of pid \(processID) via SCShareableContent.currentProcess") + metrics.label( + "captureOwnedWindowFramePoints", + "\(window.frame.width)x\(window.frame.height)@\(window.frame.minX),\(window.frame.minY)") + metrics.label("captureOwnedWindowOnScreen", String(window.isOnScreen)) + try await startResolved( + filter: SCContentFilter(desktopIndependentWindow: window), configuration: configuration, + sender: sender, metrics: metrics, generation: generation) + } + + /// A filter returned by the system content picker carries its own scoped + /// authorization. SCK validates it when starting; no persistent grant is needed. + public func start( + pickedFilter: SCContentFilter, configuration: ScreenSharingVideoConfiguration, + sender: ScreenSharingFrameSender, metrics: ScreenSharingMetrics + ) async throws { + guard stream == nil, !starting else { throw ScreenSharingError.invalid("Capture is already running.") } + generation += 1 + let generation = generation + starting = true + defer { if self.generation == generation { starting = false } } + try Task.checkCancellation() + try await startResolved( + filter: pickedFilter, configuration: configuration, sender: sender, metrics: metrics, generation: generation) + } + + private func startResolved( + filter: SCContentFilter, configuration: ScreenSharingVideoConfiguration, + sender: ScreenSharingFrameSender, metrics: ScreenSharingMetrics, generation: Int + ) async throws { + guard (3...8).contains(queueDepth) else { throw ScreenSharingError.invalid("Capture queue depth must be 3...8.") } + guard [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_32BGRA].contains(pixelFormat) else { + throw ScreenSharingError.invalid("Unsupported capture pixel format.") + } + // The interval request is validated on EVERY start and update (no silent fallback); telemetry + // describes only requests that were actually applied. + let interval = try requestState.validated(video: configuration, override: captureIntervalFPS) + let streamConfiguration = Self.streamConfiguration( + configuration, interval: interval, queueDepth: queueDepth, pixelFormat: pixelFormat) + requestState.commit(override: captureIntervalFPS, request: interval, metrics: metrics) + metrics.label("captureQueueDepth", String(queueDepth)) + metrics.label("capturePixelFormat", String(pixelFormat)) + metrics.label("captureSurface", copySurface ? "separate pool experiment" : "SCK surface") + metrics.label("captureContentStyle", String(describing: filter.style)) + metrics.label("captureContentWidthPoints", String(Double(filter.contentRect.width))) + metrics.label("captureContentHeightPoints", String(Double(filter.contentRect.height))) + metrics.label("captureContentPixelScale", String(filter.pointPixelScale)) + let output = CaptureOutput(sender: sender, metrics: metrics, copySurface: copySurface) { [weak self] message in + Task { @MainActor in + guard let self, self.generation == generation else { return } + self.onStopped?(message) + } + } + let stream = SCStream( + filter: filter, configuration: streamConfiguration, delegate: output) + try stream.addStreamOutput(output, type: .screen, sampleHandlerQueue: output.queue) + self.output = output + self.stream = stream + do { + try await stream.startCapture() + metrics.label("captureStartedAtNs", String(ScreenSharingMetrics.nowNs)) + guard self.generation == generation, !Task.isCancelled else { + try? await stream.stopCapture() + throw CancellationError() + } + } catch { + if self.generation == generation { self.stream = nil; self.output = nil } + throw error + } + } + + /// Unchanged behaviour for the probe and every existing caller: keeps whatever interval request is currently in + /// force — the value this capture was constructed with until an explicit level-aware update changes it, and that + /// changed value afterwards. + public func update(configuration: ScreenSharingVideoConfiguration) async throws { + try await update(configuration: configuration, captureIntervalFPS: captureIntervalFPS) + } + + /// The smallest explicit override needed for a level-aware request: the SAME single validated configuration path, + /// with the interval request supplied by the caller for this update (nil = the video rate). + /// + /// Ordering matters and is deliberate: validate, then call ScreenCaptureKit, and only then — still in the same + /// generation — commit the stored request state and its labels. A validation failure, a failed stream update or a + /// generation change leaves the previous request and telemetry exactly as they were. + public func update(configuration: ScreenSharingVideoConfiguration, captureIntervalFPS: Int?) async throws { + guard let stream, let output else { throw ScreenSharingError.unavailable("Capture is not running.") } + let generation = generation + // An override that was valid for the previous video rate may be invalid for the new one + // (e.g. video 30 + override 30, then video 60): the update throws instead of silently + // requesting the video rate while the telemetry still says 30. + try await applyIntervalUpdate( + configuration: configuration, override: captureIntervalFPS, metrics: output.metrics, + apply: { interval in + try await stream.updateConfiguration( + Self.streamConfiguration( + configuration, interval: interval, queueDepth: self.queueDepth, pixelFormat: self.pixelFormat)) + }, isCurrent: { self.generation == generation }) + } + + /// The whole update transaction in one place: validate, apply, re-check the generation, and only then commit the + /// request state and its labels. `apply` and `isCurrent` are the seams the real `update` fills with + /// ScreenCaptureKit and the generation counter, and which tests fill with a controlled apply — the ORDER and the + /// guards live here, in the code production runs, not in a test-only copy of the algorithm. + func applyIntervalUpdate( + configuration: ScreenSharingVideoConfiguration, override: Int?, metrics: ScreenSharingMetrics, + apply: (ScreenSharingCaptureIntervalRequest) async throws -> Void, isCurrent: () -> Bool + ) async throws { + let interval = try requestState.validated(video: configuration, override: override) + try await apply(interval) + guard isCurrent() else { throw CancellationError() } + requestState.commit(override: override, request: interval, metrics: metrics) + } + + /// Builds the stream configuration from an ALREADY VALIDATED interval request. Validation lives in + /// `ScreenSharingCaptureRequestState`, which both the start and update paths use, so no path can build a + /// configuration from an unvalidated request. + private static func streamConfiguration( + _ video: ScreenSharingVideoConfiguration, interval: ScreenSharingCaptureIntervalRequest, queueDepth: Int, + pixelFormat: OSType + ) -> SCStreamConfiguration { + let config = SCStreamConfiguration() + config.width = video.width; config.height = video.height + config.minimumFrameInterval = interval.minimumFrameInterval + config.queueDepth = queueDepth + config.pixelFormat = pixelFormat + config.colorSpaceName = CGColorSpace.itur_709 + config.showsCursor = true + config.capturesAudio = false + config.scalesToFit = true + return config + } + + private static func requirePermission() throws { + guard CGPreflightScreenCaptureAccess() else { + throw ScreenSharingError.unavailable( + "Screen Recording permission is required. Enable it for this app in System Settings, then relaunch." + ) + } + } + + /// Stops the stream and records the request/completion boundary so a + /// report can order it against the window's close. + public func stop() async throws { + generation += 1 + starting = false + let generation = generation + guard let stream else { + // Nothing to stop (never started, already stopped, or cleared by a + // failed stop). Recorded as skipped; this is not completion evidence. + output?.metrics.increment("captureStopSkippedNoStream") + return + } + self.stream = nil + let metrics = output?.metrics + defer { if self.generation == generation { output = nil } } + metrics?.label("captureStopRequestedAtNs", String(ScreenSharingMetrics.nowNs)) + do { + try await stream.stopCapture() + } catch { + // A failed stop is evidence too; it never produces a completion label. + metrics?.label("captureStopFailed", error.localizedDescription) + metrics?.increment("captureStopFailures") + throw error + } + metrics?.label("captureStopCompletedAtNs", String(ScreenSharingMetrics.nowNs)) + metrics?.increment("captureStops") + } + } + + private final class CaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable { + let queue = DispatchQueue(label: "codevisor.screen-sharing.capture", qos: .userInteractive) + let sender: ScreenSharingFrameSender + let metrics: ScreenSharingMetrics + let onStopped: @Sendable (String) -> Void + private let surfaceCopy: ScreenSharingCaptureBufferCopy? + + init( + sender: ScreenSharingFrameSender, metrics: ScreenSharingMetrics, copySurface: Bool, + onStopped: @escaping @Sendable (String) -> Void + ) { + self.sender = sender + self.metrics = metrics + self.onStopped = onStopped + surfaceCopy = copySurface ? ScreenSharingCaptureBufferCopy() : nil + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard type == .screen else { return } + let attachments = + CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) + as? [[SCStreamFrameInfo: Any]] + let status = attachments?.first?[.status] as? Int + let pixel = sampleBuffer.imageBuffer + // Every callback is counted by status; only a complete frame with an + // image is delivered. Absent callbacks are not a measured drop count. + guard + ScreenSharingCaptureCallbackAccounting.record( + valid: sampleBuffer.isValid, rawStatus: status, hasImage: pixel != nil, metrics: metrics), + let pixel + else { return } + // SCK's presentation timestamp uses the host clock. Forward it without + // copying pixels; WebRTC maps the local clock onto RTP timestamps. + let time = CMTimeConvertScale(sampleBuffer.presentationTimeStamp, timescale: 1_000_000_000, method: .default) + guard time.isNumeric, time.value >= 0 else { return } + let hostTime = CMClockGetTime(CMClockGetHostTimeClock()) + metrics.observe( + "captureDeliveryAge", milliseconds: CMTimeGetSeconds(hostTime - sampleBuffer.presentationTimeStamp) * 1000) + do { + let frame: CVPixelBuffer + if let surfaceCopy { + let started = ScreenSharingMetrics.nowNs + guard let copied = try surfaceCopy.copy(pixel) else { + metrics.increment("captureCopyPoolDrops"); return + } + metrics.observe("captureSurfaceCopy", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + frame = copied + } else { + frame = pixel + } + sender.push(ScreenSharingVideoFrame(pixelBuffer: frame, timestampNs: time.value)) + } catch { + metrics.increment("captureCopyErrors") + metrics.label("captureError", error.localizedDescription) + onStopped(error.localizedDescription) + } + } + + func stream(_ stream: SCStream, didStopWithError error: any Error) { + metrics.label("captureError", error.localizedDescription) + onStopped(error.localizedDescription) + } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureBufferCopy.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureBufferCopy.swift new file mode 100644 index 000000000..1a82049e9 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureBufferCopy.swift @@ -0,0 +1,50 @@ +import CoreVideo +import VideoToolbox + +/// Used only on the serial capture callback queue. The experiment releases +/// SCK's surface after a transfer into a separately bounded, IOSurface-backed pool. +final class ScreenSharingCaptureBufferCopy { + private var transfer: VTPixelTransferSession? + private var pool: CVPixelBufferPool? + private var format: (width: Int, height: Int, type: OSType)? + static let maximumBuffers = 6 + + deinit { + if let transfer { VTPixelTransferSessionInvalidate(transfer) } + } + + func copy(_ source: CVPixelBuffer) throws -> CVPixelBuffer? { + if transfer == nil { + let status = VTPixelTransferSessionCreate(allocator: nil, pixelTransferSessionOut: &transfer) + guard status == noErr else { throw ScreenSharingError.codec("Create capture transfer", status) } + } + let width = CVPixelBufferGetWidth(source) + let height = CVPixelBufferGetHeight(source) + let type = CVPixelBufferGetPixelFormatType(source) + if format?.width != width || format?.height != height || format?.type != type { + let attributes: [CFString: Any] = [ + kCVPixelBufferWidthKey: width, kCVPixelBufferHeightKey: height, + kCVPixelBufferPixelFormatTypeKey: type, kCVPixelBufferIOSurfacePropertiesKey: [:], + kCVPixelBufferMetalCompatibilityKey: true, + ] + var created: CVPixelBufferPool? + let status = CVPixelBufferPoolCreate(nil, nil, attributes as CFDictionary, &created) + guard status == kCVReturnSuccess else { throw ScreenSharingError.codec("Create capture copy pool", status) } + pool = created + format = (width, height, type) + } + guard let pool, let transfer else { throw ScreenSharingError.unavailable("Capture transfer is unavailable.") } + var destination: CVPixelBuffer? + let status = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( + nil, pool, + [kCVPixelBufferPoolAllocationThresholdKey: Self.maximumBuffers] as CFDictionary, &destination) + if status == kCVReturnWouldExceedAllocationThreshold { return nil } + guard status == kCVReturnSuccess, let destination else { + throw ScreenSharingError.codec("Allocate capture copy", status) + } + CVBufferPropagateAttachments(source, destination) + let copied = VTPixelTransferSessionTransferImage(transfer, from: source, to: destination) + guard copied == noErr else { throw ScreenSharingError.codec("Copy capture surface", copied) } + return destination + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureIntervalRequest.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureIntervalRequest.swift new file mode 100644 index 000000000..759e21871 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureIntervalRequest.swift @@ -0,0 +1,31 @@ +import CoreMedia +import Foundation + +/// The ScreenCaptureKit minimum-frame-interval REQUEST, isolated from the +/// negotiated video/encoder rate. Without an override the request equals the +/// video rate (existing behaviour). An override must be at least the video +/// rate and at most 120 fps. This describes what is asked of ScreenCaptureKit; +/// it never implies that frames complete at that cadence. +public struct ScreenSharingCaptureIntervalRequest: Equatable, Sendable { + public static let maximumFramesPerSecond = 120 + + public let videoFramesPerSecond: Int + public let requestedFramesPerSecond: Int + public var isOverride: Bool { requestedFramesPerSecond != videoFramesPerSecond } + public var minimumFrameInterval: CMTime { CMTime(value: 1, timescale: Int32(requestedFramesPerSecond)) } + + public init(videoFramesPerSecond: Int, overrideFramesPerSecond: Int?) throws { + guard videoFramesPerSecond > 0 else { throw ScreenSharingError.invalid("Video frame rate must be positive.") } + self.videoFramesPerSecond = videoFramesPerSecond + guard let override = overrideFramesPerSecond else { + self.requestedFramesPerSecond = videoFramesPerSecond + return + } + guard override >= videoFramesPerSecond, override <= Self.maximumFramesPerSecond else { + throw ScreenSharingError.invalid( + "Capture interval request must be at least the video rate (\(videoFramesPerSecond)) and at most " + + "\(Self.maximumFramesPerSecond) fps.") + } + self.requestedFramesPerSecond = override + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureRequestState.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureRequestState.swift new file mode 100644 index 000000000..37007a770 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingCaptureRequestState.swift @@ -0,0 +1,34 @@ +import Foundation + +/// The capture-interval REQUEST state of one running capture, separated from ScreenCaptureKit so the +/// validate → apply → commit ordering is testable without a stream. +/// +/// The rule it enforces: nothing about the stored request or its telemetry changes until the stream update has +/// actually succeeded in the same generation. A validation failure, a failed update or a stale generation leaves the +/// previous request and labels exactly as they were — the telemetry never describes a request that is not in force. +public struct ScreenSharingCaptureRequestState: Sendable { + /// The override currently in force; nil means the request equals the video rate. + public private(set) var overrideFramesPerSecond: Int? + + public init(overrideFramesPerSecond: Int?) { self.overrideFramesPerSecond = overrideFramesPerSecond } + + /// Validates a candidate request WITHOUT changing any state. Throws exactly as the single validated path does, so an + /// override that was valid for the previous video rate but not for the new one fails here, before the stream call. + public func validated( + video: ScreenSharingVideoConfiguration, override: Int? + ) throws -> ScreenSharingCaptureIntervalRequest { + try ScreenSharingCaptureIntervalRequest( + videoFramesPerSecond: video.framesPerSecond, overrideFramesPerSecond: override) + } + + /// Commits a request that HAS been applied, and only then publishes its telemetry. Call this after a successful + /// stream update in the same generation, never before. + public mutating func commit( + override: Int?, request: ScreenSharingCaptureIntervalRequest, metrics: ScreenSharingMetrics + ) { + overrideFramesPerSecond = override + metrics.label("captureRequestedMinimumFrameIntervalFPS", String(request.requestedFramesPerSecond)) + metrics.label( + "captureRequestedFrameIntervalOverride", request.isOverride ? String(request.requestedFramesPerSecond) : "none") + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardMessage.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardMessage.swift new file mode 100644 index 000000000..bc5417d15 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardMessage.swift @@ -0,0 +1,41 @@ +import Foundation + +/// Clipboard requests are explicit, connection-local operations. Text never +/// enters the workspace database, signaling server, metrics or diagnostics. +public enum ScreenSharingClipboardMessage: Codable, Sendable, Equatable { + case read(id: UUID) + case begin(id: UUID, bytes: Int) + case chunk(id: UUID, index: Int, data: Data) + case ack(id: UUID, nextIndex: Int) + case end(id: UUID) + case result(id: UUID, error: String?) + case cancel(id: UUID) + + public static let maximumTextBytes = 64 * 1024 + public static let chunkBytes = 2048 + public var id: UUID { + switch self { + case .read(let id), .begin(let id, _), .chunk(let id, _, _), .ack(let id, _), .end(let id), + .result(let id, _), .cancel(let id): + id + } + } + public func encoded() throws -> Data { + let encoder = JSONEncoder() + // Base64 can consist almost entirely of slashes. Escaping each slash + // would double a chunk's wire size and exceed the bounded channel. + encoder.outputFormatting = [.withoutEscapingSlashes] + let data = try encoder.encode(Envelope(version: 1, message: self)) + guard data.count <= 4096 else { throw ScreenSharingError.invalid("Clipboard message is too large.") } + return data + } + public static func decode(_ data: Data) throws -> Self { + guard data.count <= 4096 else { throw ScreenSharingError.invalid("Clipboard message is too large.") } + let envelope = try JSONDecoder().decode(Envelope.self, from: data) + guard envelope.version == 1 else { throw ScreenSharingError.invalid("Unsupported clipboard protocol.") } + return envelope.message + } + private struct Envelope: Codable { let version: Int; let message: ScreenSharingClipboardMessage } +} + +public typealias ScreenSharingClipboardChannel = ScreenSharingDataChannel diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardTransfer.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardTransfer.swift new file mode 100644 index 000000000..8ca9c42f1 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingClipboardTransfer.swift @@ -0,0 +1,126 @@ +import Foundation + +/// Stop-and-wait acknowledgments admit only one 2 KiB clipboard chunk at a time. +/// An explicit read authorizes exactly one incoming transfer on a viewer. +@MainActor +public final class ScreenSharingClipboardTransfer { + public var onFinished: ((String?) -> Void)? + public var isBusy: Bool { outgoing != nil || incoming != nil || requested != nil } + private let send: (ScreenSharingClipboardMessage) -> Bool + private let now: () -> TimeInterval + private let canReceiveUnsolicited: () -> Bool + private let read: () throws -> String + private let write: (String) throws -> Void + private var requested: UUID? + private var outgoing: Outgoing? + private var incoming: Incoming? + private var deadline: TimeInterval = 0 + private struct Outgoing { let id: UUID; let data: Data; var nextIndex = 0; var awaitingEnd = false } + private struct Incoming { let id: UUID; let count: Int; var data = Data(); var nextIndex = 0 } + + public init( + send: @escaping (ScreenSharingClipboardMessage) -> Bool, + now: @escaping () -> TimeInterval = { ProcessInfo.processInfo.systemUptime }, + canReceiveUnsolicited: @escaping () -> Bool = { false }, + read: @escaping () throws -> String, write: @escaping (String) throws -> Void + ) { + self.send = send; self.now = now; self.canReceiveUnsolicited = canReceiveUnsolicited + self.read = read; self.write = write + } + + public func requestText() { + guard !isBusy else { return } + let id = UUID(); requested = id; deadline = now() + 10 + transmit(.read(id: id)) + } + public func sendText(_ text: String) { + guard !isBusy else { return } + start(text, id: UUID()) + } + public func tick() { + if isBusy, now() >= deadline { cancel(reason: "Clipboard transfer timed out. Try again.") } + } + public func cancel(reason: String? = nil) { + let id = outgoing?.id ?? incoming?.id ?? requested + guard id != nil else { return } + finish(reason) + if let id { _ = send(.cancel(id: id)) } + } + + public func receive(_ message: ScreenSharingClipboardMessage) { + tick() + switch message { + case .read(let id): + guard !isBusy, canReceiveUnsolicited() else { reject(id, "Clipboard is unavailable or busy."); return } + do { start(try read(), id: id) } catch { reject(id, error.localizedDescription) } + case .begin(let id, let count): + guard outgoing == nil, incoming == nil, + requested == id || (requested == nil && canReceiveUnsolicited()), + (0...ScreenSharingClipboardMessage.maximumTextBytes).contains(count) + else { reject(id, "Clipboard transfer was not requested or is too large."); return } + incoming = Incoming(id: id, count: count); deadline = now() + 10 + transmit(.ack(id: id, nextIndex: 0)) + case .chunk(let id, let index, let data): + guard var current = incoming, current.id == id else { return } + guard index == current.nextIndex, !data.isEmpty, data.count <= ScreenSharingClipboardMessage.chunkBytes, + current.data.count + data.count <= current.count + else { rejectCurrent(id, "Invalid clipboard chunk."); return } + current.data.append(data); current.nextIndex += 1; incoming = current + transmit(.ack(id: id, nextIndex: current.nextIndex)) + case .ack(let id, let index): + guard var current = outgoing, current.id == id, !current.awaitingEnd else { return } + guard index == current.nextIndex else { cancel(reason: "Invalid clipboard acknowledgment."); return } + let offset = index * ScreenSharingClipboardMessage.chunkBytes + if offset >= current.data.count { + current.awaitingEnd = true; outgoing = current + transmit(.end(id: id)) + } else { + let end = min(current.data.count, offset + ScreenSharingClipboardMessage.chunkBytes) + current.nextIndex += 1; outgoing = current + transmit(.chunk(id: id, index: index, data: current.data.subdata(in: offset.. [RTCVideoCodecInfo] { + [RTCVideoCodecInfo(name: codec.payloadName, parameters: codec.sdpParameters)] + } + + func createEncoder(_ info: RTCVideoCodecInfo) -> (any RTCVideoEncoder)? { + guard info.name == codec.payloadName else { return nil } + return ScreenSharingRTCEncoder( + metrics: metrics, useLowLatencyRateControl: useLowLatencyRateControl, codec: codec, + disableLookAhead: disableLookAhead, maximumPendingFrames: maximumPendingFrames, staticCodecRate: staticCodecRate, + completeEachFrame: completeEachFrame, prioritizeSpeed: prioritizeSpeed, + keyframeIntervalSeconds: keyframeIntervalSeconds, dropCheck: encoderDropCheck, + refreshRequest: encoderRefreshRequest, idleMonitor: sourceIdleMonitor) + } + + func createDecoder(_ info: RTCVideoCodecInfo) -> (any RTCVideoDecoder)? { + guard info.name == codec.payloadName else { return nil } + return ScreenSharingRTCDecoder( + metrics: metrics, codec: codec, recoveryCheck: recoveryCheck, refreshSignal: refreshSignal, + deliveryAudit: deliveryAudit, frameAudit: frameDeliveryAudit) + } +} + +/// WebRTC serializes the encoder methods; only the callback crosses VT's +/// output queue. It is copied under the lock before being invoked. +private final class ScreenSharingRTCEncoder: NSObject, RTCVideoEncoder, @unchecked Sendable { + let metrics: ScreenSharingMetrics + private let lock = NSLock() + private var callback: RTCVideoEncoderCallback? + private var encoder: ScreenSharingEncoder? + private let useLowLatencyRateControl: Bool + private let disableLookAhead: Bool + private let maximumPendingFrames: Int + private let staticCodecRate: Bool + private let completeEachFrame: Bool + private let prioritizeSpeed: Bool + private let keyframeIntervalSeconds: Int + private let codec: ScreenSharingVideoCodec + private let dropCheck: ScreenSharingEncoderDropCheck + private let refreshRequest: ScreenSharingEncoderRefreshRequest + private let idleMonitor: ScreenSharingSourceIdleMonitor + var resolutionAlignment: Int { 2 } + var applyAlignmentToAllSimulcastLayers: Bool { true } + var supportsNativeHandle: Bool { true } + + init( + metrics: ScreenSharingMetrics, useLowLatencyRateControl: Bool, codec: ScreenSharingVideoCodec, + disableLookAhead: Bool, maximumPendingFrames: Int, staticCodecRate: Bool, completeEachFrame: Bool, + prioritizeSpeed: Bool, keyframeIntervalSeconds: Int, dropCheck: ScreenSharingEncoderDropCheck, + refreshRequest: ScreenSharingEncoderRefreshRequest, idleMonitor: ScreenSharingSourceIdleMonitor + ) { + self.metrics = metrics; self.useLowLatencyRateControl = useLowLatencyRateControl + self.codec = codec + self.disableLookAhead = disableLookAhead + self.maximumPendingFrames = maximumPendingFrames + self.staticCodecRate = staticCodecRate + self.completeEachFrame = completeEachFrame + self.prioritizeSpeed = prioritizeSpeed + self.keyframeIntervalSeconds = keyframeIntervalSeconds + self.dropCheck = dropCheck + self.refreshRequest = refreshRequest + self.idleMonitor = idleMonitor + } + func implementationName() -> String { "CodevisorVideoToolbox\(codec.payloadName)" } + func scalingSettings() -> RTCVideoEncoderQpThresholds? { nil } + func setCallback(_ callback: RTCVideoEncoderCallback?) { lock.withLock { self.callback = callback } } + + func startEncode(with settings: RTCVideoEncoderSettings, numberOfCores: Int32) -> Int { + do { + encoder?.stop() + let configuration = try ScreenSharingVideoConfiguration( + width: Int(settings.width), height: Int(settings.height), framesPerSecond: max(1, Int(settings.maxFramerate)), + bitrate: max(100_000, Int(settings.startBitrate) * 1000)) + let encoder = try ScreenSharingEncoder( + configuration: configuration, metrics: metrics, useLowLatencyRateControl: useLowLatencyRateControl, + codec: codec, + disableLookAhead: disableLookAhead, maximumPendingFrames: maximumPendingFrames, + completeEachFrame: completeEachFrame, prioritizeSpeed: prioritizeSpeed, + keyframeIntervalSeconds: keyframeIntervalSeconds + ) + encoder.onFrame { [weak self] frame in self?.deliver(frame) } + encoder.useDropCheck(dropCheck) + self.encoder = encoder + return 0 + } catch { metrics.label("encoderError", error.localizedDescription); return -1 } + } + + func encode( + _ frame: RTCVideoFrame, codecSpecificInfo info: (any RTCCodecSpecificInfo)?, frameTypes: [NSNumber] + ) -> Int { + guard let encoder, let native = frame.buffer as? RTCCVPixelBuffer, !native.requiresCropping() else { + metrics.increment("unsupportedEncoderInput"); return -1 + } + // WebRTC translated this frame's timestamp; identity rides on the buffer. + guard let identity = ScreenSharingFrameIdentity.required(of: native.pixelBuffer, metrics: metrics) else { + return -1 + } + do { + let refresh = refreshRequest.consume() + let requestedKey = frameTypes.contains(where: { $0.intValue == RTCFrameType.videoFrameKey.rawValue }) + metrics.trace( + "encoderInput", + "\(ScreenSharingMetrics.nowNs) ts=\(frame.timeStampNs) id=\(identity) latch=\(refresh) webrtcKey=\(requestedKey)" + ) + try encoder.encode( + ScreenSharingVideoFrame( + pixelBuffer: native.pixelBuffer, timestampNs: frame.timeStampNs, + rtpTimestamp: UInt32(bitPattern: frame.timeStamp), sourceTimestampNs: identity), + forceKeyFrame: refresh || frameTypes.contains(where: { $0.intValue == RTCFrameType.videoFrameKey.rawValue })) + return 0 + } catch { metrics.label("encoderError", error.localizedDescription); return -1 } + } + + func release() -> Int { encoder?.stop(); encoder = nil; return 0 } + + func setBitrate(_ bitrateKbit: UInt32, framerate: UInt32) -> Int32 { + if staticCodecRate { + metrics.increment("encoderRateUpdatesHeld") + metrics.label("encoderRateUpdatePolicy", "initial rate experiment") + return 0 + } + do { + try encoder?.setBitrate(max(1, Int(bitrateKbit)) * 1000, framesPerSecond: min(60, max(1, Int(framerate)))) + return 0 + } catch { metrics.label("encoderError", error.localizedDescription); return -1 } + } + + private func deliver(_ frame: ScreenSharingEncodedFrame) { + metrics.event("encodedDeliveryInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + let image = RTCEncodedImage() + image.buffer = frame.data + image.encodedWidth = Int32(frame.width) + image.encodedHeight = Int32(frame.height) + image.timeStamp = frame.rtpTimestamp + image.captureTimeMs = frame.timestampNs / 1_000_000 + image.frameType = frame.isKeyFrame ? .videoFrameKey : .videoFrameDelta + image.contentType = .screenshare + image.rotation = ._0 + let info: any RTCCodecSpecificInfo + if codec == .h264 { + let h264 = RTCCodecSpecificInfoH264() + h264.packetizationMode = .nonInterleaved + info = h264 + } else { + info = ScreenSharingHEVCInfo() + } + // Qualify the property read: the new `callback` binding shadows it, which older compilers (Swift 6.3.3 on + // Xcode 26.6) cannot infer through `withLock`. Take the snapshot under the lock, call it outside — unchanged. + let callback = lock.withLock { self.callback } + let started = ScreenSharingMetrics.nowNs + _ = callback?(image, info) + metrics.observe("encodedDeliveryCallback", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + guard callback != nil, let sourceTimestampNs = frame.sourceTimestampNs else { return } + // Only output handed to WebRTC can be announced as the host's newest content. + idleMonitor.recordEncoded(timestampNs: sourceTimestampNs) + metrics.label("latestEncodedSourceTimestampNs", String(sourceTimestampNs)) + } +} + +/// M152's ObjC bridge has no HEVC-specific metadata type. Its native sender +/// selects the H.265 packetizer from the negotiated payload, and parses Annex B. +private final class ScreenSharingHEVCInfo: NSObject, RTCCodecSpecificInfo {} + +private final class ScreenSharingRTCDecoder: NSObject, RTCVideoDecoder, @unchecked Sendable { + let metrics: ScreenSharingMetrics + private let codec: ScreenSharingVideoCodec + private let recoveryCheck: ScreenSharingDecoderRecoveryCheck + private let refreshSignal: ScreenSharingRefreshSignal + private let deliveryAudit: ScreenSharingDeliveryAudit + /// Receiver-only diagnostic frame-delivery audit (nil = disabled, no work). + private let frameAudit: ScreenSharingFrameDeliveryAudit? + private let lock = NSLock() + private var callback: RTCVideoDecoderCallback? + private var decoder: ScreenSharingDecoder? + + init( + metrics: ScreenSharingMetrics, codec: ScreenSharingVideoCodec, recoveryCheck: ScreenSharingDecoderRecoveryCheck, + refreshSignal: ScreenSharingRefreshSignal, deliveryAudit: ScreenSharingDeliveryAudit, + frameAudit: ScreenSharingFrameDeliveryAudit? + ) { + self.metrics = metrics; self.codec = codec; self.recoveryCheck = recoveryCheck + self.refreshSignal = refreshSignal + self.deliveryAudit = deliveryAudit + self.frameAudit = frameAudit + } + func implementationName() -> String { "CodevisorVideoToolbox\(codec.payloadName)" } + func setCallback(_ callback: @escaping RTCVideoDecoderCallback) { lock.withLock { self.callback = callback } } + + func startDecode(withNumberOfCores numberOfCores: Int32) -> Int { + decoder?.stop() + decoder = ScreenSharingDecoder(metrics: metrics, codec: codec, deliveryAudit: frameAudit) { [weak self] frame in + guard let self else { return } + if let sourceTimestampNs = frame.sourceTimestampNs { + self.deliveryAudit.decoded(sourceTimestampNs: sourceTimestampNs) + self.metrics.label("latestDecodedSourceTimestampNs", String(sourceTimestampNs)) + } + let decoded = RTCVideoFrame( + buffer: RTCCVPixelBuffer(pixelBuffer: frame.pixelBuffer), rotation: ._0, timeStampNs: frame.timestampNs) + decoded.timeStamp = Int32(bitPattern: frame.rtpTimestamp) + let callback = self.lock.withLock { self.callback } + callback?(decoded) + } + decoder?.useRefreshSignal(refreshSignal) + return 0 + } + + func decode( + _ image: RTCEncodedImage, missingFrames: Bool, codecSpecificInfo info: (any RTCCodecSpecificInfo)?, + renderTimeMs: Int64 + ) -> Int { + guard let decoder, image.rotation == ._0 else { return -1 } + switch recoveryCheck.inspect( + keyFrame: image.frameType == .videoFrameKey, nowNs: ScreenSharingMetrics.nowNs) + { + case .accept: break + case .reset: + decoder.stop() + metrics.increment("injectedDecoderResets") + metrics.trace("decoderInput", "\(ScreenSharingMetrics.nowNs) rtp=\(image.timeStamp) key=false action=reset") + refreshSignal.request() + return -1 + case .rejectDelta: + metrics.increment("recoveryDeltaFramesRejected") + metrics.trace("decoderInput", "\(ScreenSharingMetrics.nowNs) rtp=\(image.timeStamp) key=false action=rejectDelta") + refreshSignal.request() + return -1 + case .recovered(let milliseconds): + metrics.increment("recoveryKeyframesReceived") + metrics.observe("decoderResetToRecoveryKeyframe", milliseconds: milliseconds) + metrics.trace("decoderInput", "\(ScreenSharingMetrics.nowNs) rtp=\(image.timeStamp) key=true action=recovered") + } + if image.frameType == .videoFrameKey { + metrics.trace("decoderInput", "\(ScreenSharingMetrics.nowNs) rtp=\(image.timeStamp) key=true action=accept") + } + metrics.event("decoderInputInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + // decoder input after WebRTC reassembly/release + let auditIdentity = frameAudit?.decoderInput(rtpTimestamp: image.timeStamp) + do { + try decoder.decode( + ScreenSharingEncodedFrame( + data: image.buffer, timestampNs: renderTimeMs * 1_000_000, rtpTimestamp: image.timeStamp, + width: Int(image.encodedWidth), height: Int(image.encodedHeight), + isKeyFrame: image.frameType == .videoFrameKey), auditIdentity: auditIdentity) + return 0 + } catch { + decoder.stop() + refreshSignal.request() + metrics.label("decoderError", error.localizedDescription) + return -1 + } + } + + func release() -> Int { decoder?.stop(); decoder = nil; return 0 } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingControlMessage.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingControlMessage.swift new file mode 100644 index 000000000..571b60e2d --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingControlMessage.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Transient control messages travel only on the authenticated media peer. +/// A fresh host-issued lease prevents delayed input from a previous control period. +public enum ScreenSharingControlMessage: Codable, Sendable, Equatable { + case request(id: UUID) + case grant(request: UUID, lease: UUID) + case denied(request: UUID, reason: String) + case release(lease: UUID) + case revoked(lease: UUID, reason: String) + case heartbeat(lease: UUID) + case input(lease: UUID, sequence: UInt64, event: ScreenSharingInputEvent) + + public static let maximumBytes = 4096 + public func encoded() throws -> Data { + let data = try JSONEncoder().encode(Envelope(version: 1, message: self)) + guard data.count <= Self.maximumBytes else { throw ScreenSharingError.invalid("Control message is too large.") } + return data + } + public static func decode(_ data: Data) throws -> Self { + guard data.count <= maximumBytes else { throw ScreenSharingError.invalid("Control message is too large.") } + let envelope = try JSONDecoder().decode(Envelope.self, from: data) + guard envelope.version == 1 else { throw ScreenSharingError.invalid("Unsupported control protocol.") } + if case .input(_, _, let event) = envelope.message, !event.isValid { + throw ScreenSharingError.invalid("Invalid control input.") + } + return envelope.message + } + private struct Envelope: Codable { let version: Int; let message: ScreenSharingControlMessage } +} + +public struct ScreenSharingPointer: Codable, Sendable, Equatable { + public let x: Double + public let y: Double + public init(x: Double, y: Double) { self.x = x; self.y = y } + public var isValid: Bool { x.isFinite && y.isFinite && (0...1).contains(x) && (0...1).contains(y) } +} + +/// Physical Mac key codes use the host's keyboard layout. Text insertion is a +/// separate operation; it must never be interpreted as a physical shortcut. +public enum ScreenSharingInputEvent: Codable, Sendable, Equatable { + case move(ScreenSharingPointer, modifiers: UInt8) + case button(ScreenSharingPointer, button: UInt8, down: Bool, clicks: UInt8, modifiers: UInt8) + case scroll(ScreenSharingPointer, x: Int32, y: Int32, modifiers: UInt8) + case key(code: UInt16, down: Bool, repeatKey: Bool, modifiers: UInt8) + case text(String) + + public var isValid: Bool { + switch self { + case .move(let point, let flags): point.isValid && flags < 64 + case .button(let point, let button, _, let clicks, let flags): + point.isValid && button <= 2 && (1...3).contains(clicks) && flags < 64 + case .scroll(let point, let x, let y, let flags): + point.isValid && (-4096...4096).contains(x) && (-4096...4096).contains(y) && flags < 64 + case .key(let code, _, _, let flags): code <= 126 && flags < 64 + case .text(let text): !text.isEmpty && text.utf16.count <= 1024 + } + } +} + +/// Coordinates are normalized in the captured display, with the origin at its +/// top left. This is independent of Retina scale and the display's global origin. +public enum ScreenSharingVideoGeometry { + public static func pointer( + x: Double, y: Double, surfaceWidth: Double, surfaceHeight: Double, + videoWidth: Double, videoHeight: Double, fit: Bool, clamp: Bool = false + ) -> ScreenSharingPointer? { + guard [x, y, surfaceWidth, surfaceHeight, videoWidth, videoHeight].allSatisfy(\.isFinite), + surfaceWidth > 0, surfaceHeight > 0, videoWidth > 0, videoHeight > 0 + else { return nil } + let scale = fit ? min(surfaceWidth / videoWidth, surfaceHeight / videoHeight) : 1 + let width = videoWidth * scale; let height = videoHeight * scale + let px = (x - (surfaceWidth - width) / 2) / width + let py = (y - (surfaceHeight - height) / 2) / height + let point = ScreenSharingPointer(x: clamp ? min(1, max(0, px)) : px, y: clamp ? min(1, max(0, py)) : py) + return point.isValid ? point : nil + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDataChannel.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDataChannel.swift new file mode 100644 index 000000000..b9c64ecd8 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDataChannel.swift @@ -0,0 +1,117 @@ +import Foundation +@preconcurrency import WebRTC + +/// A bounded native ordered channel. Independent SCTP streams carry input and +/// clipboard transfers. Video uses its own RTP transport. +@MainActor +public final class ScreenSharingDataChannel { + public var onMessage: ((Message) -> Void)? + public var onAvailabilityChanged: ((Bool) -> Void)? + public var isAvailable: Bool { !closed && channel.readyState == .open } + private let channel: RTCDataChannel + private let receiver = ScreenSharingControlReceiver() + private var closed = false + private let encode: (Message) throws -> Data + + init( + connection: RTCPeerConnection, id: Int32, label: String, + encode: @escaping (Message) throws -> Data, decode: @escaping (Data) throws -> Message + ) throws { + self.encode = encode + let options = RTCDataChannelConfiguration() + options.isOrdered = true + options.isNegotiated = true + options.channelId = id + options.protocol = label + guard let channel = connection.dataChannel(forLabel: label, configuration: options) else { + throw ScreenSharingError.unavailable("Cannot create the screen-sharing data channel.") + } + self.channel = channel + receiver.deliver = { [weak self] packets, failed in + guard let self, !self.closed else { return } + if failed { self.close(); return } + for packet in packets { + guard !self.closed else { return } + do { self.onMessage?(try decode(packet)) } catch { self.close(); return } + } + } + receiver.stateChanged = { [weak self] in + guard let self, !self.closed else { return } + self.onAvailabilityChanged?(self.isAvailable) + } + channel.delegate = receiver + } + + @discardableResult + public func send(_ message: Message) -> Bool { + guard isAvailable, let data = try? encode(message) else { return false } + guard channel.bufferedAmount + UInt64(data.count) <= 16 * 1024, + channel.sendData(RTCDataBuffer(data: data, isBinary: true)) + else { close(); return false } + return true + } + + public func close() { + guard !closed else { return } + closed = true + receiver.stop() + channel.delegate = nil + channel.close() + onAvailabilityChanged?(false) + onAvailabilityChanged = nil; onMessage = nil + } +} + +/// WebRTC callbacks must not create an unbounded backlog of main-actor tasks. +/// One scheduled drain admits at most 256 messages / 64 KiB. Overflow closes the affected channel. +private final class ScreenSharingControlReceiver: NSObject, RTCDataChannelDelegate, @unchecked Sendable { + var deliver: (@MainActor @Sendable ([Data], Bool) -> Void)? + var stateChanged: (@MainActor @Sendable () -> Void)? + private let lock = NSLock() + private var inbox = ScreenSharingControlInbox() + + func stop() { lock.withLock { inbox.stop() } } + func dataChannelDidChangeState(_ dataChannel: RTCDataChannel) { + Task { @MainActor [weak self] in self?.stateChanged?() } + } + func dataChannel(_ dataChannel: RTCDataChannel, didReceiveMessageWith buffer: RTCDataBuffer) { + let schedule = lock.withLock { inbox.enqueue(buffer.data) } + if schedule { + Task { @MainActor [weak self] in + guard let self else { return } + let batch = self.lock.withLock { self.inbox.drain() } + self.deliver?(batch.0, batch.1) + } + } + } +} + +/// Value state kept under the receiver's lock; independently exercises admission +/// limits without creating threads, media peers or event-loop timing in tests. +struct ScreenSharingControlInbox { + private var packets: [Data] = [] + private var bytes = 0 + private var scheduled = false + private var failed = false + private var stopped = false + + mutating func enqueue(_ data: Data) -> Bool { + guard !stopped else { return false } + if packets.count >= 256 || data.count > ScreenSharingControlMessage.maximumBytes || bytes + data.count > 64 * 1024 { + failed = true + } else if !failed { + packets.append(data); bytes += data.count + } + guard !scheduled else { return false } + scheduled = true + return true + } + mutating func drain() -> ([Data], Bool) { + let result = (packets, failed) + packets = []; bytes = 0; scheduled = false + return result + } + mutating func stop() { stopped = true; packets = []; bytes = 0 } +} + +public typealias ScreenSharingControlChannel = ScreenSharingDataChannel diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoder.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoder.swift new file mode 100644 index 000000000..e0d379af6 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoder.swift @@ -0,0 +1,260 @@ +import CoreMedia +import Foundation +import VideoToolbox + +/// Decode and stop are serialized by the transport's decoder queue. The VT +/// callback retains only immutable frame metadata and a thread-safe sink. +public final class ScreenSharingDecoder: @unchecked Sendable { + private final class Context { + let frame: ScreenSharingEncodedFrame + let startedNs = ScreenSharingMetrics.nowNs + let output: @Sendable (ScreenSharingVideoFrame) -> Void + let metrics: ScreenSharingMetrics + let pixelFormat: OSType + let refreshSignal: ScreenSharingRefreshSignal? + let refreshGeneration: UInt64? + let sourceTimestampNs: Int64? + let audit: ScreenSharingFrameDeliveryAudit? + let auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? + + init( + _ frame: ScreenSharingEncodedFrame, metrics: ScreenSharingMetrics, pixelFormat: OSType, + refreshSignal: ScreenSharingRefreshSignal?, sourceTimestampNs: Int64?, + audit: ScreenSharingFrameDeliveryAudit?, auditIdentity: ScreenSharingFrameDeliveryAudit.Identity?, + output: @escaping @Sendable (ScreenSharingVideoFrame) -> Void + ) { + self.frame = frame + self.metrics = metrics + self.pixelFormat = pixelFormat + self.output = output + self.refreshSignal = refreshSignal + self.refreshGeneration = frame.isKeyFrame ? refreshSignal?.keyframeGeneration : nil + self.sourceTimestampNs = sourceTimestampNs + self.audit = audit + self.auditIdentity = auditIdentity + } + } + + private var session: VTDecompressionSession? + private var format: CMVideoFormatDescription? + private var parameterSets: [Data] = [] + private let codec: ScreenSharingVideoCodec + private let output: @Sendable (ScreenSharingVideoFrame) -> Void + private var refreshSignal: ScreenSharingRefreshSignal? + public let metrics: ScreenSharingMetrics + /// Receiver-only diagnostic audit; nil = no identity, no attachment, no recording. + private let audit: ScreenSharingFrameDeliveryAudit? + + public init( + metrics: ScreenSharingMetrics, codec: ScreenSharingVideoCodec = .h264, + deliveryAudit: ScreenSharingFrameDeliveryAudit? = nil, + output: @escaping @Sendable (ScreenSharingVideoFrame) -> Void + ) { + self.metrics = metrics + self.codec = codec + self.audit = deliveryAudit + self.output = output + } + + deinit { stop() } + + /// Installed on the serialized decoder queue before accepting frames. + func useRefreshSignal(_ signal: ScreenSharingRefreshSignal) { refreshSignal = signal } + + public func decode( + _ frame: ScreenSharingEncodedFrame, auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? = nil + ) throws { + // VT callbacks only mark the reset. Session teardown belongs on this + // queue, since waiting for callbacks from inside a callback can deadlock. + if refreshSignal?.consumeDecoderReset() == true { stop() } + let received = try NALUnitBitstream.nalUnits(frame.data) + if codec != .h264 { + guard + received.allSatisfy({ $0.count >= 2 && $0[$0.startIndex] & 0x80 == 0 && $0[$0.startIndex + 1] & 7 != 0 }) + else { + throw ScreenSharingError.invalid("Malformed HEVC NAL header.") + } + } + // The host's content marker is read here and never reaches VideoToolbox. + let sourceTimestampNs = ScreenSharingSourceMarker.timestampNs(in: received, codec: codec) + let units = received.filter { !ScreenSharingSourceMarker.isMarker($0, codec: codec) } + func type(_ unit: Data) -> Int { + Int(codec == .h264 ? unit[unit.startIndex] & 0x1f : (unit[unit.startIndex] >> 1) & 0x3f) + } + let types = codec == .h264 ? [7, 8] : [32, 33, 34] + let parameters = types.compactMap { wanted in units.first { type($0) == wanted } } + if parameters.count == types.count, parameterSets != parameters { + try configure(parameters: parameters, auditIdentity: auditIdentity, rtpTimestamp: frame.rtpTimestamp) + } + guard let session, let format else { + throw ScreenSharingError.invalid("Waiting for a \(codec.payloadName) keyframe.") + } + let slices = units.filter { (codec == .h264 ? 1...5 : 0...31).contains(type($0)) } + guard !slices.isEmpty else { return } + let bytes = try NALUnitBitstream.lengthPrefixed(units) + var block: CMBlockBuffer? + try check( + CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: bytes.count, + blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0, + dataLength: bytes.count, flags: 0, blockBufferOut: &block), "Allocate decode buffer") + guard let block else { throw ScreenSharingError.unavailable("No decode buffer.") } + try bytes.withUnsafeBytes { raw in + try check( + CMBlockBufferReplaceDataBytes( + with: raw.baseAddress!, blockBuffer: block, offsetIntoDestination: 0, dataLength: raw.count), + "Copy decode buffer") + } + var sample: CMSampleBuffer? + var timing = CMSampleTimingInfo( + duration: .invalid, presentationTimeStamp: CMTime(value: frame.timestampNs, timescale: 1_000_000_000), + decodeTimeStamp: .invalid) + var size = bytes.count + try check( + CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, dataBuffer: block, formatDescription: format, sampleCount: 1, + sampleTimingEntryCount: 1, sampleTimingArray: &timing, sampleSizeEntryCount: 1, + sampleSizeArray: &size, sampleBufferOut: &sample), "Create decode sample") + guard let sample else { throw ScreenSharingError.unavailable("No decode sample.") } + let context = Unmanaged.passRetained( + Context( + frame, metrics: metrics, pixelFormat: codec.decodedPixelFormat, refreshSignal: refreshSignal, + sourceTimestampNs: sourceTimestampNs, audit: audit, auditIdentity: auditIdentity, output: output)) + let status = VTDecompressionSessionDecodeFrame( + session, sampleBuffer: sample, flags: [], frameRefcon: context.toOpaque(), infoFlagsOut: nil) + if status != noErr { + context.release() + metrics.increment("decodeErrors") + try check(status, "Decode frame") + } + } + + public func stop() { + if let session { + VTDecompressionSessionWaitForAsynchronousFrames(session) + VTDecompressionSessionInvalidate(session) + } + session = nil + format = nil + parameterSets = [] + } + + private func configure( + parameters: [Data], auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? = nil, rtpTimestamp: UInt32 = 0 + ) throws { + var description: CMFormatDescription? + // NSData keeps every parameter's storage stable across the C API call. + let retained = parameters.map { $0 as NSData } + let pointers = retained.map { $0.bytes.assumingMemoryBound(to: UInt8.self) } + let sizes = retained.map(\.length) + let status: OSStatus = withExtendedLifetime(retained) { + if codec == .h264 { + return CMVideoFormatDescriptionCreateFromH264ParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: parameters.count, parameterSetPointers: pointers, parameterSetSizes: sizes, + nalUnitHeaderLength: 4, formatDescriptionOut: &description) + } + return CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: parameters.count, parameterSetPointers: pointers, parameterSetSizes: sizes, + nalUnitHeaderLength: 4, extensions: nil, formatDescriptionOut: &description) + } + try check(status, "Read \(codec.payloadName) format") + guard let description else { throw ScreenSharingError.invalid("Missing \(codec.payloadName) format.") } + if codec != .h264 { + let actual = try ScreenSharingHEVCFormat.read(description) + try actual.validate(for: codec) + metrics.label("decodedChroma", actual.chroma == 3 ? "4:4:4" : "4:2:0") + metrics.label("decodedBitDepth", String(actual.lumaDepth)) + } + let dimensions = CMVideoFormatDescriptionGetDimensions(description) + guard (1...3840).contains(dimensions.width), (1...2160).contains(dimensions.height) else { + throw ScreenSharingError.invalid("Unsupported decoded dimensions.") + } + stop() + // A new VideoToolbox session: recorded against the triggering input's + // identity; that input keeps its epoch, later inputs carry the new one. + audit?.decoderConfigured(auditIdentity, rtpTimestamp: rtpTimestamp) + var callback = VTDecompressionOutputCallbackRecord( + decompressionOutputCallback: { _, sourceContext, status, flags, image, _, _ in + guard let sourceContext else { return } + let context = Unmanaged.fromOpaque(sourceContext).takeRetainedValue() + guard status == noErr, !flags.contains(.frameDropped), let image else { + context.refreshSignal?.request(resetDecoder: true) + context.metrics.increment("decodeErrors"); return + } + guard CVPixelBufferGetPixelFormatType(image) == context.pixelFormat else { + context.metrics.increment("decodeErrors") + context.metrics.label("decoderError", "Hardware decoder returned an unexpected pixel format.") + context.refreshSignal?.request(resetDecoder: true) + return + } + context.metrics.increment("decodedFrames") + context.metrics.observe( + "decode", milliseconds: Double(ScreenSharingMetrics.nowNs - context.startedNs) / 1_000_000) + context.metrics.label("decodedPixelFormat", String(CVPixelBufferGetPixelFormatType(image))) + if let audit = context.audit { + // Diagnostic-only scalar attachment, stamped on EVERY output while the + // audit is on: a nil identity clears the previous value, so a reused + // pool buffer never carries a stale identity to the bridge. + ScreenSharingFrameDeliveryAudit.stamp(context.auditIdentity, on: image) + audit.record(.vtOutput, context.auditIdentity, rtpTimestamp: context.frame.rtpTimestamp) + } + context.output( + ScreenSharingVideoFrame( + pixelBuffer: image, timestampNs: context.frame.timestampNs, rtpTimestamp: context.frame.rtpTimestamp, + sourceTimestampNs: context.sourceTimestampNs)) + if context.frame.isKeyFrame { + let now = ScreenSharingMetrics.nowNs + if context.metrics.increment("decodedKeyFrames") == 1 { + context.metrics.label("firstDecodedKeyframeAtNs", String(now)) + } + context.metrics.label("latestDecodedKeyframeAtNs", String(now)) + if let generation = context.refreshGeneration { + context.refreshSignal?.decodedKeyframe(generation: generation) + } + } + }, decompressionOutputRefCon: nil) + let specification = [kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder: true] as CFDictionary + let attributes: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: codec.decodedPixelFormat, + kCVPixelBufferMetalCompatibilityKey: true, + kCVPixelBufferIOSurfacePropertiesKey: [:], + ] + var created: VTDecompressionSession? + try check( + VTDecompressionSessionCreate( + allocator: kCFAllocatorDefault, formatDescription: description, decoderSpecification: specification, + imageBufferAttributes: attributes as CFDictionary, outputCallback: &callback, + decompressionSessionOut: &created), "Create hardware decoder") + guard let created else { throw ScreenSharingError.unavailable("No hardware \(codec.payloadName) decoder.") } + session = created + do { + try check( + VTSessionSetProperty(created, key: kVTDecompressionPropertyKey_RealTime, value: kCFBooleanTrue), + "Set real-time decoding") + var hardware: Unmanaged? + let hardwareStatus = VTSessionCopyProperty( + created, key: kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder, + allocator: nil, valueOut: &hardware) + let hardwareValue = hardware?.takeRetainedValue() + if hardwareStatus == noErr { + guard hardwareValue as? Bool == true else { + throw ScreenSharingError.unavailable("\(codec.payloadName) decoder is not hardware backed.") + } + } else if hardwareStatus != kVTPropertyNotSupportedErr { + try check(hardwareStatus, "Query decoder hardware") + } + // Some low-latency VT implementations omit this diagnostic property. + // RequireHardwareAcceleratedVideoDecoder still forbids software fallback. + metrics.label("decoderHardware", hardwareStatus == noErr ? "confirmed" : "required; query unsupported") + format = description + parameterSets = parameters + metrics.label("decoder", "VideoToolbox \(codec.rawValue) hardware") + } catch { stop(); throw error } + } + + private func check(_ status: OSStatus, _ operation: String) throws { + if status != noErr { throw ScreenSharingError.codec(operation, status) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoderRecoveryCheck.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoderRecoveryCheck.swift new file mode 100644 index 000000000..5962059fa --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDecoderRecoveryCheck.swift @@ -0,0 +1,49 @@ +import Foundation + +/// Opt-in diagnostic shared by the probe and the transport's decoder queue. +/// No fault is armed in the app. Time is supplied by the caller so the state +/// transitions can be checked without a codec, network, or scheduler. +final class ScreenSharingDecoderRecoveryCheck: @unchecked Sendable { + enum Action: Equatable { + case accept + case reset + case rejectDelta + case recovered(milliseconds: Double) + } + + private let lock = NSLock() + private var remainingFrames: Int? + private var failedAtNs: Int64? + private var onReset: (@Sendable () -> Void)? + + func arm(afterFrames: Int, onReset: (@Sendable () -> Void)? = nil) { + precondition(afterFrames > 0) + lock.withLock { + remainingFrames = afterFrames + failedAtNs = nil + self.onReset = onReset + } + } + + func inspect(keyFrame: Bool, nowNs: Int64) -> Action { + let result: (Action, (@Sendable () -> Void)?) = lock.withLock { + if let failedAtNs { + guard keyFrame else { return (.rejectDelta, nil) } + self.failedAtNs = nil + return (.recovered(milliseconds: Double(max(0, nowNs - failedAtNs)) / 1_000_000), nil) + } + guard let remainingFrames else { return (.accept, nil) } + if remainingFrames > 1 { + self.remainingFrames = remainingFrames - 1 + return (.accept, nil) + } + self.remainingFrames = nil + failedAtNs = nowNs + let callback = onReset + onReset = nil + return (.reset, callback) + } + result.1?() + return result.0 + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDiagnosticProfile.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDiagnosticProfile.swift new file mode 100644 index 000000000..318eff433 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingDiagnosticProfile.swift @@ -0,0 +1,76 @@ +import Foundation + +/// One explicit, immutable, DEFAULT-OFF experimental profile for the product panes. +/// +/// It is enabled only by `CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE=paced15-worker` in the app process +/// environment. An absent or empty variable means OFF. A non-empty unknown value is a configuration FAILURE: it never +/// silently selects the candidate. +/// +/// The name is deliberately "diagnostic". The profile selects receiver playout bounds, a capture-interval request and +/// the renderer preparation path. It selects NO network route, interface or candidate policy, and says nothing about +/// which physical path a pane negotiates or uses. It is unevaluated over relay, and it is NOT inert for a pane that +/// negotiates relay or changes path later. +public struct ScreenSharingDiagnosticProfile: Equatable, Sendable { + public static let environmentKey = "CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE" + public static let paced15WorkerName = "paced15-worker" + + public let name: String + /// Receiver playout bounds forwarded as the pinned M152 `WebRTC-ForcePlayoutDelay` trial. A positive minimum keeps + /// the paced branch of the pinned `timing.cc`; 15 ms is a narrower cap than the 35 ms diagnostic, and neither is a + /// latency guarantee nor an explanation of observed update gaps. + public let playoutDelayMinMs: Int + public let playoutDelayMaxMs: Int + /// SCK minimum-frame-interval request, applied ONLY at adaptive level 0; lower levels request the video rate. + public let captureIntervalFPSAtLevel0: Int + public let renderOnArrival: Bool + public let maximumDrawableCount: Int + public let offMainPreparation: Bool + + /// The one candidate: paced 1/15 playout, capture request 120 at level 0 only, synchronized arrival rendering with + /// two drawables, and the existing off-main preparation path. No unsynced option, codec/GOP/bitrate/pacer change, + /// persisted preference or UI control belongs to it. + public static let paced15Worker = ScreenSharingDiagnosticProfile( + name: paced15WorkerName, playoutDelayMinMs: 1, playoutDelayMaxMs: 15, captureIntervalFPSAtLevel0: 120, + renderOnArrival: true, maximumDrawableCount: 2, offMainPreparation: true) + + /// The profile for THIS process, parsed exactly once — including a configuration failure, which is replayed to every + /// caller instead of being re-derived per connection. The environment is immutable for a running process, so reading + /// it repeatedly could only produce inconsistent answers between roles or connections. + private static let processResult: Result = Result { + try resolve(environment: ProcessInfo.processInfo.environment) + } + + /// The cached process profile; throws the same configuration failure every time when the variable is malformed. + public static func process() throws -> ScreenSharingDiagnosticProfile? { try processResult.get() } + + /// Pure parsing, kept separate so it is testable without touching the process environment. + /// Returns nil when the profile is OFF (variable absent or empty), the profile when it names a known one, and throws + /// for any other non-empty value. Whitespace is not trimmed: an unknown spelling is a failure, not a near-match. + public static func resolve(environment: [String: String]) throws -> ScreenSharingDiagnosticProfile? { + guard let raw = environment[environmentKey], !raw.isEmpty else { return nil } + guard raw == paced15WorkerName else { + throw ScreenSharingError.invalid( + "Unknown \(environmentKey) value \"\(raw)\". Use \"\(paced15WorkerName)\" or unset the variable.") + } + return paced15Worker + } + + /// The capture-interval request for one adaptive level: the override at level 0, nil (= the video rate) below it. + public func captureIntervalFPS(adaptiveLevel: Int) -> Int? { + adaptiveLevel == 0 ? captureIntervalFPSAtLevel0 : nil + } + + /// The exact native trial string this profile installs — the same spelling the probe uses. + public var playoutExperimentLabel: String { + "WebRTC-ForcePlayoutDelay min_ms:\(playoutDelayMinMs),max_ms:\(playoutDelayMaxMs)" + } + + public var fieldTrials: [String: String] { + ["WebRTC-ForcePlayoutDelay": "min_ms:\(playoutDelayMinMs),max_ms:\(playoutDelayMaxMs)"] + } + + /// The process-wide trial selection this profile requires. + public var trialSelection: ScreenSharingFieldTrials.Selection { + .init(name: name, trials: fieldTrials) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoder.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoder.swift new file mode 100644 index 000000000..88bea835a --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoder.swift @@ -0,0 +1,376 @@ +import CoreMedia +import Foundation +import VideoToolbox + +/// WebRTC calls encode/rate/release serially. VT output arrives on its own +/// queue; callback and pending metadata are protected by the lock. Stop drains +/// VT before releasing its bounded callback metadata. +public final class ScreenSharingEncoder: @unchecked Sendable { + private final class Pending: Sendable { + let timestampNs: Int64 + let sourceTimestampNs: Int64 + let rtpTimestamp: UInt32 + let keyframeAttempt: ScreenSharingKeyframeRequest.Attempt? + let startedNs = ScreenSharingMetrics.nowNs + init(frame: ScreenSharingVideoFrame, keyframeAttempt: ScreenSharingKeyframeRequest.Attempt?) { + timestampNs = frame.timestampNs + // Direct codec users (standalone probes) identify content by their own + // input timestamp; the transport adapter always supplies buffer identity. + sourceTimestampNs = frame.sourceTimestampNs ?? frame.timestampNs + rtpTimestamp = frame.rtpTimestamp + self.keyframeAttempt = keyframeAttempt + } + } + + private let lock = NSLock() + private var pending: [Int64: Pending] = [:] + private var pendingHighWater = 0 + private var keyframeRequest = ScreenSharingKeyframeRequest() + private var dropCheck: ScreenSharingEncoderDropCheck? + private var session: VTCompressionSession? + private var output: (@Sendable (ScreenSharingEncodedFrame) -> Void)? + private let configuration: ScreenSharingVideoConfiguration + private let codec: ScreenSharingVideoCodec + private let maximumPendingFrames: Int + private let completeEachFrame: Bool + private var currentBitrate: Int + private var currentFPS: Int + public let metrics: ScreenSharingMetrics + + public init( + configuration: ScreenSharingVideoConfiguration, metrics: ScreenSharingMetrics, + useLowLatencyRateControl: Bool = true, codec: ScreenSharingVideoCodec = .h264, + disableLookAhead: Bool = false, maximumPendingFrames: Int = 2, completeEachFrame: Bool = false, + prioritizeSpeed: Bool = false, keyframeIntervalSeconds: Int = 2 + ) throws { + self.configuration = configuration + currentBitrate = configuration.bitrate + currentFPS = configuration.framesPerSecond + self.metrics = metrics + self.codec = codec + guard (1...8).contains(maximumPendingFrames) else { + throw ScreenSharingError.invalid("Encoder admission must be bounded to 1...8 frames.") + } + guard (1...60).contains(keyframeIntervalSeconds) else { + throw ScreenSharingError.invalid("Keyframe interval must be bounded to 1...60 seconds.") + } + self.maximumPendingFrames = maximumPendingFrames + self.completeEachFrame = completeEachFrame + metrics.label("encoderCompletion", completeEachFrame ? "synchronous drain experiment" : "callback") + metrics.label("encoderMaximumPendingFrames", String(maximumPendingFrames)) + metrics.label("encoderInitialBitrate", String(configuration.bitrate)) + metrics.label("encoderInitialFPS", String(configuration.framesPerSecond)) + metrics.label("encoderKeyframeIntervalSeconds", String(keyframeIntervalSeconds)) + guard codec != .hevc444 || !useLowLatencyRateControl else { + throw ScreenSharingError.invalid( + "Main444 requires standard rate control; the low-latency encoder can reduce chroma.") + } + var created: VTCompressionSession? + var specification: [CFString: Any] = [ + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: true + ] + if useLowLatencyRateControl { specification[kVTVideoEncoderSpecification_EnableLowLatencyRateControl] = true } + try check( + VTCompressionSessionCreate( + allocator: kCFAllocatorDefault, width: Int32(configuration.width), height: Int32(configuration.height), + codecType: codec.mediaType, encoderSpecification: specification as CFDictionary, + imageBufferAttributes: nil, compressedDataAllocator: nil, + outputCallback: nil, refcon: nil, compressionSessionOut: &created), "Create encoder") + guard let created else { throw ScreenSharingError.unavailable("No hardware \(codec.payloadName) encoder.") } + session = created + do { + try property(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue) + try property(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse) + if prioritizeSpeed { + try property(kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, kCFBooleanTrue) + } + metrics.label("encoderSpeedPolicy", prioritizeSpeed ? "prioritize speed" : "encoder default") + metrics.label("encoderRateControl", useLowLatencyRateControl ? "low latency" : "standard") + if codec == .hevc444 { + var supported: CFDictionary? + try check( + VTSessionCopySupportedPropertyDictionary(created, supportedPropertyDictionaryOut: &supported), + "Read encoder properties") + let profileInfo = + (supported as? [String: Any])?[kVTCompressionPropertyKey_ProfileLevel as String] as? [String: Any] + let profiles = profileInfo?[kVTPropertySupportedValueListKey as String] as? [String] ?? [] + guard let profile = profiles.first(where: { $0.contains("_Main444_") }) else { + throw ScreenSharingError.unavailable("Hardware encoder does not advertise an 8-bit Main444 profile.") + } + try property(kVTCompressionPropertyKey_ProfileLevel, profile as CFString) + metrics.label("encoderProfile", profile) + } else { + try property( + kVTCompressionPropertyKey_ProfileLevel, + codec == .h264 ? kVTProfileLevel_H264_Baseline_AutoLevel : kVTProfileLevel_HEVC_Main_AutoLevel) + } + try property(kVTCompressionPropertyKey_ExpectedFrameRate, configuration.framesPerSecond as CFNumber) + if disableLookAhead { + guard !useLowLatencyRateControl else { + throw ScreenSharingError.invalid("Lookahead is controlled by the low-latency encoder.") + } + #if os(macOS) + try property(kVTCompressionPropertyKey_SuggestedLookAheadFrameCount, 0 as CFNumber) + metrics.label("encoderSuggestedLookAheadFrames", "0") + #else + throw ScreenSharingError.unavailable("The lookahead experiment requires macOS.") + #endif + } + try property(kVTCompressionPropertyKey_AverageBitRate, configuration.bitrate as CFNumber) + try property( + kVTCompressionPropertyKey_MaxKeyFrameInterval, + (configuration.framesPerSecond * keyframeIntervalSeconds) as CFNumber) + try property(kVTCompressionPropertyKey_ColorPrimaries, kCVImageBufferColorPrimaries_ITU_R_709_2) + try property(kVTCompressionPropertyKey_TransferFunction, kCVImageBufferTransferFunction_ITU_R_709_2) + try property(kVTCompressionPropertyKey_YCbCrMatrix, kCVImageBufferYCbCrMatrix_ITU_R_709_2) + try check(VTCompressionSessionPrepareToEncodeFrames(created), "Prepare encoder") + var hardware: Unmanaged? + let hardwareStatus = VTSessionCopyProperty( + created, key: kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, + allocator: nil, valueOut: &hardware) + let hardwareValue = hardware?.takeRetainedValue() + if hardwareStatus == noErr { + guard hardwareValue as? Bool == true else { + throw ScreenSharingError.unavailable("\(codec.payloadName) encoder is not hardware backed.") + } + } else if hardwareStatus != kVTPropertyNotSupportedErr { + try check(hardwareStatus, "Query encoder hardware") + } + // Some low-latency VT implementations omit this diagnostic property. + // RequireHardwareAcceleratedVideoEncoder still forbids software fallback. + metrics.label("encoderHardware", hardwareStatus == noErr ? "confirmed" : "required; query unsupported") + metrics.label("encoder", "VideoToolbox \(codec.rawValue) hardware, no frame reordering") + metrics.label("encodedColor", "BT.709 SDR") + } catch { stop(); throw error } + } + + deinit { stop() } + + public func onFrame(_ callback: @escaping @Sendable (ScreenSharingEncodedFrame) -> Void) { + lock.withLock { output = callback } + } + + func useDropCheck(_ check: ScreenSharingEncoderDropCheck) { + lock.withLock { dropCheck = check } + } + + @discardableResult + public func encode(_ frame: ScreenSharingVideoFrame, forceKeyFrame: Bool = false) throws -> Bool { + metrics.increment("encoderInputFrames") + metrics.event("encoderInputInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + metrics.event("encoderInputTimestampInterval", atNanoseconds: frame.timestampNs) + guard let session else { throw ScreenSharingError.unavailable("Encoder stopped.") } + guard frame.timestampNs >= 0, + CVPixelBufferGetWidth(frame.pixelBuffer) == configuration.width, + CVPixelBufferGetHeight(frame.pixelBuffer) == configuration.height + else { throw ScreenSharingError.invalid("Encoder input size or timestamp does not match its configuration.") } + if forceKeyFrame { metrics.increment("encoderKeyframeRequests") } + let metadata: Pending? = lock.withLock { + if forceKeyFrame { keyframeRequest.request() } + guard pending.count < maximumPendingFrames, pending[frame.timestampNs] == nil else { + if keyframeRequest.isPending { metrics.increment("encoderDeferredKeyframeRequests") } + return nil + } + let metadata = Pending(frame: frame, keyframeAttempt: keyframeRequest.beginAttempt()) + pending[frame.timestampNs] = metadata + if pending.count > pendingHighWater { + pendingHighWater = pending.count + metrics.label("encoderPendingHighWater", String(pendingHighWater)) + } + return metadata + } + guard let metadata else { + metrics.increment("encoderBackpressureDrops") + metrics.trace("encoderAdmission", "\(ScreenSharingMetrics.nowNs) ts=\(frame.timestampNs) dropped=backpressure") + return false + } + if metadata.keyframeAttempt != nil { metrics.increment("encoderForcedKeyframesSubmitted") } + metrics.trace( + "encoderAdmission", + "\(ScreenSharingMetrics.nowNs) ts=\(frame.timestampNs) id=\(metadata.sourceTimestampNs) force=\(forceKeyFrame) attempt=\(metadata.keyframeAttempt != nil)" + ) + let submitted = ScreenSharingMetrics.nowNs + let status = VTCompressionSessionEncodeFrame( + session, imageBuffer: frame.pixelBuffer, + presentationTimeStamp: CMTime(value: frame.timestampNs, timescale: 1_000_000_000), + duration: CMTime(value: 1, timescale: Int32(configuration.framesPerSecond)), + frameProperties: metadata.keyframeAttempt != nil + ? [kVTEncodeFrameOptionKey_ForceKeyFrame: true] as CFDictionary : nil, + infoFlagsOut: nil + ) { [weak self, metadata] status, flags, buffer in + self?.didEncode(metadata: metadata, status: status, flags: flags, buffer: buffer) + } + metrics.observe("encodeSubmission", milliseconds: Double(ScreenSharingMetrics.nowNs - submitted) / 1_000_000) + if status != noErr { + _ = lock.withLock { pending.removeValue(forKey: frame.timestampNs) } + finishKeyframeAttempt(metadata, producedKeyFrame: false) + try check(status, "Encode frame") + } + if completeEachFrame { + let started = ScreenSharingMetrics.nowNs + try check( + VTCompressionSessionCompleteFrames( + session, untilPresentationTimeStamp: CMTime(value: frame.timestampNs, timescale: 1_000_000_000)), + "Complete encoded frame") + metrics.observe("encoderDrain", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + } + return true + } + + public func setBitrate(_ bitsPerSecond: Int, framesPerSecond: Int) throws { + guard session != nil else { throw ScreenSharingError.unavailable("Encoder stopped.") } + guard bitsPerSecond > 0, (1...60).contains(framesPerSecond) else { + throw ScreenSharingError.invalid("Invalid encoder rate.") + } + let started = ScreenSharingMetrics.nowNs + metrics.increment("encoderRateUpdateRequests") + if currentBitrate != bitsPerSecond { + try property(kVTCompressionPropertyKey_AverageBitRate, bitsPerSecond as CFNumber) + currentBitrate = bitsPerSecond + metrics.increment("encoderBitratePropertyUpdates") + } + if currentFPS != framesPerSecond { + try property(kVTCompressionPropertyKey_ExpectedFrameRate, framesPerSecond as CFNumber) + currentFPS = framesPerSecond + metrics.increment("encoderFPSPropertyUpdates") + } + metrics.observe("encoderRateUpdate", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + metrics.label("encoderRequestedBitrate", String(bitsPerSecond)) + metrics.label("encoderRequestedFPS", String(framesPerSecond)) + } + + public func stop() { + guard let active = session else { return } + session = nil + VTCompressionSessionCompleteFrames(active, untilPresentationTimeStamp: .invalid) + VTCompressionSessionInvalidate(active) + lock.withLock { + metrics.label("encoderPendingAtStop", String(pending.count)) + pending.removeAll(); output = nil + keyframeRequest = ScreenSharingKeyframeRequest() + } + } + + /// Frames admitted and not yet completed by VideoToolbox (diagnostic). + public var pendingCount: Int { lock.withLock { pending.count } } + + private func didEncode(metadata: Pending, status: OSStatus, flags: VTEncodeInfoFlags, buffer: CMSampleBuffer?) { + var producedKeyFrame = false + defer { finishKeyframeAttempt(metadata, producedKeyFrame: producedKeyFrame) } + let callbackStarted = ScreenSharingMetrics.nowNs + metrics.observe("videoToolboxCompletion", milliseconds: Double(callbackStarted - metadata.startedNs) / 1_000_000) + metrics.event("encoderCallbackInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + _ = lock.withLock { pending.removeValue(forKey: metadata.timestampNs) } + if flags.contains(.frameDropped), status == noErr { + metrics.increment("encoderDroppedFrames") + metrics.trace("encoderOutput", "\(callbackStarted) ts=\(metadata.timestampNs) result=vtDropped") + return + } + guard status == noErr, let buffer, CMSampleBufferDataIsReady(buffer) else { + metrics.increment("encodeErrors") + metrics.label("lastEncodeError", "VideoToolbox callback status: \(status)") + return + } + if metadata.keyframeAttempt != nil, lock.withLock({ dropCheck })?.consume() == true { + // Discard a real forced output at the same completion boundary where VT + // can report a drop. Keep injected loss distinct from hardware drop counts. + metrics.increment("injectedEncoderKeyframeDrops") + metrics.trace("encoderOutput", "\(callbackStarted) ts=\(metadata.timestampNs) result=discarded") + return + } + let timestamp = metadata.timestampNs + do { + let attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, createIfNecessary: false) as? [[CFString: Any]] + let keyFrame = attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool != true + guard let block = buffer.dataBuffer, let format = buffer.formatDescription else { + throw ScreenSharingError.invalid("Encoded frame has no data.") + } + let byteCount = CMBlockBufferGetDataLength(block) + guard (1...NALUnitBitstream.maximumFrameBytes).contains(byteCount) else { + throw ScreenSharingError.invalid("Invalid encoded frame size.") + } + var bytes = Data(count: byteCount) + let copyStatus = bytes.withUnsafeMutableBytes { raw in + CMBlockBufferCopyDataBytes(block, atOffset: 0, dataLength: raw.count, destination: raw.baseAddress!) + } + try check(copyStatus, "Read encoded frame") + var data = Data() + var lengthSize: Int32 = 0 + var parameterCount = 0 + try check( + parameterSet(format, index: 0, pointer: nil, size: nil, count: ¶meterCount, length: &lengthSize), + "Read \(codec.payloadName) configuration") + if codec != .h264 { + let actual = try ScreenSharingHEVCFormat.read(format) + try actual.validate(for: codec) + metrics.label("encodedHEVCProfileIDC", String(actual.profile)) + metrics.label("encodedChroma", actual.chroma == 3 ? "4:4:4" : "4:2:0") + metrics.label("encodedBitDepth", String(actual.lumaDepth)) + } + if keyFrame { + for index in 0..? + var length = 0 + try check( + parameterSet(format, index: index, pointer: &pointer, size: &length, count: nil, length: nil), + "Read \(codec.payloadName) parameter set") + if let pointer { data.append(contentsOf: [0, 0, 0, 1]); data.append(pointer, count: length) } + } + } + // Content identity for the viewer's idle delivery audit; precedes the slices. + data.append(contentsOf: [0, 0, 0, 1]) + data.append(ScreenSharingSourceMarker.nalUnit(timestampNs: metadata.sourceTimestampNs, codec: codec)) + data.append(try NALUnitBitstream.annexB(bytes, lengthSize: Int(lengthSize))) + metrics.increment("encodedFrames") + metrics.increment("encodedBytes", by: data.count) + metrics.increment(keyFrame ? "encodedKeyFrames" : "encodedDeltaFrames") + metrics.increment(keyFrame ? "encodedKeyFrameBytes" : "encodedDeltaFrameBytes", by: data.count) + metrics.observe("encode", milliseconds: Double(ScreenSharingMetrics.nowNs - metadata.startedNs) / 1_000_000) + metrics.observe( + "encodedOutputPreparation", milliseconds: Double(ScreenSharingMetrics.nowNs - callbackStarted) / 1_000_000) + let callback = lock.withLock { output } + callback?( + ScreenSharingEncodedFrame( + data: data, timestampNs: timestamp, rtpTimestamp: metadata.rtpTimestamp, + width: configuration.width, height: configuration.height, isKeyFrame: keyFrame, + sourceTimestampNs: metadata.sourceTimestampNs)) + producedKeyFrame = keyFrame && callback != nil + metrics.trace( + "encoderOutput", + "\(callbackStarted) ts=\(timestamp) id=\(metadata.sourceTimestampNs) key=\(keyFrame) delivered=\(callback != nil)" + ) + } catch { metrics.increment("encodeErrors"); metrics.label("lastEncodeError", error.localizedDescription) } + } + + private func finishKeyframeAttempt(_ metadata: Pending, producedKeyFrame: Bool) { + let retry = lock.withLock { + keyframeRequest.complete(metadata.keyframeAttempt, producedKeyFrame: producedKeyFrame) + } + if retry { metrics.increment("encoderRetriedKeyframes") } + } + + private func parameterSet( + _ format: CMFormatDescription, index: Int, pointer: UnsafeMutablePointer?>?, + size: UnsafeMutablePointer?, count: UnsafeMutablePointer?, length: UnsafeMutablePointer? + ) -> OSStatus { + if codec == .h264 { + return CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, parameterSetIndex: index, + parameterSetPointerOut: pointer, parameterSetSizeOut: size, parameterSetCountOut: count, + nalUnitHeaderLengthOut: length) + } + return CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + format, parameterSetIndex: index, + parameterSetPointerOut: pointer, parameterSetSizeOut: size, parameterSetCountOut: count, + nalUnitHeaderLengthOut: length) + } + + private func property(_ key: CFString, _ value: CFTypeRef) throws { + guard let session else { throw ScreenSharingError.unavailable("Encoder stopped.") } + try check(VTSessionSetProperty(session, key: key, value: value), "Set encoder \(key)") + } + + private func check(_ status: OSStatus, _ operation: String) throws { + if status != noErr { throw ScreenSharingError.codec(operation, status) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropCheck.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropCheck.swift new file mode 100644 index 000000000..aadc0b00c --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropCheck.swift @@ -0,0 +1,17 @@ +import Foundation + +/// Single-use probe fault, armed at the decoder reset edge. The app never arms it. +final class ScreenSharingEncoderDropCheck: @unchecked Sendable { + private let lock = NSLock() + private var armed = false + + func arm() { lock.withLock { armed = true } } + + func consume() -> Bool { + lock.withLock { + guard armed else { return false } + armed = false + return true + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropLog.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropLog.swift new file mode 100644 index 000000000..c85bf454a --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingEncoderDropLog.swift @@ -0,0 +1,45 @@ +import Foundation + +/// The pinned M152 encoder emits these aggregate counts periodically. Retain +/// only the three numeric counters; WebRTC's other logs can contain addresses. +public enum ScreenSharingEncoderDropLog { + public static func counters(in message: String) -> [String: Int] { + let prefix = "Number of frames: captured " + guard let start = message.range(of: prefix) else { return [:] } + let pieces = message[start.upperBound...].components(separatedBy: ", ") + guard pieces.count == 4, + pieces[1].hasPrefix("dropped (due to congestion window pushback) "), + pieces[2].hasPrefix("dropped (due to encoder blocked) "), + pieces[3].hasPrefix("interval_ms "), + let captured = Int(pieces[0]), + let congestion = pieces[1].split(separator: " ").last.flatMap({ Int($0) }), + let blocked = pieces[2].split(separator: " ").last.flatMap({ Int($0) }), + captured >= 0, congestion >= 0, blocked >= 0 + else { return [:] } + return ["rtcLoggedInputFrames": captured, "rtcCongestionWindowDrops": congestion, "rtcEncoderQueueDrops": blocked] + } + + /// Pinned M152 encoder-side drop or pause messages, with hexadecimal + /// addresses removed and length bounded; anything else yields nil. + public static func dropDiagnostic(in message: String) -> String? { + let markers = [ + "Same/old NTP timestamp", "encoder is blocked", "Too large for target bitrate", "Drop Frame:", + "encoder paused", "Video suspended", "Dropping frame", + ] + guard markers.contains(where: { message.contains($0) }) else { return nil } + var sanitized = "" + var index = message.startIndex + while index < message.endIndex { + if message[index...].hasPrefix("0x") { + var end = message.index(index, offsetBy: 2) + while end < message.endIndex, message[end].isHexDigit { end = message.index(after: end) } + sanitized += "0x…" + index = end + } else { + sanitized.append(message[index]) + index = message.index(after: index) + } + } + return String(sanitized.trimmingCharacters(in: .whitespacesAndNewlines).prefix(160)) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFieldTrials.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFieldTrials.swift new file mode 100644 index 000000000..5ae448a40 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFieldTrials.swift @@ -0,0 +1,162 @@ +import Foundation +import WebRTC + +/// The single process boundary for WebRTC field-trial initialization. +/// +/// M152 exposes these experiments through PROCESS-WIDE trials that must be installed before any RTC object exists. +/// Pinned source fact (`sdk/objc/api/peerconnection/RTCFieldTrials.mm` at 6f37672d): `RTCInitFieldTrialDictionary` +/// flattens the dictionary into a C string and calls `webrtc::DeprecatedGlobalFieldTrials::Set`. It returns `void`, +/// signals no failure, and a later call simply overwrites the global string — objects already built keep the value +/// they read. Replacement is therefore never safe, and this type refuses it rather than relying on the native call: +/// +/// * installation is SINGLE-FLIGHT and readiness is published only after the native apply has RETURNED — a caller that +/// arrives while another thread is applying waits, so no one can reach a factory on the strength of a selection that +/// has not been applied yet; +/// * the first applied selection fixes the process, including the default (empty) one — a later non-empty request +/// cannot silently replace it, it fails; +/// * a request whose TRIAL MAP equals what is installed is idempotent regardless of its provenance name, and the first +/// installer's provenance is preserved; names never create a false native conflict; +/// * a request whose trial map differs throws BEFORE any peer, factory or codec is created. +/// +/// What it does NOT do: it cannot detect WebRTC calls made through paths that bypass it. Every entry point known in +/// this repository is wired through `process` (peer construction, the probe's option-derived trials, the product host +/// and viewer, and the bare-factory test helper); that is a wiring guarantee, not runtime interception of arbitrary +/// third-party RTC use. +/// +/// Contract for the apply closure: it must not re-enter this instance (the real one calls +/// `RTCInitFieldTrialDictionary` and returns). +public final class ScreenSharingFieldTrials: @unchecked Sendable { + public struct Selection: Equatable, Sendable { + /// Human-readable provenance of the selection ("default" for the empty one). Provenance is descriptive only: it + /// never takes part in conflict detection. + public let name: String + public let trials: [String: String] + public init(name: String, trials: [String: String]) { self.name = name; self.trials = trials } + public static let `default` = Selection(name: "default", trials: [:]) + /// The exact native playout string, when this selection installs one. + public var playoutExperimentLabel: String? { + trials["WebRTC-ForcePlayoutDelay"].map { "WebRTC-ForcePlayoutDelay \($0)" } + } + /// Two selections are compatible when they ask the native library for the SAME dictionary. + public func installs(_ other: Selection) -> Bool { trials == other.trials } + + /// The selection the standalone probe derives from its options. It lives here, beside the boundary that installs + /// it, so the probe's real path is covered by the library tests. Every existing combination is preserved: + /// * `jitterWindowFrames` adds the jitter-estimator trial; + /// * `lowLatencyPlayout` selects `min_ms:0,max_ms:0` (pinned M152 rtp_video_stream_receiver2.cc and timing.cc: + /// zero minimum/maximum selects ASAP rendering — not a latency guarantee); + /// * `playoutDelayBoundsMs` selects those exact bounds (pinned timing.cc uses ASAP only for min 0 and max <= 500, + /// so a positive minimum keeps paced rendering — an experiment, not a guarantee). + /// The two playout options are mutually exclusive at parse time, so their ordering here is unobservable. + public static func probeOptions( + jitterWindowFrames: Int?, lowLatencyPlayout: Bool, playoutDelayBoundsMs: (min: Int, max: Int)? + ) -> Selection { + var trials: [String: String] = [:] + if let window = jitterWindowFrames { + trials["WebRTC-JitterEstimatorConfig"] = "max_frame_size_percentile:0.95,frame_size_window:\(window)" + } + if lowLatencyPlayout { trials["WebRTC-ForcePlayoutDelay"] = "min_ms:0,max_ms:0" } + if let bounds = playoutDelayBoundsMs { + trials["WebRTC-ForcePlayoutDelay"] = "min_ms:\(bounds.min),max_ms:\(bounds.max)" + } + return Selection(name: trials.isEmpty ? "default" : "probe options", trials: trials) + } + } + + private enum State { + case none + /// A thread is inside the native apply. Readiness is deliberately NOT published in this state. + case applying + case installed(Selection) + } + + /// Pure seam: the real process installs the native dictionary, tests inject a recorder and never mutate the + /// process-global trials. + private let apply: @Sendable ([String: String]) -> Void + private let condition = NSCondition() + private var state: State = .none + /// Internal observer fired immediately before a caller blocks on the apply in progress. It exists so a test can + /// acknowledge that a second caller actually REACHED the waiting boundary before the apply is released; production + /// never sets it. + var onWaitingForApply: (@Sendable () -> Void)? + + public init(apply: @escaping @Sendable ([String: String]) -> Void) { self.apply = apply } + + /// The real production boundary. The native call is skipped for an empty dictionary — installing "default" only + /// fixes this process's selection, exactly as the probe behaved before this type existed. + public static let process = ScreenSharingFieldTrials(apply: { trials in + if !trials.isEmpty { RTCInitFieldTrialDictionary(trials) } + }) + + /// The selection in force, or nil while nothing has been applied yet. A selection appears here only once its native + /// apply has returned. + public var installed: Selection? { + condition.lock(); defer { condition.unlock() } + if case .installed(let selection) = state { return selection } + return nil + } + + /// Installs `selection` if nothing is installed yet, waits if another thread is applying, and returns the selection + /// now in force. Throws when a selection with a DIFFERENT trial map is already installed: trials are + /// process-immutable in this design, so the caller must never be told that its request is active when it is not. + @discardableResult + public func install(_ selection: Selection) throws -> Selection { + // Apply-or-wait first, then judge: the returned value is always a selection that HAS been applied. + let inForce = applyOrWaitForFirst(selection) + guard inForce.installs(selection) else { + throw ScreenSharingError.invalid( + "WebRTC field trials are already initialized for \"\(inForce.name)\" in this process and cannot be " + + "replaced with \"\(selection.name)\". Quit and start a fresh app process to change the selection.") + } + return inForce + } + + /// Pins the default selection when nothing has been applied yet, and returns whatever is in force. Used by peer + /// construction so a peer built without a profile can never be overtaken by a later profile request. It cannot fail + /// and it never invents a result: the value returned is the selection that was actually applied in this process. + @discardableResult + public func ensureInstalled() -> Selection { applyOrWaitForFirst(.default) } + + /// Installs the selection required by `profile` (or the default when the profile is OFF) and returns what is in + /// force. A conflict throws here, before any peer exists. + @discardableResult + public func install(profile: ScreenSharingDiagnosticProfile?) throws -> Selection { + try install(profile?.trialSelection ?? .default) + } + + /// Applies `requested` when this process has applied nothing yet, waits when another thread is mid-apply, and in + /// every case returns the selection that has ACTUALLY been applied. Never throws: the native call is `void` and + /// reports no failure, so there is nothing here to fail on and nothing to invent. + private func applyOrWaitForFirst(_ requested: Selection) -> Selection { + condition.lock() + while true { + switch state { + case .none: + // Single flight: this thread owns the apply, and readiness stays unpublished until it returns. + state = .applying + condition.unlock() + apply(requested.trials) + condition.lock() + state = .installed(requested) + condition.broadcast() + condition.unlock() + return requested + case .applying: + // Another thread is mid-apply; wait for it rather than proceeding on an unapplied selection. + let observer = onWaitingForApply + if let observer { + condition.unlock() + observer() + condition.lock() + // State may have advanced while the observer ran; re-evaluate rather than waiting on a finished apply. + if case .applying = state {} else { continue } + } + condition.wait() + case .installed(let existing): + // First provenance wins; an identical map asked for under another name is simply already satisfied. + condition.unlock() + return existing + } + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameDeliveryAudit.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameDeliveryAudit.swift new file mode 100644 index 000000000..b1e716c9a --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameDeliveryAudit.swift @@ -0,0 +1,354 @@ +import CoreVideo +import Foundation +import QuartzCore + +/// Receiver-only, opt-in frame-delivery audit (diagnostic). Records scalar +/// boundary events for decoded frames on ONE receiver-local clock: +/// `CACurrentMediaTime()` in nanoseconds — the clock of the viewer's media +/// origin, of `receivedAtSeconds`, of Metal's `presentedTime` and of the +/// observer alignment. Identity is a monotonic receiver-side decoder-input +/// sequence (never restarted) plus the RTP timestamp; the identity's +/// `generation` is the decoder INPUT EPOCH (VideoToolbox sessions created +/// before that input); session creation itself is a recorded event. Records hold scalars only: no buffer, texture or object is +/// retained. Disabled (nil) means nothing is allocated and every call site is +/// a single nil branch. Earlier stages (capture, encode, network, jitter +/// buffer before decoder input) and physical scanout are outside this audit. +public final class ScreenSharingFrameDeliveryAudit: @unchecked Sendable { + public enum Stage: UInt8, CaseIterable, Sendable { + /// WebRTC handed a reassembled/released encoded frame to the decoder. + case decoderInput = 1 + /// VideoToolbox output callback delivered the decoded pixel buffer. + case vtOutput = 2 + /// WebRTC's native renderer callback delivered the frame to the mailbox. + case rtcRendererCallback = 3 + /// The coordinator selected the frame (new frames only; cached redraws are not deliveries). + case selection = 4 + /// Drawable acquisition began (main-actor `currentDrawable` or worker `nextDrawable`). + case acquisitionBegin = 5 + /// Drawable acquisition returned (value = 1 acquired, 0 none). + case acquisitionEnd = 6 + /// Off-main preparation started on the worker (textures + size + acquisition + encoding). + case preparationBegin = 7 + /// The prepared result reached the main actor (`finishPreparation`). + case preparedReceipt = 8 + /// Final submission (commit) after the coordinator's terminal/slot guard. + case submission = 9 + /// The command buffer's completion CALLBACK ran (value = 1 completed, 0 error). + case gpuCompletionCallback = 10 + /// The presented handler ran: `atNs` is the callback time, `valueNs` is + /// Metal's `presentedTime` converted to nanoseconds (0 = unpresented). + case presentedResult = 11 + /// A VideoToolbox session was created for the frame carrying this identity + /// (value = the new generation); later inputs carry that epoch. + case decoderConfigured = 12 + /// The mailbox replaced this (never selected) frame with a newer one — + /// the newest-frame policy, recorded so downstream absence is explained. + case mailboxReplaced = 13 + + public var name: String { + switch self { + case .decoderInput: "decoderInput" + case .vtOutput: "vtOutput" + case .rtcRendererCallback: "rtcRendererCallback" + case .selection: "selection" + case .acquisitionBegin: "acquisitionBegin" + case .acquisitionEnd: "acquisitionEnd" + case .preparationBegin: "preparationBegin" + case .preparedReceipt: "preparedReceipt" + case .submission: "submission" + case .gpuCompletionCallback: "gpuCompletionCallback" + case .presentedResult: "presentedResult" + case .decoderConfigured: "decoderConfigured" + case .mailboxReplaced: "mailboxReplaced" + } + } + } + + /// Decoder-input identity: the sequence never restarts within one audit; the + /// generation is the input epoch = VideoToolbox sessions created before the + /// input (the frame that triggers a session keeps the epoch it was input in; + /// the creation is its own `decoderConfigured` event). + public struct Identity: Equatable, Hashable, Sendable { + public let sequence: UInt32 + public let generation: UInt16 + public init(sequence: UInt32, generation: UInt16) { + self.sequence = sequence + self.generation = generation + } + var packed: UInt64 { UInt64(generation) << 32 | UInt64(sequence) } + init?(packed: UInt64) { + let sequence = UInt32(truncatingIfNeeded: packed) + guard sequence != 0 else { return nil } + self.init(sequence: sequence, generation: UInt16(truncatingIfNeeded: packed >> 32)) + } + } + + /// Recording window relative to the receiver-local origin (media start). + public struct Window: Equatable, Sendable { + public let beginSeconds: Double + public let durationSeconds: Double + public init(beginSeconds: Double, durationSeconds: Double) throws { + guard beginSeconds.isFinite, durationSeconds.isFinite, beginSeconds >= 0, durationSeconds > 0 else { + throw ScreenSharingError.invalid("Delivery audit window needs begin >= 0 and a positive duration.") + } + self.beginSeconds = beginSeconds + self.durationSeconds = durationSeconds + } + } + + /// 32-byte scalar record (stride); nothing is retained. + struct Record { + let stage: UInt8 + let generation: UInt16 + let sequence: UInt32 + let rtpTimestamp: UInt32 + let atNs: Int64 + let valueNs: Int64 + } + + public struct Snapshot: Encodable, Sendable { + public let kind: String + public let clock: String + public let originNs: Int64? + public let windowBeginSeconds: Double + public let windowDurationSeconds: Double + public let windowNs: [Int64]? + public let capacity: Int + public let recordByteStride: Int + public let preallocatedRecordBytes: Int + public let seenDictionaryEntries: Int + public let recorded: Int + public let overflow: Int + public let beforeOrigin: Int + public let outsideWindowBefore: Int + public let outsideWindowAfter: Int + public let lateAfterClose: Int + public let missingIdentityEvents: Int + public let duplicateEvents: Int + public let closedAtNs: Int64? + public let firstEventNs: Int64? + public let lastEventNs: Int64? + public let decoderGenerations: Int + public let sequenceRange: [UInt32]? + public let perStage: [String: Int] + /// Events counted but never recorded: these make a trace incomplete. + public let truncation: [String: Int] + /// Frames reaching each boundary, and downstream absences (which may be + /// legitimate newest-frame replacement, a failed preparation, or window + /// clipping — not by themselves an incomplete trace). + public let coverage: [String: Int] + public let stages: [UInt8] + public let generations: [UInt16] + public let sequences: [UInt32] + public let rtpTimestamps: [UInt32] + public let atNs: [Int64] + public let valueNs: [Int64] + public let notes: [String] + } + + public static let defaultCapacity = 65536 + public static let recordByteStride = MemoryLayout.stride + /// The one supported clock: CACurrentMediaTime in nanoseconds. + public static func defaultClock() -> Int64 { Int64(CACurrentMediaTime() * 1_000_000_000) } + static let attachmentKey = "com.codevisor.screenSharing.deliveryAuditIdentity" + + public let window: Window + public let capacity: Int + private let clock: @Sendable () -> Int64 + private let lock = NSLock() + private var records: [Record] + private var seen: [UInt32: UInt16] = [:] // stage bitmask per sequence (bounded by capacity) + private var nextSequence: UInt32 = 1 + private var generation: UInt16 = 0 + private var originNs: Int64? + private var closedAtNs: Int64? + private var overflow = 0 + private var beforeOrigin = 0 + private var outsideBefore = 0 + private var outsideAfter = 0 + private var late = 0 + private var missingIdentity = 0 + private var duplicates = 0 + private var firstEventNs: Int64? + private var lastEventNs: Int64? + + public init( + window: Window, capacity: Int = ScreenSharingFrameDeliveryAudit.defaultCapacity, + clock: @escaping @Sendable () -> Int64 = { ScreenSharingFrameDeliveryAudit.defaultClock() } + ) { + self.window = window + self.capacity = max(1, capacity) + self.clock = clock + records = [] + records.reserveCapacity(self.capacity) + seen.reserveCapacity(min(self.capacity, 8192)) + } + + /// Media start on the receiver-local clock. Events before it are counted, not recorded. + public func start(originNs: Int64) { + lock.withLock { if self.originNs == nil { self.originNs = originNs } } + } + + /// The audit's clock, for explicit stamps taken around a boundary. + public func now() -> Int64 { clock() } + + /// A VideoToolbox session was created while decoding `identity` (nil when + /// the triggering input carried none); sequences continue, the epoch of + /// LATER inputs increments, and the creation is recorded as an event. + @discardableResult + public func decoderConfigured(_ identity: Identity?, rtpTimestamp: UInt32) -> UInt16 { + let next: UInt16 = lock.withLock { + generation &+= 1 + return generation + } + record(.decoderConfigured, identity, rtpTimestamp: rtpTimestamp, valueNs: Int64(next)) + return next + } + + /// Assigns the next identity at decoder input and records the boundary. + public func decoderInput(rtpTimestamp: UInt32, atNs: Int64? = nil) -> Identity { + let identity: Identity = lock.withLock { + let id = Identity(sequence: nextSequence, generation: generation) + nextSequence &+= 1 + if nextSequence == 0 { nextSequence = 1 } + return id + } + record(.decoderInput, identity, rtpTimestamp: rtpTimestamp, atNs: atNs) + return identity + } + + /// Records one boundary. A nil identity is counted as a missing-identity + /// event and stored with sequence 0. Events outside the window, before the + /// origin, after close, or beyond capacity are counted, never recorded. + public func record( + _ stage: Stage, _ identity: Identity?, rtpTimestamp: UInt32, valueNs: Int64 = 0, atNs: Int64? = nil + ) { + let at = atNs ?? clock() + lock.withLock { + guard closedAtNs == nil else { late += 1; return } + guard let originNs else { beforeOrigin += 1; return } + let begin = originNs + Int64(window.beginSeconds * 1_000_000_000) + let end = begin + Int64(window.durationSeconds * 1_000_000_000) + if at < begin { outsideBefore += 1; return } + if at >= end { outsideAfter += 1; return } + guard records.count < capacity else { overflow += 1; return } + let sequence = identity?.sequence ?? 0 + if identity == nil { + missingIdentity += 1 + } else { + let bit = UInt16(1) << UInt16(stage.rawValue) + let mask = seen[sequence, default: 0] + if mask & bit != 0 { duplicates += 1 } + seen[sequence] = mask | bit + } + records.append( + Record( + stage: stage.rawValue, generation: identity?.generation ?? 0, sequence: sequence, rtpTimestamp: rtpTimestamp, + atNs: at, valueNs: valueNs)) + // Append order is preserved in the arrays; the bounds are true min/max + // (the clock is sampled before the lock, so appends can be out of order). + firstEventNs = min(firstEventNs ?? at, at) + lastEventNs = max(lastEventNs ?? at, at) + } + } + + /// Terminal: later events are counted as late and never recorded. + public func close(atNs: Int64? = nil) { + let at = atNs ?? clock() + lock.withLock { if closedAtNs == nil { closedAtNs = at } } + } + + // MARK: identity on the decoded pixel buffer (diagnostic-only scalar attachment) + + /// Attaches the identity to a decoded buffer so it survives the RTC native + /// bridge. Reused pool buffers get a fresh value on every VT output while + /// the audit is enabled; nothing is attached when the audit is nil. + public static func attach(_ identity: Identity, to buffer: CVPixelBuffer) { + CVBufferSetAttachment(buffer, attachmentKey as CFString, NSNumber(value: identity.packed), .shouldNotPropagate) + } + + /// The bridge helper used at VideoToolbox output while the audit is enabled: + /// a nil identity CLEARS any previous value so a reused pool buffer can never + /// carry a stale identity to the renderer. Not called when the audit is off. + public static func stamp(_ identity: Identity?, on buffer: CVPixelBuffer) { + if let identity { attach(identity, to: buffer) } else { removeIdentity(from: buffer) } + } + + public static func readIdentity(from buffer: CVPixelBuffer) -> Identity? { + guard let value = CVBufferCopyAttachment(buffer, attachmentKey as CFString, nil) as? NSNumber else { return nil } + return Identity(packed: value.uint64Value) + } + + public static func removeIdentity(from buffer: CVPixelBuffer) { + CVBufferRemoveAttachment(buffer, attachmentKey as CFString) + } + + public var isClosed: Bool { lock.withLock { closedAtNs != nil } } + + public func snapshot() -> Snapshot { + lock.withLock { + var perStage: [String: Int] = [:] + for stage in Stage.allCases { perStage[stage.name] = 0 } + for r in records { if let stage = Stage(rawValue: r.stage) { perStage[stage.name, default: 0] += 1 } } + let frames = seen.keys.filter { $0 != 0 } + func reached(_ stage: Stage) -> Int { + frames.filter { seen[$0]! & (UInt16(1) << UInt16(stage.rawValue)) != 0 }.count + } + func has(_ mask: UInt16, _ stage: Stage) -> Bool { mask & (UInt16(1) << UInt16(stage.rawValue)) != 0 } + var coverage: [String: Int] = ["framesWithIdentity": frames.count] + for stage in Stage.allCases { coverage["reached." + stage.name] = reached(stage) } + coverage["absent.vtOutputAfterDecoderInput"] = + frames.filter { has(seen[$0]!, .decoderInput) && !has(seen[$0]!, .vtOutput) }.count + coverage["absent.rendererCallbackAfterVtOutput"] = + frames.filter { has(seen[$0]!, .vtOutput) && !has(seen[$0]!, .rtcRendererCallback) }.count + coverage["absent.selectionAfterRendererCallback"] = + frames.filter { has(seen[$0]!, .rtcRendererCallback) && !has(seen[$0]!, .selection) }.count + coverage["absent.selectionAfterRendererCallback.mailboxReplaced"] = + frames.filter { + has(seen[$0]!, .rtcRendererCallback) && !has(seen[$0]!, .selection) && has(seen[$0]!, .mailboxReplaced) + }.count + coverage["absent.submissionAfterSelection"] = + frames.filter { has(seen[$0]!, .selection) && !has(seen[$0]!, .submission) }.count + coverage["absent.completionCallbackAfterSubmission"] = + frames.filter { has(seen[$0]!, .submission) && !has(seen[$0]!, .gpuCompletionCallback) }.count + coverage["absent.presentedResultAfterSubmission"] = + frames.filter { has(seen[$0]!, .submission) && !has(seen[$0]!, .presentedResult) }.count + coverage["presentedResultZero"] = + records.filter { $0.stage == Stage.presentedResult.rawValue && $0.valueNs == 0 }.count + records.filter { $0.stage == Stage.presentedResult.rawValue && $0.valueNs == 0 }.count + let sequences = frames.isEmpty ? nil : [frames.min()!, frames.max()!] + let begin = originNs.map { $0 + Int64(window.beginSeconds * 1_000_000_000) } + return Snapshot( + kind: "receiver-frame-delivery-audit", + clock: + "receiver-local CACurrentMediaTime × 1e9 for origin, window and every atNs; presentedResult.valueNs = Metal presentedTime × 1e9 on that same clock (0 = unpresented); gpuCompletionCallback.atNs = when the completion CALLBACK ran, not GPU hardware end", + originNs: originNs, windowBeginSeconds: window.beginSeconds, windowDurationSeconds: window.durationSeconds, + windowNs: begin.map { [$0, $0 + Int64(window.durationSeconds * 1_000_000_000)] }, + capacity: capacity, recordByteStride: Self.recordByteStride, + preallocatedRecordBytes: capacity * Self.recordByteStride, + seenDictionaryEntries: seen.count, + recorded: records.count, overflow: overflow, beforeOrigin: beforeOrigin, outsideWindowBefore: outsideBefore, + outsideWindowAfter: outsideAfter, + lateAfterClose: late, missingIdentityEvents: missingIdentity, duplicateEvents: duplicates, + closedAtNs: closedAtNs, + firstEventNs: firstEventNs, lastEventNs: lastEventNs, decoderGenerations: Int(generation), + sequenceRange: sequences, + perStage: perStage, + truncation: [ + "overflow": overflow, "beforeOrigin": beforeOrigin, "outsideWindowBefore": outsideBefore, + "outsideWindowAfter": outsideAfter, "lateAfterClose": late, "missingIdentityEvents": missingIdentity, + ], + coverage: coverage, + stages: records.map(\.stage), generations: records.map(\.generation), sequences: records.map(\.sequence), + rtpTimestamps: records.map(\.rtpTimestamp), + atNs: records.map(\.atNs), valueNs: records.map(\.valueNs), + notes: [ + "identity = receiver decoder-input sequence (monotonic across decoder generations) + RTP timestamp; no source code, no remote clock", + "stages before decoder input (capture, encode, network, jitter buffer) and physical scanout are not covered", + "truncation counts (overflow, outside-window, late, missing identity) make a trace incomplete; downstream absences are explained per frame (mailboxReplaced, preparedReceipt value 0, window clipping) and are not by themselves incompleteness", + "memory: preallocatedRecordBytes covers the record array only; the seen dictionary (one entry per identity) and the one-time snapshot/JSON serialization are additional and not bounded by the record stride", + "observer events relate to these boundaries only by time proximity on the shared local clock, never by an exact code→RTP join", + ]) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameIdentity.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameIdentity.swift new file mode 100644 index 000000000..6beaa93c2 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameIdentity.swift @@ -0,0 +1,34 @@ +import CoreVideo +import Foundation + +/// Content identity travels with the pixel buffer. WebRTC's native source +/// truncates the submitted timestamp to microseconds and translates it through +/// its timestamp aligner before the frame reaches the encoder, so a timestamp +/// alone cannot identify a frame across that boundary. The identity is the +/// capture presentation timestamp; a refresh resubmits the same buffer and +/// therefore the same content identity, and a new capture into a recycled +/// buffer sets a new one before submission. +enum ScreenSharingFrameIdentity { + private static let key = "com.codevisor.screen-sharing.source-timestamp-ns" + + static func attach(sourceTimestampNs: Int64, to buffer: CVPixelBuffer) { + CVBufferSetAttachment(buffer, key as CFString, sourceTimestampNs as CFNumber, .shouldNotPropagate) + } + + static func sourceTimestampNs(of buffer: CVPixelBuffer) -> Int64? { + (CVBufferCopyAttachment(buffer, key as CFString, nil) as? NSNumber)?.int64Value + } + + /// The transport path requires identity. A missing attachment means the + /// bridge no longer forwards the native buffer; substituting WebRTC's + /// translated timestamp would mix clocks, so the frame is refused and the + /// established encoder failure lifecycle ends the session. + static func required(of buffer: CVPixelBuffer, metrics: ScreenSharingMetrics) -> Int64? { + guard let identity = sourceTimestampNs(of: buffer) else { + metrics.increment("encoderInputWithoutIdentity") + metrics.label("encoderError", "Encoder input carries no content identity.") + return nil + } + return identity + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameSender.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameSender.swift new file mode 100644 index 000000000..3d80e9bc8 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingFrameSender.swift @@ -0,0 +1,108 @@ +import CoreMedia +import Foundation +@preconcurrency import WebRTC + +/// Capture feeds WebRTC directly on its capture queue. The lock serializes +/// admission with shutdown; no per-frame tasks accumulate on the main actor. +public final class ScreenSharingFrameSender: @unchecked Sendable { + private let source: RTCVideoSource + private let capturer: RTCVideoCapturer + private let metrics: ScreenSharingMetrics + private let idleMonitor: ScreenSharingSourceIdleMonitor + private let lock = NSLock() + private var active = true + private var captureSuspended = false + private var refreshFrames = ScreenSharingRefreshFrameStore() + private var expectedSize: (width: Int, height: Int)? + private var onActivity: (@Sendable () -> Void)? + + init(source: RTCVideoSource, metrics: ScreenSharingMetrics, idleMonitor: ScreenSharingSourceIdleMonitor) { + self.source = source + self.metrics = metrics + self.idleMonitor = idleMonitor + capturer = RTCVideoCapturer(delegate: source) + } + + /// Invoked once per idle-to-active transition, outside the sender's lock. + func onActivity(_ callback: (@Sendable () -> Void)?) { lock.withLock { onActivity = callback } } + + public func push(_ frame: ScreenSharingVideoFrame) { + let activated: (@Sendable () -> Void)? = lock.withLock { + guard active, !captureSuspended else { return nil } + if let expectedSize, + CVPixelBufferGetWidth(frame.pixelBuffer) != expectedSize.width + || CVPixelBufferGetHeight(frame.pixelBuffer) != expectedSize.height + { + metrics.increment("captureTransitionDrops"); return nil + } + let wasHolding = refreshFrames.isHolding + guard let submission = refreshFrames.capture(frame) else { + metrics.increment("captureTimestampDrops"); return nil + } + // New content in a (possibly recycled) buffer: identify it before WebRTC sees it. + ScreenSharingFrameIdentity.attach(sourceTimestampNs: frame.timestampNs, to: frame.pixelBuffer) + metrics.increment("capturedFrames") + // Cache ownership events (not a reference balance): a fill of an empty + // cache, or a replacement that releases the previously held buffer. + metrics.increment(wasHolding ? "refreshCacheReplacements" : "refreshCacheFills") + metrics.label("latestCapturedTimestampNs", String(frame.timestampNs)) + let started = ScreenSharingMetrics.nowNs + metrics.event("captureArrivalInterval", atNanoseconds: started) + metrics.event("captureTimestampInterval", atNanoseconds: frame.timestampNs) + let activated = idleMonitor.recordSubmission(timestampNs: frame.timestampNs, nowNs: started) + submit(submission) + metrics.observe("capturePush", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + return activated ? onActivity : nil + } + activated?() + } + + /// Transition the source format before updating SCK. Frames still arriving + /// at the old size are expendable raw frames and never enter the encoder. + public func configure(_ configuration: ScreenSharingVideoConfiguration) { + lock.withLock { + if refreshFrames.isHolding { metrics.increment("refreshCacheReleases") } + refreshFrames.clear() + expectedSize = (configuration.width, configuration.height) + source.adaptOutputFormat( + toWidth: Int32(configuration.width), height: Int32(configuration.height), + fps: Int32(configuration.framesPerSecond)) + } + } + + func refreshLatest(reason: String = "request") { + lock.withLock { + guard active else { return } + let now = CMTimeConvertScale( + CMClockGetTime(CMClockGetHostTimeClock()), timescale: 1_000_000_000, method: .default) + guard let frame = refreshFrames.refresh(nowNs: now.value) else { return } + metrics.increment("refreshFrames") + metrics.trace( + "refreshSubmission", + "\(ScreenSharingMetrics.nowNs) \(reason) ts=\(frame.timestampNs) id=\(frame.sourceTimestampNs ?? -1)") + submit(frame) + } + } + + /// Probe-only suspension keeps the same cached frame available for recovery. + package func suspendCaptureDelivery() { lock.withLock { captureSuspended = true } } + + func stop() { + lock.withLock { + active = false + if refreshFrames.isHolding { metrics.increment("refreshCacheReleases") } + refreshFrames.clear() + onActivity = nil + } + } + + /// Whether the one-frame cache currently owns a buffer (diagnostic). + package var isHoldingCachedFrame: Bool { lock.withLock { refreshFrames.isHolding } } + + private func submit(_ frame: ScreenSharingVideoFrame) { + source.capturer( + capturer, + didCapture: RTCVideoFrame( + buffer: RTCCVPixelBuffer(pixelBuffer: frame.pixelBuffer), rotation: ._0, timeStampNs: frame.timestampNs)) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingICEConfiguration.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingICEConfiguration.swift new file mode 100644 index 000000000..2b6b1f22b --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingICEConfiguration.swift @@ -0,0 +1,46 @@ +import Foundation +@preconcurrency import WebRTC + +public struct ScreenSharingICEServer: Sendable { + public let urls: [String] + public let username: String + public let credential: String + public init(urls: [String], username: String = "", credential: String = "") throws { + guard (1...8).contains(urls.count), username.utf8.count <= 512, credential.utf8.count <= 1024 else { + throw ScreenSharingError.invalid("Invalid Screen Sharing connectivity configuration.") + } + for url in urls { + guard Self.validURL(url), + !url.hasPrefix("turn") || (!username.isEmpty && !credential.isEmpty) + else { throw ScreenSharingError.invalid("Invalid Screen Sharing STUN or TURN server.") } + } + self.urls = urls; self.username = username; self.credential = credential + } + public static func validURL(_ value: String) -> Bool { + guard value.utf8.count <= 2048, let colon = value.firstIndex(of: ":") else { return false } + let scheme = String(value[.. Attempt? { + guard isPending, inFlight == nil else { return nil } + nextAttempt &+= 1 + let attempt = Attempt(id: nextAttempt, generation: requested) + inFlight = attempt + return attempt + } + + /// Returns true when this attempt failed and a later frame must retry it. + @discardableResult + mutating func complete(_ attempt: Attempt?, producedKeyFrame: Bool) -> Bool { + guard let attempt, inFlight == attempt else { return false } + inFlight = nil + if producedKeyFrame { fulfilled = attempt.generation } + return !producedKeyFrame + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetalView.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetalView.swift new file mode 100644 index 000000000..eaa197e12 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetalView.swift @@ -0,0 +1,478 @@ +import CoreVideo +import MetalKit + +/// One queued frame and one GPU command buffer maximum. NV12 planes are +/// bound directly from the decoder's CVPixelBuffer through the texture cache. +/// Scheduling, caching and the stop boundary live in the coordinator; this +/// class encodes and submits (or, in the off-main diagnostic, hands the +/// selected frame to a worker and commits its result). +@MainActor +public final class ScreenSharingMetalView: MTKView, MTKViewDelegate { + public let mailbox: ScreenSharingFrameMailbox + public let metrics: ScreenSharingMetrics + public var onFrameSize: ((CGSize) -> Void)? { + get { coordinator.onFrameSize } + set { coordinator.onFrameSize = newValue } + } + public var fitToWindow = true { didSet { coordinator.setNeedsRedraw() } } + public var onPresented: ((UInt32) -> Void)? { + get { coordinator.onPresented } + set { coordinator.onPresented = newValue } + } + /// True after `stop()`; nothing is scheduled, cached or notified afterwards. + public var isStopped: Bool { coordinator.stopped } + package let coordinator: ScreenSharingRenderCoordinator + private let encoder: ScreenSharingMetalEncoder + /// Diagnostic off-main preparation worker (nil = the ordinary MTKView path); + /// assigned once after `super.init` because it needs the view's layer. + private var preparer: (any ScreenSharingRenderPreparer)? + #if os(macOS) + private var offMainWorker: ScreenSharingMetalPreparer? + #endif + private let renderOnArrival: Bool + + /// `offMainPreparation` (diagnostic, macOS, requires `renderOnArrival`): + /// drawable acquisition (`CAMetalLayer.nextDrawable`, default 1 s timeout + /// kept) and command encoding run on a dedicated serial worker; selection, + /// the single slot, commit and every product callback stay on the main actor. + public init( + mailbox: ScreenSharingFrameMailbox, metrics: ScreenSharingMetrics, renderOnArrival: Bool = false, + maximumDrawableCount: Int = 3, unsyncedPresentation: Bool = false, offMainPreparation: Bool = false, + deliveryAudit: ScreenSharingFrameDeliveryAudit? = nil + ) throws { + guard (2...3).contains(maximumDrawableCount) else { + throw ScreenSharingError.invalid("Drawable count must be two or three.") + } + #if !os(macOS) + guard !unsyncedPresentation else { throw ScreenSharingError.invalid("Unsynced presentation requires macOS.") } + guard !offMainPreparation else { throw ScreenSharingError.invalid("Off-main preparation requires macOS.") } + #endif + guard !offMainPreparation || renderOnArrival else { + throw ScreenSharingError.invalid("Off-main preparation requires arrival-driven rendering.") + } + guard let device = MTLCreateSystemDefaultDevice(), let queue = device.makeCommandQueue() else { + throw ScreenSharingError.unavailable("Metal is unavailable.") + } + self.mailbox = mailbox + self.metrics = metrics + self.renderOnArrival = renderOnArrival + let coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: metrics, renderOnArrival: renderOnArrival, redrawsOnDemand: offMainPreparation) + coordinator.audit = deliveryAudit + self.coordinator = coordinator + let library = try device.makeLibrary(source: Self.shader, options: nil) + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: "screenVertex") + descriptor.fragmentFunction = library.makeFunction(name: "screenFragment") + descriptor.colorAttachments[0].pixelFormat = .bgra8Unorm + let pipeline = try device.makeRenderPipelineState(descriptor: descriptor) + encoder = try ScreenSharingMetalEncoder(device: device, commandQueue: queue, pipeline: pipeline) + // The worker owns its own texture cache; the command queue is shared (thread-safe per Metal). + let workerEncoder = + offMainPreparation ? try ScreenSharingMetalEncoder(device: device, commandQueue: queue, pipeline: pipeline) : nil + preparer = nil + super.init(frame: .zero, device: device) + colorPixelFormat = .bgra8Unorm + clearColor = MTLClearColorMake(0.025, 0.025, 0.025, 1) + preferredFramesPerSecond = 60 + framebufferOnly = true + isPaused = renderOnArrival + enableSetNeedsDisplay = renderOnArrival + delegate = self + #if os(macOS) + wantsLayer = true + layer?.isOpaque = true + if let metalLayer = layer as? CAMetalLayer { + metalLayer.maximumDrawableCount = maximumDrawableCount + metalLayer.displaySyncEnabled = !unsyncedPresentation + } + if let workerEncoder { + guard let metalLayer = layer as? CAMetalLayer else { + throw ScreenSharingError.unavailable("The view has no Metal layer.") + } + let worker = ScreenSharingMetalPreparer( + layer: metalLayer, encoder: workerEncoder, metrics: metrics, audit: deliveryAudit) + preparer = worker + offMainWorker = worker + // The worker owns every layer mutation and acquisition in this mode: each + // request carries the view's backing size as an immutable snapshot (no + // lock the main actor could wait on), so MTKView's automatic resize is off. + autoResizeDrawable = false + } + #else + isOpaque = true + (layer as? CAMetalLayer)?.maximumDrawableCount = maximumDrawableCount + _ = workerEncoder + #endif + metrics.label("maximumDrawableCount", String(maximumDrawableCount)) + metrics.label("displaySync", unsyncedPresentation ? "disabled experiment" : "enabled") + metrics.label("frameSelection", "before drawable acquisition") + metrics.label("renderPreparation", preparer == nil ? "main actor (MTKView)" : "off-main serial worker (diagnostic)") + metrics.label( + "drawableAcquisitionPath", + preparer == nil + ? "MTKView.currentDrawable on main actor" : "CAMetalLayer.nextDrawable on render worker, default timeout") + coordinator.bind { [weak self] in self?.draw() } + } + + /// Terminal, idempotent stop boundary (see `ScreenSharingRenderCoordinator.stop`). + /// Pauses the display-link drive as well; a submission already in flight + /// keeps its buffer and textures until the GPU completes it, and a + /// preparation already on the worker finishes or times out on its own. + public func stop() { + coordinator.stop() + isPaused = true + } + + required init(coder: NSCoder) { fatalError("Use init(mailbox:metrics:).") } + + public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { coordinator.setNeedsRedraw() } + + #if os(macOS) + public override func layout() { + super.layout() + resizeDrawableOffMain() + } + + public override func viewDidChangeBackingProperties() { + super.viewDidChangeBackingProperties() + resizeDrawableOffMain() + } + + /// Off-main mode only (MTKView auto-resize is off there): a resize or + /// backing-scale change asks the coordinator for a redraw of the cached + /// frame; the draw is scheduled (never reentrant here) and its request + /// carries the new backing size, which the worker applies before acquiring. + private func resizeDrawableOffMain() { + guard offMainWorker != nil else { return } + let size = convertToBacking(bounds).size + guard size.width >= 1, size.height >= 1 else { return } + coordinator.setNeedsRedraw() + } + #endif + + public func draw(in view: MTKView) { + if let preparer { + guard !coordinator.stopped else { return } + metrics.event("renderDriveInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + // Seeded from the view's current backing size; before the first layout the + // frame stays in the mailbox and the resize-driven redraw picks it up. + let size = backingDrawableSize + guard size.width >= 1, size.height >= 1 else { metrics.increment("renderDeferredUntilLayout"); return } + let clear = clearColor + coordinator.prepare( + with: preparer, + geometry: .init( + fitToWindow: fitToWindow, clearColor: SIMD4(clear.red, clear.green, clear.blue, clear.alpha), + drawableSize: size)) + } else { + renderFrame(surface: nil) + } + } + + private var backingDrawableSize: CGSize { + #if os(macOS) + convertToBacking(bounds).size + #else + CGSize(width: bounds.width * contentScaleFactor, height: bounds.height * contentScaleFactor) + #endif + } + + /// The standalone Metal display-link experiment supplies its drawable. The + /// ordinary MTKView path retains its existing acquisition/scheduling policy. + package func draw(displayLinkDrawable drawable: any CAMetalDrawable) { + let pass = MTLRenderPassDescriptor() + pass.colorAttachments[0].texture = drawable.texture + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].storeAction = .store + pass.colorAttachments[0].clearColor = clearColor + renderFrame(surface: Surface(drawable: drawable, pass: pass)) + } + + /// The ordinary main-actor path, in the Stage 3g order: select → validate the + /// pixel format and create the plane textures → acquire the MTKView drawable + /// (as late as possible) → encode → commit. The submission clock starts in + /// the coordinator at final submission, not during acquisition. + private func renderFrame(surface: Surface?) { + guard !coordinator.stopped else { return } + metrics.event("renderDriveInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + guard let (frame, isNewFrame) = coordinator.select() else { return } + let audit = isNewFrame ? coordinator.audit : nil + guard let textures = encoder.textures(for: frame) else { metrics.increment("renderDrops"); return } + // The acquisition pair brackets only the actual MTKView acquisition; a + // supplied drawable (display-link experiment) records no acquisition. + let target: Surface? + if let surface { + target = surface + } else { + if let audit { audit.record(.acquisitionBegin, frame.deliveryAuditIdentity, rtpTimestamp: frame.rtpTimestamp) } + target = acquireSurface() + if let audit { + audit.record( + .acquisitionEnd, frame.deliveryAuditIdentity, rtpTimestamp: frame.rtpTimestamp, valueNs: target == nil ? 0 : 1 + ) + } + } + guard let target else { metrics.increment("renderDrops"); return } + coordinator.reportSize(ScreenSharingMetalEncoder.videoSize(of: frame)) + guard let encoded = encoder.encode(textures, into: target, fitToWindow: fitToWindow) else { + metrics.increment("renderDrops") + return + } + // Submission boundary after successful encoding, immediately before commit — + // the same boundary the off-main worker's result uses. + let submittedAt = CACurrentMediaTime() + // Refused after a stop that happened during encoding (e.g. inside the + // size callback): the encoded buffer is never committed or presented. + guard + coordinator.commit( + MetalSubmission(buffer: encoded.buffer, drawable: target.drawable, metrics: metrics), + retaining: encoded.retained, frame: frame, isNewFrame: isNewFrame, submittedAt: submittedAt) + else { metrics.increment("renderDrops"); return } + } + + /// The real submission: command-buffer completion and drawable presentation. + /// Metal/CA objects are handed between threads by design (Metal documents + /// the command queue as thread-safe and one thread per command buffer). + struct MetalSubmission: ScreenSharingRenderSubmission, @unchecked Sendable { + let buffer: any MTLCommandBuffer + let drawable: any CAMetalDrawable + let metrics: ScreenSharingMetrics + + func onCompleted(_ handler: @escaping @Sendable (Bool) -> Void) { + buffer.addCompletedHandler { command in handler(command.status == .completed) } + } + + func onPresented(_ handler: @escaping @Sendable (Double) -> Void) { + #if !targetEnvironment(simulator) + drawable.addPresentedHandler { presented in handler(presented.presentedTime) } + #else + // Simulator Metal has no drawable presentation callback. GPU completion + // remains observable, but it must not be reported as physical presentation. + _ = handler + metrics.label("presentationTelemetry", "unavailable in simulator") + #endif + } + + func commit() { + buffer.present(drawable) + buffer.commit() + } + } + + private func acquireSurface() -> Surface? { + let started = ScreenSharingMetrics.nowNs + defer { + metrics.observe("drawableAcquisition", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + } + guard let drawable = currentDrawable, let pass = currentRenderPassDescriptor else { return nil } + return Surface(drawable: drawable, pass: pass) + } + + private static let shader = """ + #include + using namespace metal; + struct ScreenVertex { float4 position [[position]]; float2 uv; }; + vertex ScreenVertex screenVertex(uint id [[vertex_id]]) { + float2 positions[3] = {float2(-1,-1), float2(3,-1), float2(-1,3)}; + float2 p = positions[id]; + return {float4(p,0,1), float2((p.x+1)*0.5, (1-p.y)*0.5)}; + } + fragment float4 screenFragment(ScreenVertex in [[stage_in]], + texture2d yPlane [[texture(0)]], texture2d uvPlane [[texture(1)]], + constant float& fullRange [[buffer(0)]]) { + constexpr sampler sample(filter::linear, address::clamp_to_edge); + float y = yPlane.sample(sample,in.uv).r; + float2 uv = uvPlane.sample(sample,in.uv).rg - float2(128.0/255.0); + if (fullRange < 0.5) { y = (y-16.0/255.0)*(255.0/219.0); uv *= 255.0/224.0; } + return float4(y+1.5748*uv.y, y-0.187324*uv.x-0.468124*uv.y, y+1.8556*uv.x, 1); + } + """ +} + +/// A drawable with its render pass (MTKView's on the main actor, or one built +/// by the off-main worker from `CAMetalLayer.nextDrawable`). +struct Surface { + let drawable: any CAMetalDrawable + let pass: MTLRenderPassDescriptor +} + +/// Pixel buffer + its CVMetalTextures, held until actual GPU completion so the +/// IOSurface storage stays alive throughout GPU reads. +final class TextureFrame: @unchecked Sendable { + let frame: ScreenSharingVideoFrame + let textures: [CVMetalTexture] + init(frame: ScreenSharingVideoFrame, textures: [CVMetalTexture]) { self.frame = frame; self.textures = textures } +} + +/// Pure Metal encoding of one NV12 frame into a surface — no actor, no AppKit. +/// One instance per thread (the texture cache is not shared); the command +/// queue and pipeline are shared (thread-safe / immutable per Metal). +struct ScreenSharingMetalEncoder: @unchecked Sendable { + /// Validated plane textures of one frame, created before any drawable is acquired. + struct Textures { + let frame: ScreenSharingVideoFrame + let y: CVMetalTexture + let uv: CVMetalTexture + let fullRange: Bool + } + struct Encoded { + let buffer: any MTLCommandBuffer + let retained: TextureFrame + } + private let commandQueue: any MTLCommandQueue + private let pipeline: any MTLRenderPipelineState + private let textureCache: CVMetalTextureCache + + init(device: any MTLDevice, commandQueue: any MTLCommandQueue, pipeline: any MTLRenderPipelineState) throws { + var cache: CVMetalTextureCache? + let status = CVMetalTextureCacheCreate(nil, nil, device, nil, &cache) + guard status == kCVReturnSuccess, let cache else { throw ScreenSharingError.codec("Create texture cache", status) } + self.commandQueue = commandQueue + self.pipeline = pipeline + textureCache = cache + } + + static func videoSize(of frame: ScreenSharingVideoFrame) -> CGSize { + CGSize(width: CVPixelBufferGetWidth(frame.pixelBuffer), height: CVPixelBufferGetHeight(frame.pixelBuffer)) + } + + /// Pixel-format validation and plane textures; nil for an unsupported frame. + func textures(for frame: ScreenSharingVideoFrame) -> Textures? { + let pixel = frame.pixelBuffer + let format = CVPixelBufferGetPixelFormatType(pixel) + guard + [ + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, + ].contains(format), + let y = texture(pixel, plane: 0, format: .r8Unorm), + let uv = texture(pixel, plane: 1, format: .rg8Unorm) + else { return nil } + let fullRange = [kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, kCVPixelFormatType_444YpCbCr8BiPlanarFullRange] + .contains(format) + return Textures(frame: frame, y: y, uv: uv, fullRange: fullRange) + } + + /// Encodes and ends encoding; the buffer is neither presented nor committed here. + func encode(_ textures: Textures, into target: Surface, fitToWindow: Bool) -> Encoded? { + guard + let yTexture = CVMetalTextureGetTexture(textures.y), let uvTexture = CVMetalTextureGetTexture(textures.uv), + let buffer = commandQueue.makeCommandBuffer(), + let encoder = buffer.makeRenderCommandEncoder(descriptor: target.pass) + else { return nil } + let pixel = textures.frame.pixelBuffer + let width = Double(CVPixelBufferGetWidth(pixel)) + let height = Double(CVPixelBufferGetHeight(pixel)) + let targetSize = CGSize(width: target.drawable.texture.width, height: target.drawable.texture.height) + let scale = fitToWindow ? min(targetSize.width / width, targetSize.height / height) : 1 + encoder.setViewport( + MTLViewport( + originX: (targetSize.width - width * scale) / 2, originY: (targetSize.height - height * scale) / 2, + width: width * scale, height: height * scale, znear: 0, zfar: 1)) + encoder.setRenderPipelineState(pipeline) + encoder.setFragmentTexture(yTexture, index: 0) + encoder.setFragmentTexture(uvTexture, index: 1) + var fullRange: Float = textures.fullRange ? 1 : 0 + encoder.setFragmentBytes(&fullRange, length: MemoryLayout.size, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + return Encoded(buffer: buffer, retained: TextureFrame(frame: textures.frame, textures: [textures.y, textures.uv])) + } + + private func texture(_ buffer: CVPixelBuffer, plane: Int, format: MTLPixelFormat) -> CVMetalTexture? { + var texture: CVMetalTexture? + let status = CVMetalTextureCacheCreateTextureFromImage( + nil, textureCache, buffer, nil, format, CVPixelBufferGetWidthOfPlane(buffer, plane), + CVPixelBufferGetHeightOfPlane(buffer, plane), plane, &texture) + return status == kCVReturnSuccess ? texture : nil + } +} + +#if os(macOS) + /// Diagnostic off-main preparation. A dedicated serial worker acquires from + /// `CAMetalLayer.nextDrawable` (the layer's default 1 s timeout is kept — + /// `allowsNextDrawableTimeout` is never disabled), builds the render pass + /// and encodes; the result goes back to the coordinator, which alone commits. + /// At most one preparation is outstanding (the coordinator's slot); an + /// acquisition already blocking when the renderer stops finishes or times + /// out on its own, touches no AppKit state, and its result is dropped. + final class ScreenSharingMetalPreparer: ScreenSharingRenderPreparer, @unchecked Sendable { + private let queue = DispatchQueue(label: "codevisor.screen-sharing.render-worker", qos: .userInteractive) + private let layer: CAMetalLayer + private let encoder: ScreenSharingMetalEncoder + private let metrics: ScreenSharingMetrics + private let audit: ScreenSharingFrameDeliveryAudit? + + init( + layer: CAMetalLayer, encoder: ScreenSharingMetalEncoder, metrics: ScreenSharingMetrics, + audit: ScreenSharingFrameDeliveryAudit? = nil + ) { + self.layer = layer + self.encoder = encoder + self.metrics = metrics + self.audit = audit + } + + func prepare( + _ request: ScreenSharingPreparationRequest, + completion: @escaping @Sendable (ScreenSharingPreparedSubmission?) -> Void + ) { + let layer = layer + let encoder = encoder + let metrics = metrics + let audit = request.isNewFrame ? audit : nil + let identity = request.auditIdentity + let rtp = request.frame.rtpTimestamp + queue.async { + // One autorelease-pool boundary per preparation: a drawable or texture + // that is not handed back is released here, never at a later drain. + let prepared: ScreenSharingPreparedSubmission? = autoreleasepool { + let started = ScreenSharingMetrics.nowNs + if let audit { audit.record(.preparationBegin, identity, rtpTimestamp: rtp, atNs: audit.now()) } + metrics.observe("renderPreparationQueueWait", milliseconds: Double(started - request.queuedAtNs) / 1_000_000) + // Textures first (as on the main path), then the layer size from the + // request's snapshot — the only layer mutation, on this queue — then the + // acquisition, as late as possible. + guard let textures = encoder.textures(for: request.frame) else { return nil } + if layer.drawableSize != request.geometry.drawableSize { + layer.drawableSize = request.geometry.drawableSize + metrics.increment("drawableSizeUpdatesOnWorker") + } + let acquisitionStarted = ScreenSharingMetrics.nowNs + if let audit { audit.record(.acquisitionBegin, identity, rtpTimestamp: rtp, atNs: audit.now()) } + let drawable = layer.nextDrawable() // blocks at most the layer's default 1 s timeout + let acquisitionEnded = ScreenSharingMetrics.nowNs + if let audit { + audit.record( + .acquisitionEnd, identity, rtpTimestamp: rtp, valueNs: drawable == nil ? 0 : 1, atNs: audit.now()) + } + metrics.observe( + "drawableAcquisition", milliseconds: Double(acquisitionEnded - acquisitionStarted) / 1_000_000) + guard let drawable else { + // nil = timeout OR invalid layer properties; the layer does not say which. + metrics.increment("renderPreparationNoDrawable") + return nil + } + let pass = MTLRenderPassDescriptor() + pass.colorAttachments[0].texture = drawable.texture + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].storeAction = .store + let clear = request.geometry.clearColor + pass.colorAttachments[0].clearColor = MTLClearColor( + red: clear.x, green: clear.y, blue: clear.z, alpha: clear.w) + guard + let encoded = encoder.encode( + textures, into: Surface(drawable: drawable, pass: pass), fitToWindow: request.geometry.fitToWindow) + else { return nil } // the drawable is released with this pool, never presented + metrics.observe("renderPreparation", milliseconds: Double(ScreenSharingMetrics.nowNs - started) / 1_000_000) + return .init( + submission: ScreenSharingMetalView.MetalSubmission( + buffer: encoded.buffer, drawable: drawable, metrics: metrics), + retained: encoded.retained, videoSize: ScreenSharingMetalEncoder.videoSize(of: request.frame)) + } + completion(prepared) + } + } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics+Presentation.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics+Presentation.swift new file mode 100644 index 000000000..c13997c7d --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics+Presentation.swift @@ -0,0 +1,23 @@ +import Foundation + +extension ScreenSharingMetrics { + /// Metal calls presented handlers for skipped drawables too. Only a positive + /// presentedTime establishes on-screen presentation. All times use CA's clock. + @discardableResult + public func recordPresentation( + isNewFrame: Bool, presentedAt: Double, submittedAt: Double, receivedAt: Double? + ) -> Bool { + guard isNewFrame else { return false } + increment("presentationCallbacks") + guard presentedAt.isFinite, presentedAt > 0 else { + increment("unpresentedDrawables") + return false + } + increment("presentedFrames") + observe("submissionToPresentation", milliseconds: (presentedAt - submittedAt) * 1000) + if let receivedAt { + observe("receiverCallbackToPresentation", milliseconds: (presentedAt - receivedAt) * 1000) + } + return true + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics.swift new file mode 100644 index 000000000..db2e614f3 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingMetrics.swift @@ -0,0 +1,100 @@ +import Foundation + +/// Bounded measurements from a single process's monotonic clock. No remote +/// clock subtraction: network and input-to-photon measurements are separate. +public final class ScreenSharingMetrics: @unchecked Sendable { + private let lock = NSLock() + private var counters: [String: Int] = [:] + private var labels: [String: String] = [:] + private var samples: [String: [Double]] = [:] + private var indices: [String: Int] = [:] + private var previousEvents: [String: Int64] = [:] + private var traces: [String: [String]] = [:] + private var tracing = false + private let capacity = 1800 + private let traceCapacity = 64 + + public init() {} + public static var nowNs: Int64 { Int64(DispatchTime.now().uptimeNanoseconds) } + + /// Returns the new value so callers can label first occurrences atomically. + @discardableResult + public func increment(_ name: String, by count: Int = 1) -> Int { + lock.withLock { + counters[name, default: 0] += count + return counters[name, default: 0] + } + } + + public func label(_ name: String, _ value: String) { + lock.withLock { labels[name] = value } + } + + /// Probe-only boundary tracing. Disabled by default: the entry autoclosure + /// is never evaluated, so no string is built on media paths in the app. + public func enableTracing() { lock.withLock { tracing = true } } + public var isTracing: Bool { lock.withLock { tracing } } + + /// Bounded ordered diagnostic events (newest 64 per name) for boundary + /// tracing; a single string per event keeps the cost to one lock and copy. + public func trace(_ name: String, _ entry: @autoclosure () -> String) { + guard isTracing else { return } + let value = entry() + lock.withLock { + if traces[name, default: []].count == traceCapacity { traces[name]?.removeFirst() } + traces[name, default: []].append(value) + } + } + + public func observe(_ name: String, milliseconds: Double) { + guard milliseconds.isFinite, milliseconds >= 0 else { return } + lock.withLock { + if samples[name, default: []].count < capacity { + samples[name, default: []].append(milliseconds) + } else { + let index = indices[name, default: 0] + samples[name]?[index] = milliseconds + indices[name] = (index + 1) % capacity + } + } + } + + /// Cadence within one stage and clock. Repeated timestamps are valid zero + /// intervals; backwards timestamps reset the baseline without a huge sample. + public func event(_ name: String, atNanoseconds timestamp: Int64) { + guard timestamp >= 0 else { return } + let elapsed: Double? = lock.withLock { + let previous = previousEvents.updateValue(timestamp, forKey: name) + guard let previous, timestamp >= previous else { return nil } + return Double(timestamp - previous) / 1_000_000 + } + if let elapsed { observe(name, milliseconds: elapsed) } + } + + public struct Snapshot: Codable, Sendable { + public let counters: [String: Int] + public let labels: [String: String] + public let timings: [String: Timing] + public let traces: [String: [String]]? + } + + public struct Timing: Codable, Sendable { + public let count: Int + public let p50Ms: Double + public let p95Ms: Double + public let maximumMs: Double + } + + public func snapshot() -> Snapshot { + lock.withLock { + Snapshot( + counters: counters, labels: labels, + timings: samples.mapValues { values in + let sorted = values.sorted() + return Timing( + count: sorted.count, p50Ms: sorted[min(sorted.count - 1, Int(Double(sorted.count) * 0.5))], + p95Ms: sorted[min(sorted.count - 1, Int(Double(sorted.count) * 0.95))], maximumMs: sorted.last ?? 0) + }, traces: traces.isEmpty ? nil : traces) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowCapture.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowCapture.swift new file mode 100644 index 000000000..2f83b9fc4 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowCapture.swift @@ -0,0 +1,222 @@ +import Foundation + +/// Pure decision types for the diagnostic owned-window capture mode. They are +/// platform-neutral so the boundaries are testable without ScreenCaptureKit. + +/// Exact selection of the process's own window from current-process shareable +/// content: the window ID must match and the owning process must be this one. +/// Anything else fails closed; there is no display or broader-window fallback. +package enum ScreenSharingOwnedWindowSelection { + package struct Candidate: Equatable, Sendable { + package let windowID: UInt32 + package let owningProcessID: Int32? + package init(windowID: UInt32, owningProcessID: Int32?) { + self.windowID = windowID; self.owningProcessID = owningProcessID + } + } + + package enum Failure: Error, Equatable { + case windowNotListed + case owningProcessNotReported + case owningProcessMismatch(reported: Int32) + case ambiguous(count: Int) + } + + package static func select( + windowID: UInt32, processID: Int32, from candidates: [Candidate] + ) -> Result { + let matches = candidates.filter { $0.windowID == windowID } + guard !matches.isEmpty else { return .failure(.windowNotListed) } + guard matches.count == 1, let match = matches.first else { return .failure(.ambiguous(count: matches.count)) } + guard let owner = match.owningProcessID else { return .failure(.owningProcessNotReported) } + guard owner == processID else { return .failure(.owningProcessMismatch(reported: owner)) } + return .success(match) + } +} + +/// Bounded accounting of ScreenCaptureKit sample callbacks by frame status. +/// Only a complete sample with an image buffer is delivered; every other +/// callback is counted under a fixed name and never treated as a pool drop. +package enum ScreenSharingCaptureCallbackAccounting { + package enum Status: Int, CaseIterable, Sendable { + case complete = 0, idle = 1, blank = 2, suspended = 3, started = 4, stopped = 5 + + package var counterName: String { + switch self { + case .complete: "captureCallbacksComplete" + case .idle: "captureCallbacksIdle" + case .blank: "captureCallbacksBlank" + case .suspended: "captureCallbacksSuspended" + case .started: "captureCallbacksStarted" + case .stopped: "captureCallbacksStopped" + } + } + } + + package static let otherStatusCounter = "captureCallbacksOtherStatus" + package static let invalidSampleCounter = "captureSamplesInvalid" + package static let missingStatusCounter = "captureSamplesWithoutStatus" + package static let missingImageCounter = "captureSamplesWithoutImage" + package static let latestStatusLabel = "captureLatestStatus" + + /// Total number of sample callbacks from the counters: every status bucket + /// plus the unknown-status, invalid-sample and missing-status buckets — which + /// are mutually exclusive per callback. `captureSamplesWithoutImage` is NOT + /// added: it is a sub-count of complete callbacks (a complete sample without + /// an image increments both) and would count such a callback twice. + package static func callbackTotal(counters: [String: Int]) -> Int { + let buckets = Status.allCases.map(\.counterName) + [otherStatusCounter, invalidSampleCounter, missingStatusCounter] + return buckets.reduce(0) { $0 + counters[$1, default: 0] } + } + + /// Records one callback and returns whether its frame may be delivered. + @discardableResult + package static func record( + valid: Bool, rawStatus: Int?, hasImage: Bool, metrics: ScreenSharingMetrics + ) -> Bool { + guard valid else { metrics.increment(invalidSampleCounter); return false } + guard let rawStatus else { metrics.increment(missingStatusCounter); return false } + guard let status = Status(rawValue: rawStatus) else { + metrics.increment(otherStatusCounter) + metrics.label(latestStatusLabel, "other(\(rawStatus))") + return false + } + metrics.increment(status.counterName) + metrics.label(latestStatusLabel, String(describing: status)) + guard status == .complete else { return false } + guard hasImage else { metrics.increment(missingImageCounter); return false } + return true + } +} + +/// Finite lifecycle of the diagnostic's one owned workload window and its +/// capture stream, with the boundaries the report must record. Transitions +/// are explicit; an out-of-order request is refused rather than reordered. +package struct ScreenSharingOwnedWorkloadLifecycle: Sendable { + package enum State: String, Sendable { + case created, shown, ready, capturing, workloadPaused, captureStopped, closed + /// Closed during cleanup without a completed capture stop (setup failed or + /// was cancelled after the window was shown, or the stop did not complete). + case abandoned + } + + package enum Transition: String, Sendable { + case show, ready, startCapture, pauseWorkload, stopCapture, close, abandon + } + + package enum CleanupOutcome: Equatable, Sendable { + case neverShown + case closedAfterCaptureStop + case abandoned(from: State) + case alreadyFinished(State) + } + + package struct Refusal: Error, Equatable { + package let transition: Transition + package let state: State + } + + package private(set) var state: State = .created + package private(set) var timestampsNs: [Transition: Int64] = [:] + + package init() {} + + private static let allowed: [Transition: Set] = [ + .show: [.created], + .ready: [.shown], + .startCapture: [.ready], + .pauseWorkload: [.capturing], + // Capture stops from the animating or the paused state; the window may + // only close after its stream has stopped so no callback outlives it. + .stopCapture: [.capturing, .workloadPaused], + .close: [.captureStopped], + // Cleanup of a shown window whose capture never started, never stopped + // completely, or failed: the window still closes, without claiming a stop. + .abandon: [.shown, .ready, .capturing, .workloadPaused], + ] + + private static let next: [Transition: State] = [ + .show: .shown, .ready: .ready, .startCapture: .capturing, .pauseWorkload: .workloadPaused, + .stopCapture: .captureStopped, .close: .closed, .abandon: .abandoned, + ] + + package private(set) var abandonedFrom: State? + + package mutating func apply(_ transition: Transition, atNs timestamp: Int64) throws { + guard Self.allowed[transition, default: []].contains(state) else { + throw Refusal(transition: transition, state: state) + } + if transition == .abandon { abandonedFrom = state } + state = Self.next[transition]! + timestampsNs[transition] = timestamp + } + + /// The one cleanup boundary the probe uses on every exit path. A capture + /// stop is credited only when the capture itself recorded its completion; + /// the window closes in order only from `captureStopped`, otherwise it is + /// abandoned from wherever setup or the run got to. A never-shown window + /// needs nothing. + package mutating func cleanUp(captureStopCompleted: Bool, atNs timestamp: Int64) -> CleanupOutcome { + switch state { + case .created: return .neverShown + case .closed, .abandoned: return .alreadyFinished(state) + case .capturing, .workloadPaused: + if captureStopCompleted { try? apply(.stopCapture, atNs: timestamp) } + case .shown, .ready, .captureStopped: break + } + if state == .captureStopped { + try? apply(.close, atNs: timestamp) + return .closedAfterCaptureStop + } + let from = state + try? apply(.abandon, atNs: timestamp) + return .abandoned(from: from) + } + + /// True when the window went through every boundary in order and the + /// capture stopped before the window closed. + package var completedInOrder: Bool { + guard state == .closed, let stop = timestampsNs[.stopCapture], let close = timestampsNs[.close] else { + return false + } + return stop <= close + } +} + +/// Time-driven frame code of the diagnostic workload that can be frozen at a +/// pause boundary. Draws record the code they rendered; `freezeAtLastDrawn` +/// holds exactly that code — never a newly time-derived one — so a pause that +/// lands between frames freezes what was actually drawn last, and every later +/// query (including incidental redraws requested by the window server) returns +/// it. Freezing is distinct from stopping a capture stream. +package struct ScreenSharingWorkloadSequence: Sendable { + package let framesPerSecond: Int + package let startedAtSeconds: Double + package private(set) var lastDrawnCode: Int? + package private(set) var lastDrawnAtSeconds: Double? + package private(set) var frozenCode: Int? + package private(set) var frozenAtSeconds: Double? + + package init(framesPerSecond: Int, startedAtSeconds: Double) { + self.framesPerSecond = framesPerSecond + self.startedAtSeconds = startedAtSeconds + } + + package var isFrozen: Bool { frozenCode != nil } + + /// The code a draw starting at `now` renders; recorded as the last drawn. + package mutating func drawn(atSeconds now: Double) -> Int { + let code = frozenCode ?? Int(max(0, now - startedAtSeconds) * Double(framesPerSecond)) + lastDrawnCode = code + lastDrawnAtSeconds = now + return code + } + + /// Freezes at the last DRAWN code (0 if nothing was drawn yet). Idempotent: + /// the first boundary wins. + package mutating func freezeAtLastDrawn(atSeconds now: Double) { + guard frozenCode == nil else { return } + frozenCode = lastDrawnCode ?? 0 + frozenAtSeconds = now + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowObservation.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowObservation.swift new file mode 100644 index 000000000..2b2d14f4b --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowObservation.swift @@ -0,0 +1,160 @@ +import Foundation + +/// Pure semantics for the owned-window diagnostic's observations. Platform +/// neutral so they are testable; the probe supplies the measured values. + +/// First-observed delivery over the probe's measurement ticks (≈1 s). Each +/// metric records the first tick at which its counter became non-zero, plus +/// the initial and final values. Resolution is the tick, never a callback or +/// presentation timestamp; a metric that never became non-zero stays +/// explicitly "never observed". Bounded: one entry per metric, no history. +package struct ScreenSharingFirstObservation: Sendable { + package struct Metric: Equatable, Sendable { + package var initialValue: Int? + package var finalValue: Int? + package var firstObservedTick: Int? + package var firstObservedAtSeconds: Double? + package var valueWhenFirstObserved: Int? + } + + package static let resolution = "measurement tick (about 1 s); not a callback, delivery or presentation timestamp" + package static let neverObserved = "never observed" + package private(set) var metrics: [String: Metric] = [:] + package private(set) var ticks = 0 + + package init(names: [String]) { + for name in names { metrics[name] = Metric() } + } + + /// Records one tick of counter values. Returns the metric names that became + /// non-zero for the first time on this tick, in sorted-name order. + @discardableResult + package mutating func record(elapsedSeconds: Double, values: [String: Int]) -> [String] { + let tick = ticks + ticks += 1 + var newlyObserved: [String] = [] + for name in metrics.keys.sorted() { + guard var metric = metrics[name] else { continue } + let value = values[name] ?? 0 + if metric.initialValue == nil { metric.initialValue = value } + metric.finalValue = value + if metric.firstObservedTick == nil, value > 0 { + metric.firstObservedTick = tick + metric.firstObservedAtSeconds = elapsedSeconds + metric.valueWhenFirstObserved = value + newlyObserved.append(name) + } + metrics[name] = metric + } + return newlyObserved + } + + /// Serialisable summary with explicit never-observed values. + package var summary: [String: [String: String]] { + var result: [String: [String: String]] = [:] + for (name, metric) in metrics { + result[name] = [ + "initialValue": metric.initialValue.map(String.init) ?? "not recorded", + "finalValue": metric.finalValue.map(String.init) ?? "not recorded", + "firstObservedAtSeconds": metric.firstObservedAtSeconds.map { String($0) } ?? Self.neverObserved, + "firstObservedTick": metric.firstObservedTick.map(String.init) ?? Self.neverObserved, + "valueWhenFirstObserved": metric.valueWhenFirstObserved.map(String.init) ?? Self.neverObserved, + "resolution": Self.resolution, + ] + } + return result + } +} + +/// Own-window geometry and identity records. Coordinates are kept in their +/// native conventions; the one conversion is explicit about the display +/// height it uses. +package enum ScreenSharingOwnedWindowGeometry { + package struct Rect: Equatable, Sendable { + package var x: Double, y: Double, width: Double, height: Double + package init(x: Double, y: Double, width: Double, height: Double) { + self.x = x; self.y = y; self.width = width; self.height = height + } + } + + /// A CGWindowList entry reduced to the fields the diagnostic persists. + package struct WindowListEntry: Equatable, Sendable { + package var number: UInt32 + package var ownerPID: Int32? + package var bounds: Rect? + package var layer: Int? + package var isOnscreen: Bool? + package var alpha: Double? + package init(number: UInt32, ownerPID: Int32?, bounds: Rect?, layer: Int?, isOnscreen: Bool?, alpha: Double?) { + self.number = number; self.ownerPID = ownerPID; self.bounds = bounds; self.layer = layer; + self.isOnscreen = isOnscreen; self.alpha = alpha + } + } + + /// Cocoa (origin bottom-left of the main display) → CG global top-left, + /// using the MAIN display's height (`CGDisplayBounds(CGMainDisplayID())`): + /// y' = mainDisplayHeight − (y + height). Only valid with that height. + package static func cocoaToTopLeft(_ frame: Rect, mainDisplayHeight: Double) -> Rect { + Rect(x: frame.x, y: mainDisplayHeight - (frame.y + frame.height), width: frame.width, height: frame.height) + } + + /// Exactly one entry with the own window number AND the own pid; every other + /// outcome is an explicit state, never a guess: the query itself unavailable + /// (nil), no entry with that number, an exact-number entry whose owner the + /// window server did not report, an exact-number entry owned by another pid, + /// or duplicates. Only a `found` entry's fields may be persisted. + package enum Selection: Equatable, Sendable { + case found(WindowListEntry) + case queryUnavailable + case absent + case ownerUnreported + case ownerMismatch(reportedPID: Int32) + case duplicate(count: Int) + } + + package static func ownWindow(in entries: [WindowListEntry]?, number: UInt32, pid: Int32) -> Selection { + guard let entries else { return .queryUnavailable } + let matching = entries.filter { $0.number == number } + guard !matching.isEmpty else { return .absent } + guard matching.count == 1, let entry = matching.first else { return .duplicate(count: matching.count) } + guard let owner = entry.ownerPID else { return .ownerUnreported } + guard owner == pid else { return .ownerMismatch(reportedPID: owner) } + return .found(entry) + } +} + +/// Bounded record of the first AppKit draw-call start per marker code — the +/// exact semantics the image-age analyzer consumes: one entry per code, kept +/// only when the code differs from the previously recorded one (a repeated +/// code, e.g. a frozen paused workload, is not re-recorded), at most `limit` +/// entries with an explicit truncation flag. Draw-call starts are CPU timing, +/// not presentation. +package struct ScreenSharingDrawTimestampRecord: Sendable { + package struct Sample: Equatable, Sendable { + package let code: Int + package let startedAtSeconds: Double + package init(code: Int, startedAtSeconds: Double) { + self.code = code + self.startedAtSeconds = startedAtSeconds + } + } + + package static let defaultLimit = 20_000 + package let limit: Int + package private(set) var samples: [Sample] = [] + package private(set) var truncated = false + + package init(limit: Int = ScreenSharingDrawTimestampRecord.defaultLimit) { self.limit = max(1, limit) } + + /// Records a draw start; returns true when a new entry was retained. + @discardableResult + package mutating func record(code: Int, startedAtSeconds: Double) -> Bool { + guard samples.last?.code != code else { return false } + guard samples.count < limit else { + truncated = true + return false + } + samples.append(Sample(code: code, startedAtSeconds: startedAtSeconds)) + return true + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowSession.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowSession.swift new file mode 100644 index 000000000..910198702 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingOwnedWindowSession.swift @@ -0,0 +1,192 @@ +import Foundation + +/// Readiness gate for the owned workload window's first completed draw. +/// Cancellation-safe: a pending wait is resumed exactly once by the first of +/// signal / timeout / cancellation; once it failed, the gate stays failed so a +/// later wait can never let capture start. +@MainActor +package final class ScreenSharingFirstDrawGate { + package enum Failure: Error, Equatable { case timedOut, cancelled, tornDown } + + private var continuation: CheckedContinuation? + package private(set) var isSignalled = false + package private(set) var failure: Failure? + package private(set) var resumeCount = 0 + + package init() {} + + /// Called from the first completed draw. Ignored after a failure. + package func signal() { + guard failure == nil else { return } + isSignalled = true + resume(.success(())) + } + + /// Waits for the first draw; the deadline is a deadlock guard only. + package func wait(timeout: Duration, sleep: @escaping @Sendable (Duration) async throws -> Void) async throws { + if let failure { throw failure } + if isSignalled { return } + let deadline = Task { @MainActor [weak self] in + do { try await sleep(timeout) } catch { return } + self?.fail(.timedOut) + } + defer { deadline.cancel() } + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (pending: CheckedContinuation) in + if isSignalled { pending.resume(); return } + if let failure { pending.resume(throwing: failure); return } + continuation = pending + } + } onCancel: { + Task { @MainActor [weak self] in self?.fail(.cancelled) } + } + } + + /// Teardown by the owner (cleanup, error exit): resolves a pending wait + /// once with `tornDown` and blocks any later wait, independent of task + /// cancellation. + package func teardown() { fail(.tornDown) } + + private func fail(_ reason: Failure) { + guard failure == nil, !isSignalled else { return } + failure = reason + resume(.failure(reason)) + } + + private func resume(_ result: Result) { + guard let pending = continuation else { return } + continuation = nil + resumeCount += 1 + switch result { + case .success: pending.resume() + case .failure(let error): pending.resume(throwing: error) + } + } +} + +/// Drives the owned window's lifecycle through the asynchronous boundary the +/// probe uses: show → readiness → capture start, optional workload pause, +/// capture stop, and one cleanup on every exit. The stream stop's success or +/// failure is preserved and only a successful stop can lead to an in-order +/// close; any failure or cancellation abandons the window (hidden exactly +/// once) without inventing lifecycle evidence. +@MainActor +package final class ScreenSharingOwnedWindowSession { + package typealias Lifecycle = ScreenSharingOwnedWorkloadLifecycle + + package private(set) var lifecycle = Lifecycle() + package private(set) var hideCount = 0 + package private(set) var startFailure: String? + package private(set) var stopSucceeded: Bool? + package private(set) var stopFailure: String? + package private(set) var cleanupOutcome: Lifecycle.CleanupOutcome? + private let show: @MainActor () throws -> Void + private let hide: @MainActor () -> Void + private let stop: @MainActor () async throws -> Void + private let now: @MainActor () -> Int64 + package private(set) var stopAttempts = 0 + + /// `stopCapture` is owned by the session so that every path on which the + /// stream may have started — including cancellation right after a + /// successful start — awaits a real stop before the window is hidden. + package init( + show: @escaping @MainActor () throws -> Void, hide: @escaping @MainActor () -> Void, + stopCapture: @escaping @MainActor () async throws -> Void, + now: @escaping @MainActor () -> Int64 = { ScreenSharingMetrics.nowNs } + ) { + self.show = show + self.hide = hide + self.stop = stopCapture + self.now = now + } + + /// Shows the window, waits for readiness, then starts capture. Cancellation + /// is checked by the session itself after readiness returns and again before + /// capture starts, so a readiness that merely returns after cancellation + /// never leads to a capture start. Once `startCapture` has returned the + /// stream is live regardless of cancellation: that boundary is recorded + /// first, and the error path then awaits the real stop before hiding. On any + /// error the session cleans up once and rethrows. + package func start(ready: () async throws -> Void, startCapture: () async throws -> Void) async throws { + do { + try lifecycle.apply(.show, atNs: now()) + try show() + try await ready() + try Task.checkCancellation() + try lifecycle.apply(.ready, atNs: now()) + try Task.checkCancellation() + try await startCapture() + try lifecycle.apply(.startCapture, atNs: now()) + try Task.checkCancellation() + } catch { + startFailure = String(describing: error) + if lifecycle.state == .capturing { await stopCapture() } + finish() + throw error + } + } + + package func pauseWorkload(_ pause: () throws -> Void) throws { + try lifecycle.apply(.pauseWorkload, atNs: now()) + try pause() + } + + package private(set) var stopRetriesRefused = 0 + + /// Stops the stream, recording success or failure truthfully. Returns true + /// only when the stop completed; the lifecycle advances only then. The first + /// attempt's result is final: after a failed stop, repeated cleanup must not + /// call the stop again (a capture whose stream is already cleared would + /// return trivially — a no-op is not stop-completion evidence) and can never + /// replace the recorded failure. + @discardableResult + package func stopCapture() async -> Bool { + guard lifecycle.state == .capturing || lifecycle.state == .workloadPaused else { return false } + if stopSucceeded == false { + stopRetriesRefused += 1 + return false + } + stopAttempts += 1 + do { + try await stop() + stopSucceeded = true + try? lifecycle.apply(.stopCapture, atNs: now()) + return true + } catch { + stopSucceeded = false + stopFailure = String(describing: error) + return false + } + } + + /// One cleanup on every exit path. Idempotent: the window is hidden at most + /// once and the first outcome is kept. + @discardableResult + package func finish() -> Lifecycle.CleanupOutcome { + if let cleanupOutcome { return cleanupOutcome } + let outcome = lifecycle.cleanUp(captureStopCompleted: stopSucceeded == true, atNs: now()) + switch outcome { + case .closedAfterCaptureStop, .abandoned: + hideCount += 1 + hide() + case .neverShown, .alreadyFinished: break + } + cleanupOutcome = outcome + return outcome + } + + package var completedInOrder: Bool { lifecycle.completedInOrder && stopSucceeded == true } + + package var record: [String: Any] { + [ + "state": lifecycle.state.rawValue, "completedInOrder": completedInOrder, + "abandonedFrom": lifecycle.abandonedFrom?.rawValue ?? "none", + "startFailure": startFailure ?? "none", "stopSucceeded": stopSucceeded.map { String($0) } ?? "not attempted", + "stopFailure": stopFailure ?? "none", "stopAttempts": stopAttempts, "stopRetriesRefused": stopRetriesRefused, + "cleanup": cleanupOutcome.map { String(describing: $0) } ?? "pending", + "hideCount": hideCount, + "timestampsUptimeNs": Dictionary( + uniqueKeysWithValues: lifecycle.timestampsNs.map { ($0.key.rawValue, $0.value) }), + ] + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingPeer.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingPeer.swift new file mode 100644 index 000000000..400cc51ac --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingPeer.swift @@ -0,0 +1,586 @@ +import Foundation +import QuartzCore +@preconcurrency import WebRTC + +public struct ScreenSharingDescription: Codable, Sendable { + public let version: Int + public let kind: String + public let sdp: String + public init(version: Int = 1, kind: String, sdp: String) { + self.version = version; self.kind = kind; self.sdp = sdp + } +} + +/// Native media endpoint. Signaling is deliberately injected: the probe can +/// exchange files, the app uses authenticated machine channels. +@MainActor +public final class ScreenSharingPeer { + public let metrics: ScreenSharingMetrics + public let control: ScreenSharingControlChannel + public let clipboard: ScreenSharingClipboardChannel + public let mailbox = ScreenSharingFrameMailbox() + public var onConnectionChanged: ((String) -> Void)? + private let factory: RTCPeerConnectionFactory + private let codecFactory: ScreenSharingCodecFactory + private let delegate: ScreenSharingPeerDelegate + private let renderer: ScreenSharingPeerRenderer + private let connection: RTCPeerConnection + private let source: RTCVideoSource + private let videoRefresh: ScreenSharingDataChannel + private let refreshRequester: ScreenSharingRefreshRequester + private let idleNotifier: ScreenSharingSourceIdleNotifier + private let deliveryVerifier: ScreenSharingDeliveryVerifier + private var refreshRateLimit = ScreenSharingRefreshRateLimit() + public nonisolated let frameSender: ScreenSharingFrameSender + private var gathering: CheckedContinuation? + private var gatheringTimeout: Task? + private var remoteTrack: RTCVideoTrack? + private var closed = false + private var negotiating = false + private let ownedWork = ScreenSharingOwnedWork() + /// Receiver-only diagnostic frame-delivery audit (nil = disabled); closed with the peer. + public let frameDeliveryAudit: ScreenSharingFrameDeliveryAudit? + + /// Pins the process trial selection and publishes what is ACTUALLY installed. Separated from `init` so the + /// ordering contract (trials pinned strictly before any RTC object exists) can be exercised over a controlled + /// factory boundary in tests without a fake initializer ever reaching a real factory. + @discardableResult + static func bootstrapTrials( + _ trials: ScreenSharingFieldTrials = .process, publishingInto metrics: ScreenSharingMetrics + ) -> ScreenSharingFieldTrials.Selection { + let installed = trials.ensureInstalled() + // Published from the ACTUAL installed map, never from a caller's request: absence stays absence. + if let playout = installed.playoutExperimentLabel { metrics.label("playoutExperiment", playout) } + // Provenance of the FIRST installer, which after semantic idempotence need not name this caller's profile. + metrics.label("fieldTrialProvenance", installed.name) + return installed + } + + public init( + sending: Bool, configuration: ScreenSharingVideoConfiguration, metrics: ScreenSharingMetrics, + connectivity: ScreenSharingICEConfiguration? = nil, useLowLatencyRateControl: Bool = true, + codec: ScreenSharingVideoCodec = .h264, disableLookAhead: Bool = false, maximumPendingFrames: Int = 2, + maintainSourceRate: Bool = false, staticCodecRate: Bool = false, completeEachFrame: Bool = false, + prioritizeSpeed: Bool = false, keyframeIntervalSeconds: Int = 2, + sourceIdleThresholdNs: Int64? = nil, deliveryGrace: Duration? = nil, deliveryGraceExtensions: Int? = nil, + frameDeliveryAudit: ScreenSharingFrameDeliveryAudit? = nil + ) throws { + self.metrics = metrics + self.frameDeliveryAudit = frameDeliveryAudit + // Process-wide WebRTC trials must exist before ANY RTC object. Real peers always bootstrap through the REAL + // process boundary — there is deliberately no injection point here, because a fake initializer must never be able + // to authorize a real RTC factory or publish a playout label that nothing installed. + ScreenSharingPeer.bootstrapTrials(publishingInto: metrics) + // Idle threshold and grace are diagnostic experiments; nil keeps the product defaults. + let codecFactory = ScreenSharingCodecFactory( + metrics: metrics, useLowLatencyRateControl: useLowLatencyRateControl, codec: codec, + disableLookAhead: disableLookAhead, maximumPendingFrames: maximumPendingFrames, staticCodecRate: staticCodecRate, + completeEachFrame: completeEachFrame, prioritizeSpeed: prioritizeSpeed, + keyframeIntervalSeconds: keyframeIntervalSeconds, + sourceIdleThresholdNs: sourceIdleThresholdNs ?? ScreenSharingSourceIdleMonitor.defaultThresholdNs, + frameDeliveryAudit: frameDeliveryAudit) + self.codecFactory = codecFactory + factory = RTCPeerConnectionFactory(encoderFactory: codecFactory, decoderFactory: codecFactory) + delegate = ScreenSharingPeerDelegate() + renderer = ScreenSharingPeerRenderer(mailbox: mailbox, metrics: metrics, audit: frameDeliveryAudit) + let rtcConfiguration = RTCConfiguration() + rtcConfiguration.sdpSemantics = .unifiedPlan + rtcConfiguration.bundlePolicy = .maxBundle + rtcConfiguration.rtcpMuxPolicy = .require + // Direct LAN is the default. Relay credentials arrive through authenticated + // signaling and are never embedded in the client or persisted with a pane. + rtcConfiguration.iceServers = connectivity?.servers.map(\.native) ?? [] + rtcConfiguration.iceTransportPolicy = connectivity?.relayOnly == true ? .relay : .all + let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) + guard let connection = factory.peerConnection(with: rtcConfiguration, constraints: constraints, delegate: delegate) + else { + throw ScreenSharingError.unavailable("Cannot create native WebRTC peer.") + } + self.connection = connection + control = try ScreenSharingControlChannel( + connection: connection, id: 0, label: "codevisor.control.v1", + encode: { try $0.encoded() }, decode: ScreenSharingControlMessage.decode) + clipboard = try ScreenSharingClipboardChannel( + connection: connection, id: 2, label: "codevisor.clipboard.v1", + encode: { try $0.encoded() }, decode: ScreenSharingClipboardMessage.decode) + let videoRefresh = try ScreenSharingDataChannel( + connection: connection, id: 4, label: "codevisor.video-refresh.v1", + encode: { $0.encoded() }, decode: ScreenSharingVideoRefreshMessage.decode) + self.videoRefresh = videoRefresh + // Diagnostic: the first channel send after a verified shortfall decision. + let requestSendMeasurement = ScreenSharingEncoderRefreshRequest() + refreshRequester = ScreenSharingRefreshRequester( + available: { [weak videoRefresh] in videoRefresh?.isAvailable == true }, + send: { [weak videoRefresh] in + guard videoRefresh?.send(.keyframe) == true else { return false } + metrics.increment("videoRefreshRequestsSent") + if requestSendMeasurement.consume() { + metrics.label("sourceIdleRequestSentAtNs", String(ScreenSharingMetrics.nowNs)) + } + return true + }) + source = factory.videoSource(forScreenCast: true) + let frameSender = ScreenSharingFrameSender( + source: source, metrics: metrics, idleMonitor: codecFactory.sourceIdleMonitor) + self.frameSender = frameSender + frameSender.configure(configuration) + // Host: announce idle once per activity period so the viewer can verify + // that the newest frame arrived despite loss that no later packet reveals. + idleNotifier = ScreenSharingSourceIdleNotifier( + monitor: codecFactory.sourceIdleMonitor, + evaluated: { metrics.increment("sourceIdleEvaluations") }, + resubmit: { [monitor = codecFactory.sourceIdleMonitor] in + metrics.increment("sourceIdleResubmissions") + metrics.label( + "sourceIdleState", + monitor.resubmissionCount >= ScreenSharingSourceIdleMonitor.maximumQuickResubmissions + ? "re-offering the latest capture at the slow bounded rate" : "re-offering the latest capture") + frameSender.refreshLatest(reason: "reoffer") + }, + notify: { [weak videoRefresh, monitor = codecFactory.sourceIdleMonitor] latestTimestampNs in + guard videoRefresh?.send(.sourceIdle(latestTimestampNs: latestTimestampNs)) == true else { + metrics.increment("sourceIdleNoticesDeferred") + return false + } + let now = ScreenSharingMetrics.nowNs + metrics.increment("sourceIdleNotices") + metrics.label("sourceIdleLatestTimestampNs", String(latestTimestampNs)) + metrics.label("sourceIdleState", "latest capture announced") + // Host clock only: last capture submission to notice. + metrics.label("sourceIdleNoticeAtNs", String(now)) + if let submitted = monitor.latestSubmittedNs { + metrics.observe("sourceIdleNoticeDelay", milliseconds: Double(now - submitted) / 1_000_000) + } + return true + }) + // Viewer: a verified shortfall drives the existing keyframe recovery path + // and keeps its target until the announced content is decoded. + deliveryVerifier = ScreenSharingDeliveryVerifier( + audit: codecFactory.deliveryAudit, grace: deliveryGrace ?? ScreenSharingDeliveryVerifier.defaultGrace, + graceExtensions: deliveryGraceExtensions ?? ScreenSharingDeliveryVerifier.defaultGraceExtensions, + refresh: { [refreshSignal = codecFactory.refreshSignal] in + // Demand attribution: only a true result creates new recovery traffic. + let now = String(ScreenSharingMetrics.nowNs) + if refreshSignal.requestUnlessPending() { + if metrics.increment("sourceIdleRecoveryInitiated") == 1 { + metrics.label("sourceIdleRecoveryFirstInitiatedAtNs", now) + } + metrics.label("sourceIdleRecoveryLatestInitiatedAtNs", now) + } else { + if metrics.increment("sourceIdleRefreshAlreadyPending") == 1 { + metrics.label("sourceIdleRefreshFirstAlreadyPendingAtNs", now) + } + metrics.label("sourceIdleRefreshLatestAlreadyPendingAtNs", now) + } + }, + report: { [audit = codecFactory.deliveryAudit] outcome in + // Viewer clock only: notice received, refresh requested, target decoded. + let now = ScreenSharingMetrics.nowNs + func labelTargetDecoded() { + if let met = audit.targetMetAtTimestampNs { metrics.label("sourceIdleTargetDecodedAtNs", String(met)) } + } + switch outcome { + case .verified: + metrics.increment("sourceIdleVerified") + metrics.label("sourceIdleOutcome", "verified at notice") + labelTargetDecoded() + case .verifiedAfterGrace: + metrics.increment("sourceIdleVerifiedAfterGrace") + metrics.label("sourceIdleOutcome", "verified during grace") + labelTargetDecoded() + case .graceExtended: + metrics.increment("sourceIdleGraceExtensions") + case .refresh: + metrics.increment("sourceIdleRefreshRequests") + metrics.label("sourceIdleOutcome", "refresh requested") + metrics.label("sourceIdleRefreshDecisionAtNs", String(now)) + requestSendMeasurement.request() + case .recovered: + metrics.increment("sourceIdleRecovered") + metrics.label("sourceIdleOutcome", "recovered after refresh") + labelTargetDecoded() + case .retry: metrics.increment("sourceIdleRefreshRetries") + case .retryExecuted: + metrics.increment("sourceIdleRetriesExecuted") + metrics.label("sourceIdleRetryLatestExecutedAtNs", String(now)) + } + }) + if sending { + let track = factory.videoTrack(with: source, trackId: "screen") + // addTrack permits the remote viewer's offer to associate this sender + // with its video m-line. An explicit unassociated addTransceiver stays + // separate when answering, leaving ICE connected with no media sender. + guard let sender = connection.add(track, streamIds: ["screen"]), + let transceiver = connection.transceivers.first(where: { $0.sender.senderId == sender.senderId }) + else { + throw ScreenSharingError.unavailable("Cannot create screen video sender.") + } + var directionError: NSError? + transceiver.setDirection(.sendOnly, error: &directionError) + if let directionError { connection.close(); throw directionError } + let parameters = transceiver.sender.parameters + parameters.degradationPreference = NSNumber( + value: (maintainSourceRate + ? RTCDegradationPreference.maintainFramerateAndResolution : .maintainResolution).rawValue) + metrics.label("sourceAdaptation", maintainSourceRate ? "fixed format experiment" : "maintain resolution") + for encoding in parameters.encodings { + encoding.maxBitrateBps = NSNumber(value: configuration.bitrate) + encoding.maxFramerate = NSNumber(value: configuration.framesPerSecond) + } + transceiver.sender.parameters = parameters + connection.setBweMinBitrateBps( + 100_000, currentBitrateBps: NSNumber(value: configuration.bitrate), + maxBitrateBps: NSNumber(value: configuration.bitrate)) + } + if !sending { + let options = RTCRtpTransceiverInit() + options.direction = .recvOnly + guard connection.addTransceiver(of: .video, init: options) != nil else { + throw ScreenSharingError.unavailable("Cannot create screen video receiver.") + } + } + videoRefresh.onMessage = { [weak self] message in + guard let self, !self.closed else { return } + switch message { + case .keyframe: + guard sending else { return } + self.metrics.increment("videoRefreshRequestsReceived") + guard self.refreshRateLimit.allow(nowNs: ScreenSharingMetrics.nowNs) else { + self.metrics.increment("videoRefreshRequestsThrottled") + return + } + self.codecFactory.encoderRefreshRequest.request() + self.frameSender.refreshLatest() + case .sourceIdle(let latestTimestampNs): + guard !sending else { return } + self.metrics.increment("sourceIdleNoticesReceived") + self.metrics.label("sourceIdleNoticeReceivedAtNs", String(ScreenSharingMetrics.nowNs)) + // Evidence of what the viewer held when the host announced idle. + self.metrics.label( + "sourceIdleNoticeDecodedTimestampNs", + self.codecFactory.deliveryAudit.latestDecodedTimestampNs.map(String.init) ?? "none") + self.metrics.label("sourceIdleNoticeTargetTimestampNs", String(latestTimestampNs)) + self.deliveryVerifier.noticed(latestTimestampNs: latestTimestampNs) + } + } + if sending { + frameSender.onActivity { [weak self] in Task { @MainActor in self?.idleNotifier.activate() } } + } + videoRefresh.onAvailabilityChanged = { [weak self] available in + guard available, let self else { return } + self.refreshRequester.wake() + self.idleNotifier.flush() + } + codecFactory.refreshSignal.onChange { [weak self] event in + // Recovery state trace (viewer clock): when keyframe recovery became + // pending and when a decoded keyframe cleared it. + let now = String(ScreenSharingMetrics.nowNs) + if event.needed { + if metrics.increment("refreshRecoveryPendingEvents") == 1 { + metrics.label("refreshRecoveryFirstPendingAtNs", now) + } + metrics.label("refreshRecoveryLatestPendingAtNs", now) + } else { + if metrics.increment("refreshRecoveryClearedEvents") == 1 { + metrics.label("refreshRecoveryFirstClearedAtNs", now) + } + metrics.label("refreshRecoveryLatestClearedAtNs", now) + } + Task { @MainActor in + self?.refreshRequester.update(event) + self?.deliveryVerifier.recoveryChanged(event) + } + } + if !sending { codecFactory.refreshSignal.request() } + delegate.onGathered = { [weak self] in Task { @MainActor in self?.finishGathering() } } + delegate.onConnection = { [weak self] state in + Task { @MainActor in + guard let self, !self.closed else { return } + self.metrics.label("connection", state) + self.onConnectionChanged?(state) + } + } + delegate.onVideoTrack = { [weak self] track in + Task { @MainActor in + guard let self, !self.closed else { return } + self.remoteTrack?.remove(self.renderer) + self.remoteTrack = track + track.add(self.renderer) + } + } + } + + /// Package-only fault injection for the standalone recovery probe. The + /// decoder discards its VT state, then refuses deltas until a fresh keyframe. + package func simulateDecoderLoss( + afterFrames: Int, droppingRecoveryKeyframeFrom sender: ScreenSharingPeer? = nil, + idlingCaptureFrom idleSender: ScreenSharingPeer? = nil + ) { + let dropCheck = sender?.codecFactory.encoderDropCheck + let idleCapture = idleSender?.frameSender + let idleMetrics = idleSender?.metrics + codecFactory.recoveryCheck.arm(afterFrames: afterFrames) { + dropCheck?.arm() + idleCapture?.suspendCaptureDelivery() + idleMetrics?.increment("captureDeliveryStoppedAtDecoderReset") + } + sender?.metrics.label("encoderRecoveryExperiment", "discard one forced output after decoder reset") + idleMetrics?.label("sourceIdleExperiment", "capture delivery stops at decoder reset") + metrics.label("decoderRecoveryExperiment", "discard reference state after \(afterFrames) input frames") + } + + /// Returns SDP with gathered candidates. Caller must carry this over a + /// trusted/authenticated signaling path; SDP fingerprints alone are not identity. + public func makeDescription(offer: Bool) async throws -> ScreenSharingDescription { + guard !closed, !negotiating else { throw ScreenSharingError.invalid("Peer is closed or already negotiating.") } + negotiating = true + defer { negotiating = false } + let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil) + let rtc: RTCSessionDescription = try await withCheckedThrowingContinuation { continuation in + let complete: @Sendable (RTCSessionDescription?, (any Error)?) -> Void = { description, error in + if let error { + continuation.resume(throwing: error) + } else if let description { + continuation.resume(returning: description) + } else { + continuation.resume(throwing: ScreenSharingError.unavailable("No session description.")) + } + } + if offer { + connection.offer(for: constraints, completionHandler: complete) + } else { + connection.answer(for: constraints, completionHandler: complete) + } + } + try Task.checkCancellation() + guard !closed else { throw CancellationError() } + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + connection.setLocalDescription(rtc) { error in + if let error { continuation.resume(throwing: error) } else { continuation.resume() } + } + } + return try await withTaskCancellationHandler { + try Task.checkCancellation() + return try await withCheckedThrowingContinuation { continuation in + gathering = continuation + if closed { cancelGathering(CancellationError()); return } + if connection.iceGatheringState == .complete { finishGathering(); return } + gatheringTimeout = Task { [weak self] in + do { try await Task.sleep(for: .seconds(15)) } catch { return } + self?.cancelGathering(ScreenSharingError.unavailable("ICE gathering timed out.")) + } + } + } onCancel: { [weak self] in + Task { @MainActor in self?.cancelGathering(CancellationError()) } + } + } + + public func accept(_ description: ScreenSharingDescription) async throws { + guard !closed, description.version == 1, ["offer", "answer"].contains(description.kind), + description.sdp.utf8.count <= 256 * 1024, description.sdp.contains("a=fingerprint:sha-256 ") + else { throw ScreenSharingError.invalid("Unsupported or invalid screen-sharing description.") } + let rtc = RTCSessionDescription(type: description.kind == "offer" ? .offer : .answer, sdp: description.sdp) + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + connection.setRemoteDescription(rtc) { error in + if let error { continuation.resume(throwing: error) } else { continuation.resume() } + } + } + } + + public func statistics() async -> [String: String] { + await withCheckedContinuation { continuation in + connection.statistics { report in + var result: [String: String] = [:] + let fields = [ + "framesEncoded", "framesDecoded", "framesDropped", "framesPerSecond", "bytesSent", "bytesReceived", + "packetsLost", "jitter", "currentRoundTripTime", "availableOutgoingBitrate", "encoderImplementation", + "decoderImplementation", "mimeType", "protocol", "relayProtocol", "candidateType", "state", "nominated", + "jitterBufferDelay", "jitterBufferTargetDelay", "jitterBufferMinimumDelay", "jitterBufferEmittedCount", + "totalDecodeTime", "totalProcessingDelay", "totalEncodeTime", "totalPacketSendDelay", "packetsSent", + "packetsReceived", "framesReceived", "keyFramesDecoded", "nackCount", "pliCount", "firCount", + "freezeCount", "totalFreezesDuration", "retransmittedPacketsSent", "qualityLimitationReason", + "networkType", "retransmittedPacketsReceived", "retransmittedBytesReceived", "rtxSsrc", + "retransmittedBytesSent", "headerBytesSent", "headerBytesReceived", "fecPacketsReceived", + ] + let selectedIDs = Set( + report.statistics.values.filter { $0.type == "transport" } + .compactMap { $0.values["selectedCandidatePairId"] as? String }) + let selectedPairs = report.statistics.values.filter { + guard $0.type == "candidate-pair" else { return false } + if !selectedIDs.isEmpty { return selectedIDs.contains($0.id) } + return ($0.values["nominated"] as? NSNumber)?.boolValue == true + && ($0.values["state"] as? String) == "succeeded" + } + let candidateIDs = Set( + selectedPairs.flatMap { + [$0.values["localCandidateId"] as? String, $0.values["remoteCandidateId"] as? String].compactMap { $0 } + }) + for statistic in report.statistics.values { + if statistic.type == "candidate-pair", !selectedPairs.contains(where: { $0.id == statistic.id }) { continue } + if ["local-candidate", "remote-candidate"].contains(statistic.type), !candidateIDs.contains(statistic.id) { + continue + } + guard + ["outbound-rtp", "inbound-rtp", "candidate-pair", "codec", "local-candidate", "remote-candidate"] + .contains(statistic.type) + else { continue } + for field in fields { + if let value = statistic.values[field] { + result["\(statistic.type).\(statistic.id).\(field)"] = value.description + } + } + } + continuation.resume(returning: result) + } + } + } + + public func updateVideoConfiguration(_ configuration: ScreenSharingVideoConfiguration) { + guard !closed else { return } + frameSender.configure(configuration) + for sender in connection.senders where sender.track?.kind == "video" { + let parameters = sender.parameters + for encoding in parameters.encodings { + encoding.maxFramerate = NSNumber(value: configuration.framesPerSecond) + encoding.maxBitrateBps = NSNumber(value: configuration.bitrate) + } + sender.parameters = parameters + } + metrics.label("captureSize", "\(configuration.width) × \(configuration.height)") + metrics.label("captureFPS", String(configuration.framesPerSecond)) + } + + /// Diagnostic boundary for the receiver RTC event-log diagnostic: the shipped + /// ObjC API, called synchronously by the caller; the peer schedules nothing. + /// Returns the API's Bool (an accepted output, not a complete file); a closed + /// peer never starts a log. + public func startRtcEventLog(path: String, maxSizeBytes: Int64) -> Bool { + guard !closed else { return false } + return connection.startRtcEventLog(withFilePath: path, maxSizeInBytes: maxSizeBytes) + } + + /// Stops a log started through `startRtcEventLog`; the caller stops exactly + /// once before `close()`. Returns true only when the native stop API was + /// invoked; a closed peer reports false so a no-op is never credited. + @discardableResult + public func stopRtcEventLog() -> Bool { + guard !closed else { return false } + connection.stopRtcEventLog() + return true + } + + public func close() { + guard !closed else { return } + closed = true + codecFactory.refreshSignal.close() + // Owned main-actor work is cancelled here and can be awaited by awaitClosed(). + ownedWork.close(with: [refreshRequester.close(), idleNotifier.close(), deliveryVerifier.close()].compactMap { $0 }) + codecFactory.sourceIdleMonitor.stop() + videoRefresh.close() + clipboard.close() + control.close() + frameSender.stop() + cancelGathering(CancellationError()) + remoteTrack?.remove(renderer) + renderer.stop() + remoteTrack = nil + connection.close() + mailbox.clear() + frameDeliveryAudit?.close() // later decoder/VT/RTC/GPU/presented callbacks are counted as late, never recorded + } + + /// Completion boundary for the peer's own cancelled tasks (requester, idle + /// notifier, delivery verifier). The handles stay shared, so concurrent and + /// repeated callers all wait for the same completions. Returns the number of + /// owned tasks awaited, or nil when the peer has not been closed (the call + /// then returns immediately and establishes nothing). WebRTC and + /// VideoToolbox threads are not covered; their late callbacks are ignored by + /// the closed signal, sender and renderer. + @discardableResult + public func awaitClosed() async -> Int? { await ownedWork.join() } + + private func finishGathering() { + guard let description = connection.localDescription, let continuation = gathering else { return } + gathering = nil + gatheringTimeout?.cancel() + gatheringTimeout = nil + continuation.resume( + returning: ScreenSharingDescription( + version: 1, kind: description.type == .offer ? "offer" : "answer", sdp: description.sdp)) + } + + private func cancelGathering(_ error: any Error) { + gatheringTimeout?.cancel() + gatheringTimeout = nil + let continuation = gathering + gathering = nil + continuation?.resume(throwing: error) + } +} + +private final class ScreenSharingPeerRenderer: NSObject, RTCVideoRenderer, @unchecked Sendable { + private let lock = NSLock() + private var active = true + let mailbox: ScreenSharingFrameMailbox + let metrics: ScreenSharingMetrics + private let audit: ScreenSharingFrameDeliveryAudit? + init(mailbox: ScreenSharingFrameMailbox, metrics: ScreenSharingMetrics, audit: ScreenSharingFrameDeliveryAudit? = nil) + { + self.mailbox = mailbox; self.metrics = metrics; self.audit = audit + } + func setSize(_ size: CGSize) {} + func stop() { + lock.withLock { + active = false; mailbox.clear() + } + } + func renderFrame(_ frame: RTCVideoFrame?) { + lock.withLock { + guard active, let frame, let native = frame.buffer as? RTCCVPixelBuffer, frame.rotation == ._0 else { return } + metrics.increment("receivedFrames") + metrics.event("receiverCallbackInterval", atNanoseconds: ScreenSharingMetrics.nowNs) + // Audit identity crosses WebRTC's native-buffer bridge as the buffer attachment (nil when the audit is off). + var identity: ScreenSharingFrameDeliveryAudit.Identity? + if let audit { + identity = ScreenSharingFrameDeliveryAudit.readIdentity(from: native.pixelBuffer) + audit.record(.rtcRendererCallback, identity, rtpTimestamp: UInt32(bitPattern: frame.timeStamp)) + } + let replaced = mailbox.put( + ScreenSharingVideoFrame( + pixelBuffer: native.pixelBuffer, timestampNs: frame.timeStampNs, + rtpTimestamp: UInt32(bitPattern: frame.timeStamp), receivedAtSeconds: CACurrentMediaTime(), + deliveryAuditIdentity: identity)) + if let audit, let replaced { + audit.record(.mailboxReplaced, replaced.deliveryAuditIdentity, rtpTimestamp: replaced.rtpTimestamp) + } + } + } +} + +/// Delegate callbacks originate on WebRTC threads. Hooks are installed before +/// negotiation and only hop to the main actor; their storage is then immutable. +private final class ScreenSharingPeerDelegate: NSObject, RTCPeerConnectionDelegate, @unchecked Sendable { + var onGathered: (@Sendable () -> Void)? + var onConnection: (@Sendable (String) -> Void)? + var onVideoTrack: (@Sendable (RTCVideoTrack) -> Void)? + func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {} + func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) { + if newState == .complete { onGathered?() } + } + func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {} + func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) { dataChannel.close() } + func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCPeerConnectionState) { + let names = ["new", "connecting", "connected", "disconnected", "failed", "closed"] + onConnection?(names[min(max(0, newState.rawValue), names.count - 1)]) + } + func peerConnection( + _ peerConnection: RTCPeerConnection, didAdd rtpReceiver: RTCRtpReceiver, streams: [RTCMediaStream] + ) { + if let track = rtpReceiver.track as? RTCVideoTrack { onVideoTrack?(track) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRTCIntervalMetrics.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRTCIntervalMetrics.swift new file mode 100644 index 000000000..057759192 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRTCIntervalMetrics.swift @@ -0,0 +1,41 @@ +import Foundation + +/// W3C cumulative seconds/count pairs become weighted interval means, not +/// per-frame percentiles. Baseline each stream independently across resets. +public struct ScreenSharingRTCIntervalMetrics: Sendable { + private var previous: [String: String] = [:] + public init() {} + + public mutating func update(_ statistics: [String: String]) -> [String: Double] { + defer { previous = statistics } + let pairs = [ + ("inbound-rtp", "jitterBufferDelay", "jitterBufferEmittedCount", "jitterBufferMeanMs"), + ("inbound-rtp", "jitterBufferTargetDelay", "jitterBufferEmittedCount", "jitterBufferTargetMeanMs"), + ("inbound-rtp", "jitterBufferMinimumDelay", "jitterBufferEmittedCount", "jitterBufferMinimumMeanMs"), + ("inbound-rtp", "totalDecodeTime", "framesDecoded", "rtcDecodeMeanMs"), + ("inbound-rtp", "totalProcessingDelay", "framesDecoded", "receiveToDecodeMeanMs"), + ("outbound-rtp", "totalEncodeTime", "framesEncoded", "rtcEncodeMeanMs"), + ("outbound-rtp", "totalPacketSendDelay", "packetsSent", "packetSendDelayMeanMs"), + ] + var result: [String: Double] = [:] + func number(_ values: [String: String], _ key: String) -> Double? { + guard let raw = values[key], let value = Double(raw), value.isFinite, value >= 0 else { return nil } + return value + } + for (type, total, count, name) in pairs { + var seconds = 0.0 + var observations = 0.0 + for key in statistics.keys where key.hasPrefix(type + ".") && key.hasSuffix("." + total) { + let countKey = String(key.dropLast(total.count)) + count + guard let currentTotal = number(statistics, key), let oldTotal = number(previous, key), + let currentCount = number(statistics, countKey), let oldCount = number(previous, countKey), + currentTotal >= oldTotal, currentCount > oldCount + else { continue } + seconds += currentTotal - oldTotal + observations += currentCount - oldCount + } + if observations > 0 { result[name] = seconds / observations * 1000 } + } + return result + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRenderCoordinator.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRenderCoordinator.swift new file mode 100644 index 000000000..b7d89ea59 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRenderCoordinator.swift @@ -0,0 +1,363 @@ +import CoreGraphics +import Foundation +import QuartzCore + +/// The GPU/compositor side of one render submission. The real adapter wraps an +/// `MTLCommandBuffer` and its `CAMetalDrawable`; tests supply a controlled one. +/// Handlers are installed before `commit()`; each is invoked at most once, on +/// any thread. +package protocol ScreenSharingRenderSubmission: Sendable { + /// Invoked when the GPU work finished (`completed` = status `.completed`). + func onCompleted(_ handler: @escaping @Sendable (_ completed: Bool) -> Void) + /// Invoked when the drawable is presented, or resolved unpresented (`presentedTime` 0). + func onPresented(_ handler: @escaping @Sendable (_ presentedTime: Double) -> Void) + func commit() +} + +/// Immutable main-actor values a worker needs to size the layer and encode a +/// frame; never AppKit. `drawableSize` is the view's backing size at request +/// time: the worker applies it to the layer right before acquiring, so layer +/// sizing and acquisition are serialized on the worker without any lock the +/// main actor could wait on. +package struct ScreenSharingRenderGeometry: Equatable, Sendable { + package let fitToWindow: Bool + package let clearColor: SIMD4 + package let drawableSize: CGSize + package init(fitToWindow: Bool, clearColor: SIMD4, drawableSize: CGSize) { + self.fitToWindow = fitToWindow + self.clearColor = clearColor + self.drawableSize = drawableSize + } +} + +/// One off-main preparation: the selected frame plus its geometry snapshot. +package struct ScreenSharingPreparationRequest: Sendable { + package let frame: ScreenSharingVideoFrame + package let isNewFrame: Bool + package let geometry: ScreenSharingRenderGeometry + /// When the main actor queued the request (worker queue-wait telemetry only). + package let queuedAtNs: Int64 + /// Receiver-only diagnostic identity (nil when the audit is off or the frame carried none). + package let auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? +} + +/// A prepared (acquired + encoded, not committed) submission returned to the +/// main actor. `retained` is the pixel buffer with its Metal textures. +package struct ScreenSharingPreparedSubmission: Sendable { + package let submission: any ScreenSharingRenderSubmission + package let retained: AnyObject & Sendable + package let videoSize: CGSize + package init(submission: any ScreenSharingRenderSubmission, retained: AnyObject & Sendable, videoSize: CGSize) { + self.submission = submission + self.retained = retained + self.videoSize = videoSize + } +} + +/// Diagnostic off-main preparation: acquires a drawable and encodes on its own +/// serial worker, then hands the result back exactly once. The real preparer +/// uses `CAMetalLayer.nextDrawable`; tests supply a controlled one. +package protocol ScreenSharingRenderPreparer: Sendable { + func prepare( + _ request: ScreenSharingPreparationRequest, + completion: @escaping @Sendable (ScreenSharingPreparedSubmission?) -> Void) +} + +/// Main-actor scheduling and ownership state of the Metal renderer, used by +/// `ScreenSharingMetalView` for every draw. Frame selection, completion pacing +/// and the retained-until-completion rule are unchanged; `stop()` is the one +/// terminal boundary after which nothing is scheduled, cached, submitted or notified. +@MainActor +package final class ScreenSharingRenderCoordinator { + package typealias Hop = @Sendable (_ work: @escaping @Sendable @MainActor () -> Void) -> Void + + package let mailbox: ScreenSharingFrameMailbox + package let metrics: ScreenSharingMetrics + package let renderOnArrival: Bool + /// Reported once per video size change; cleared by `stop()`. + package var onFrameSize: ((CGSize) -> Void)? { + didSet { if stopped { onFrameSize = nil } } + } + /// Product presentation notification; cleared by `stop()`. + package var onPresented: ((UInt32) -> Void)? { + didSet { if stopped { onPresented = nil } } + } + /// The single slot: true while a frame is being prepared or its submission is in flight. + package private(set) var inFlight = false + /// Set while the slot is held by an off-main preparation (nil otherwise). + package var preparationToken: Int? { pending?.token } + private struct PendingPreparation { + let token: Int + let rtpTimestamp: UInt32 + let receivedAt: Double? + let isNewFrame: Bool + let auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? + } + /// Receiver-only diagnostic frame-delivery audit (nil = disabled: every hook is one nil branch). + package var audit: ScreenSharingFrameDeliveryAudit? + /// Scalars of the frame being prepared, kept here so the worker never has + /// to echo them back. Survives `stop()`: the executing preparation still + /// returns once, and that return clears the slot. + private var pending: PendingPreparation? + /// Off-main mode: a redraw request (resize, backing scale, fit) schedules a + /// draw through the hop when the renderer is idle, instead of waiting for + /// the next video frame. Never synchronous from the caller. + private let redrawsOnDemand: Bool + /// One scheduled redraw hop at a time: a burst of resize/fit changes + /// coalesces into a single draw request. + private var redrawHopPending = false + package private(set) var stopped = false + /// Ownership observation only: the cached redraw frame is held. + package var isHoldingCachedFrame: Bool { lastFrame != nil } + private var lastFrame: ScreenSharingVideoFrame? + private var needsRedraw = false + private var reportedSize = CGSize.zero + private var requestDraw: @MainActor () -> Void = {} + private let hop: Hop + /// Incremented whenever the slot is taken; a completion or preparation + /// result only counts for the occupancy it was created under. + private var occupancy = 0 + + /// `hop` moves queued work (arrival, completion, presentation, prepared + /// results) onto the main actor; the default is a Task. Tests supply a + /// controlled queue. + package init( + mailbox: ScreenSharingFrameMailbox, metrics: ScreenSharingMetrics, renderOnArrival: Bool, + redrawsOnDemand: Bool = false, + hop: @escaping Hop = { work in Task { @MainActor in work() } } + ) { + self.mailbox = mailbox + self.metrics = metrics + self.renderOnArrival = renderOnArrival + self.redrawsOnDemand = redrawsOnDemand + self.hop = hop + } + + /// Installs the view's draw request and, for the arrival scheduler, + /// subscribes to frame arrival (a frame already waiting is scheduled at once, + /// mailbox semantics). Ignored once stopped. + package func bind(requestDraw: @escaping @MainActor () -> Void) { + guard !stopped else { return } + self.requestDraw = requestDraw + guard renderOnArrival else { return } + let hop = hop + mailbox.onFrameAvailable { [weak self] in hop { self?.arrivalScheduled() } } + } + + /// Main-actor entry of a queued arrival notification. + package func arrivalScheduled() { + guard !stopped else { return } + requestDraw() + } + + /// A resize or fit change asks for one redraw of the cached frame. With + /// `redrawsOnDemand` the draw is scheduled through the hop (coalesced by the + /// flag: later hops find nothing to do); never a synchronous reentrant draw. + package func setNeedsRedraw() { + guard !stopped else { return } + needsRedraw = true + guard redrawsOnDemand, !redrawHopPending else { return } + redrawHopPending = true + let hop = hop + hop { [weak self] in self?.redrawScheduled() } + } + + /// Main-actor entry of a scheduled redraw: draws only when idle and still + /// needed (a redraw needed while the slot is held is picked up by the + /// completion-driven draw request). + package func redrawScheduled() { + redrawHopPending = false + guard !stopped, needsRedraw, !inFlight else { return } + requestDraw() + } + + /// Frame selection, unchanged: nothing while stopped or while the slot is + /// held; the incoming frame first, else the cached frame for a redraw. The + /// selected frame becomes the cached frame before encoding, as before. + package func select() -> (frame: ScreenSharingVideoFrame, isNewFrame: Bool)? { + guard !stopped, !inFlight else { return nil } + let incoming = mailbox.take() + guard let frame = incoming ?? (needsRedraw ? lastFrame : nil) else { return nil } + lastFrame = frame + needsRedraw = false + if let audit, incoming != nil { + audit.record(.selection, frame.deliveryAuditIdentity, rtpTimestamp: frame.rtpTimestamp) + } + return (frame, incoming != nil) + } + + package func reportSize(_ size: CGSize) { + guard !stopped, size != reportedSize else { return } + reportedSize = size + onFrameSize?(size) + metrics.label("videoSize", "\(Int(size.width)) × \(Int(size.height))") + } + + /// Main-actor path: installs the completion and presentation handlers and + /// commits an already encoded submission. `retained` (the pixel buffer and + /// its Metal textures) is held by the completion handler until the GPU + /// actually finished — also after `stop()`. The presentation handler + /// captures only the scalars it reports. Refused (false, nothing installed, + /// nothing retained, nothing submitted) once stopped — a callback between + /// `select()` and here may have stopped the renderer — or while the slot is + /// held by another submission or preparation. + @discardableResult + package func commit( + _ submission: any ScreenSharingRenderSubmission, retaining retained: AnyObject & Sendable, + frame: ScreenSharingVideoFrame, + isNewFrame: Bool, submittedAt: Double + ) -> Bool { + guard !stopped, !inFlight else { return false } + inFlight = true + occupancy += 1 + submit( + submission, retaining: retained, rtpTimestamp: frame.rtpTimestamp, receivedAt: frame.receivedAtSeconds, + isNewFrame: isNewFrame, submittedAt: submittedAt, auditIdentity: frame.deliveryAuditIdentity) + return true + } + + /// Off-main diagnostic path: selects with the unchanged ordering, reserves + /// the single slot, snapshots the geometry and hands the frame to the + /// preparer. Exactly one preparation or submission is outstanding; a frame + /// arriving meanwhile only replaces the mailbox's newest frame. Returns + /// false when nothing was selected or the slot is held. + @discardableResult + package func prepare(with preparer: any ScreenSharingRenderPreparer, geometry: ScreenSharingRenderGeometry) -> Bool { + guard let (frame, isNewFrame) = select() else { return false } + inFlight = true + occupancy += 1 + let token = occupancy + pending = .init( + token: token, rtpTimestamp: frame.rtpTimestamp, receivedAt: frame.receivedAtSeconds, isNewFrame: isNewFrame, + auditIdentity: frame.deliveryAuditIdentity) + let hop = hop + preparer.prepare( + .init( + frame: frame, isNewFrame: isNewFrame, geometry: geometry, queuedAtNs: ScreenSharingMetrics.nowNs, + auditIdentity: frame.deliveryAuditIdentity) + ) { [weak self] prepared in + hop { self?.finishPreparation(token: token, prepared) } + } + return true + } + + /// Main-actor entry of a preparation result. Stale results (another + /// occupancy, or a token already consumed) are ignored. The held slot is + /// always cleared by the actual return of its preparation: after `stop()` + /// the result is dropped uncommitted and the slot cleared without + /// rescheduling; a failed preparation releases the slot; the size callback + /// may stop the renderer, in which case the prepared result is dropped and + /// the slot cleared as well; otherwise the prepared submission is committed. + package func finishPreparation(token: Int, _ prepared: ScreenSharingPreparedSubmission?) { + guard let pending, pending.token == token, occupancy == token else { return } + self.pending = nil + if let audit { + audit.record( + .preparedReceipt, pending.auditIdentity, rtpTimestamp: pending.rtpTimestamp, valueNs: prepared == nil ? 0 : 1) + } + guard !stopped else { + inFlight = false // the worker's result is dropped here; nothing else will clear the slot + return + } + guard let prepared else { + metrics.increment("renderDrops") + releaseSlot(occupancy: token) + return + } + reportSize(prepared.videoSize) + guard !stopped else { + inFlight = false // prepared but never submitted: no GPU completion will clear it + return + } + submit( + prepared.submission, retaining: prepared.retained, rtpTimestamp: pending.rtpTimestamp, + receivedAt: pending.receivedAt, isNewFrame: pending.isNewFrame, submittedAt: CACurrentMediaTime(), + auditIdentity: pending.auditIdentity) + } + + /// Final submission boundary for both paths: `gpuSubmissionToCompletion` + /// starts here (never during acquisition or preparation), and + /// `receiverCallbackToSubmission` is observed exactly once per new frame. + private func submit( + _ submission: any ScreenSharingRenderSubmission, retaining retained: AnyObject & Sendable, + rtpTimestamp: UInt32, receivedAt: Double?, isNewFrame: Bool, submittedAt: Double, + auditIdentity: ScreenSharingFrameDeliveryAudit.Identity? = nil + ) { + let metrics = metrics + let hop = hop + let renderOnArrival = renderOnArrival + let mine = occupancy + let startedNs = ScreenSharingMetrics.nowNs + let audit = audit + if isNewFrame, let receivedAt { + metrics.observe("receiverCallbackToSubmission", milliseconds: (submittedAt - receivedAt) * 1000) + } + if let audit, isNewFrame { audit.record(.submission, auditIdentity, rtpTimestamp: rtpTimestamp) } + submission.onCompleted { [weak self, retained, audit, auditIdentity] completed in + withExtendedLifetime(retained) {} + if let audit, isNewFrame { + audit.record(.gpuCompletionCallback, auditIdentity, rtpTimestamp: rtpTimestamp, valueNs: completed ? 1 : 0) + } + if completed { + if isNewFrame { metrics.increment("renderedFrames") } + metrics.observe( + "gpuSubmissionToCompletion", milliseconds: Double(ScreenSharingMetrics.nowNs - startedNs) / 1_000_000) + } else { + metrics.increment("renderErrors") + } + hop { + guard let self, self.occupancy == mine, self.inFlight else { return } + self.inFlight = false + guard !self.stopped, renderOnArrival else { return } + self.requestDraw() + } + } + submission.onPresented { + [weak self, metrics, hop, isNewFrame, submittedAt, receivedAt, rtpTimestamp, audit, auditIdentity] presentedTime + in + if let audit, isNewFrame { + audit.record( + .presentedResult, auditIdentity, rtpTimestamp: rtpTimestamp, + valueNs: presentedTime.isFinite && presentedTime > 0 ? Int64(presentedTime * 1_000_000_000) : 0) + } + guard + metrics.recordPresentation( + isNewFrame: isNewFrame, presentedAt: presentedTime, submittedAt: submittedAt, receivedAt: receivedAt) + else { return } + hop { + guard let self, !self.stopped else { return } + self.onPresented?(rtpTimestamp) + } + } + submission.commit() + } + + private func releaseSlot(occupancy mine: Int) { + guard occupancy == mine, inFlight else { return } + inFlight = false + guard !stopped, renderOnArrival else { return } + requestDraw() + } + + /// Terminal and idempotent. Releases immediately: the draw request, the + /// arrival subscription, the mailbox frame, the cached redraw frame, the + /// pending redraw and both product callbacks. Not released here: a + /// submission in flight keeps its buffer and textures until its own + /// completion handler runs (that handler then only clears the slot); a + /// preparation already executing on its worker finishes or fails on its + /// own, cannot touch AppKit, and its one return clears the slot here with + /// the result dropped uncommitted (the reservation identity is kept for + /// that). Nothing joins WebRTC, VideoToolbox, GPU or worker threads. + package func stop() { + guard !stopped else { return } + stopped = true + requestDraw = {} + mailbox.onFrameAvailable(nil) + mailbox.clear() + lastFrame = nil + needsRedraw = false + onFrameSize = nil + onPresented = nil + metrics.label("rendererStopped", "true") + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRtcEventLogDiagnostic.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRtcEventLogDiagnostic.swift new file mode 100644 index 000000000..820011770 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingRtcEventLogDiagnostic.swift @@ -0,0 +1,333 @@ +import Foundation +import QuartzCore + +/// Role-tagged (receiver or sender), opt-in lifecycle for WebRTC's shipped RTC +/// event log (`startRtcEventLogWithFilePath:maxSizeInBytes:` / +/// `stopRtcEventLog`), a diagnostic. Disabled (nil) means nothing is allocated +/// and nothing is scheduled; the product path never creates one. The object +/// decides only WHEN the two synchronous API calls happen (from the probe's +/// existing measurement ticks) and records, immutably once final, the requested +/// window, the actual `CACurrentMediaTime()` nanoseconds read immediately +/// before and after each call ("bracket A"), the start call's Bool, whether a +/// successful start was stopped exactly once, and the close reason. It never +/// inspects the file: an absent end event stays an offline finding (stop +/// returns void) and a `true` start is an accepted output, not a complete file. +/// Packet timestamps are this role's own WebRTC logging point — never a socket +/// or NIC boundary — and the role note below states which point that is. +public final class ScreenSharingRtcEventLogDiagnostic { + /// Requested window in seconds relative to media start (measured seconds). + public struct Window: Equatable, Sendable { + public static let maximumDurationSeconds = 120.0 + public let beginSeconds: Double + public let durationSeconds: Double + public var endSeconds: Double { beginSeconds + durationSeconds } + public init(beginSeconds: Double, durationSeconds: Double) throws { + guard beginSeconds.isFinite, durationSeconds.isFinite, beginSeconds >= 0, + (1...Self.maximumDurationSeconds).contains(durationSeconds) + else { + throw ScreenSharingError.invalid( + "RTC event log window needs begin >= 0 and a duration of 1...\(Int(Self.maximumDurationSeconds)) seconds.") + } + self.beginSeconds = beginSeconds + self.durationSeconds = durationSeconds + } + } + + /// Injectable boundaries: the clock and the two synchronous peer calls. + public struct Boundaries { + public let clock: () -> Int64 + public let start: (_ path: String, _ maxSizeBytes: Int64) -> Bool + /// Returns true only when the native stop API was actually invoked (a closed peer reports false). + public let stop: () -> Bool + public init( + clock: @escaping () -> Int64, start: @escaping (String, Int64) -> Bool, stop: @escaping () -> Bool + ) { + self.clock = clock + self.start = start + self.stop = stop + } + } + + /// Which peer the log belongs to. A sender log stamps an outgoing packet after a successful `SendRtp`, which either + /// sends directly when it is already on the network thread or queues a task there, returning true either way: the + /// post-pacer TRANSPORT HAND-OFF, never socket or NIC egress. A receiver log stamps a packet where the video receiver + /// logs it, which is not NIC arrival. + public enum Role: String, Encodable, Sendable { + case receiver + case sender + } + + public enum State: String, Encodable, Sendable { + /// Before the first tick at or after the requested begin. + case waiting + /// Options requested the log but the probe failed before the receiver/log were constructed; no API call. + case notInitialized + /// The exclusive output reservation failed at start time (an entry appeared after preflight); no API call. + case startRefused + /// The start call returned true; the stop call has not happened. + case started + /// The start call returned false; no stop call will ever be made. + case startFailed + /// A successful start was stopped exactly once: the native stop API was invoked. + case stopped + /// A stop was attempted once but the boundary reported the native API was not invoked (peer already closed). + case stopSkipped + /// Closed (normal end, failure, cancellation or early close) before any start call, or the first + /// start opportunity was already past the requested end; no API call was made. + case closedWithoutStart + } + + /// The final record; identical in the report and the failure record. + public struct Record: Encodable, Equatable, Sendable { + public let kind: String + public let role: String + public let clock: String + public let path: String + public let maxSizeBytes: Int64 + public let requestedBeginSeconds: Double + public let requestedDurationSeconds: Double + public let requestedEndSeconds: Double + public let state: State + public let final: Bool + /// The exclusive owned-file reservation made immediately before the start API ("reserved" or the refusal). + public let outputReservation: String? + public let startApiInvoked: Bool + /// Measured seconds (the probe's media clock) at the tick that made the start call; nil = no call. + public let startAttemptedAtMeasuredSeconds: Double? + public let startCallBeforeNs: Int64? + public let startCallAfterNs: Int64? + public let startReturned: Bool? + public let stopAttemptedAtMeasuredSeconds: Double? + public let stopCallBeforeNs: Int64? + public let stopCallAfterNs: Int64? + /// A stop was attempted (bracket taken); `stopApiInvoked` says whether the native API actually ran. + public let stopAttempted: Bool + public let stopApiInvoked: Bool + public let stoppedSuccessfulStart: Bool + public let closeReason: String? + public let notes: [String] + + /// The record for a requested log whose diagnostic was never constructed (failure before the receiver/log). + public static func notInitialized( + beginSeconds: Double, durationSeconds: Double, path: String, maxSizeBytes: Int64, reason: String, + role: Role = .receiver + ) -> Record { + Record( + kind: ScreenSharingRtcEventLogDiagnostic.kind, role: role.rawValue, + clock: "CACurrentMediaTime nanoseconds (Int64)", path: path, + maxSizeBytes: maxSizeBytes, requestedBeginSeconds: beginSeconds, requestedDurationSeconds: durationSeconds, + requestedEndSeconds: beginSeconds + durationSeconds, state: .notInitialized, final: true, + outputReservation: nil, startApiInvoked: false, startAttemptedAtMeasuredSeconds: nil, startCallBeforeNs: nil, + startCallAfterNs: nil, startReturned: nil, stopAttemptedAtMeasuredSeconds: nil, stopCallBeforeNs: nil, + stopCallAfterNs: nil, stopAttempted: false, stopApiInvoked: false, stoppedSuccessfulStart: false, + closeReason: reason + "; no API call", notes: ScreenSharingRtcEventLogDiagnostic.notes(for: role)) + } + } + + public static let fixedMaxSizeBytes: Int64 = 8 * 1024 * 1024 + public static let kind = "codevisor.rtcEventLogDiagnostic" + /// The one supported clock: CACurrentMediaTime in nanoseconds (the audit, viewer origin and observer clock). + public static func defaultClock() -> Int64 { Int64(CACurrentMediaTime() * 1_000_000_000) } + public static let notes = [ + "bracket A: CACurrentMediaTime nanoseconds read immediately before and after the synchronous start and stop API calls; the pinned WebRTC start reads its own clock before posting the begin event and stop waits for the task that stamps the end event, so each bracket encloses that event's true instant", + "clock contract for the offline oracle: a nonnegative logged integer t_ms represents [t, t+1) ms of the log clock; a bracket bounds the log-to-CA offset to [beforeNs - (t+1)*1e6, afterNs - t*1e6]; start and stop bounds must each be finite, narrow and mutually consistent, never averaged", + "WebRTC keeps a bounded pre-start history that is written after the begin event: the file is not exclusively the requested window", + "startReturned true means the log accepted its output, not that the file is complete; stop returns void, so an absent end event is an offline finding (8 MiB cap reached, a write failure, or an abnormal stop)", + "the output path is reserved as an owned empty file (O_EXCL) immediately before the start API, which may then overwrite only that reserved file; an entry appearing after option preflight refuses the start and the API is not invoked; on a false start the empty reservation stays (it holds no evidence)", + "requested and actual bounds are recorded separately; a late tick or a missed stop is stated as such", + "packet timestamps are this role's own WebRTC logging point (see the role note), never a socket or NIC boundary; a marker bit is not a decodable frame; no FrameDecodedEvents join; parse success, byte-cap completeness and packet or audit coverage are separate offline judgements, not a media verdict", + ] + + /// Role-specific note appended to the role-neutral notes. + public static func notes(for role: Role) -> [String] { + switch role { + case .receiver: + notes + [ + "receiver log: incoming packet events are stamped where the video receiver logs the packet " + + "(RtpVideoStreamReceiver2::OnRtpPacket), which is not NIC arrival" + ] + case .sender: + notes + [ + "sender log: outgoing packet events are stamped after a successful SendRtp, which either sends directly when it is already on the network thread or queues a task there and returns true either way: the post-pacer TRANSPORT HAND-OFF, never socket/NIC egress; any network-thread queueing and send, the OS, the network and the receiver's ingress scheduling are downstream unknowns" + ] + } + } + + public let window: Window + public let path: String + public let maxSizeBytes: Int64 + public let role: Role + private let boundaries: Boundaries + private let lock = NSLock() + private var state = State.waiting + private var final = false + private var startAttemptedAt: Double? + private var startBefore: Int64? + private var startAfter: Int64? + private var startReturned: Bool? + private var stopAttemptedAt: Double? + private var stopBefore: Int64? + private var stopAfter: Int64? + private var stopInvoked = false + private var reservation: String? + private var closeReason: String? + + public init( + window: Window, path: String, maxSizeBytes: Int64 = ScreenSharingRtcEventLogDiagnostic.fixedMaxSizeBytes, + role: Role = .receiver, boundaries: Boundaries + ) { + self.window = window + self.path = path + self.maxSizeBytes = maxSizeBytes + self.role = role + self.boundaries = boundaries + } + + /// Option-time preflight: refuses any existing entry at the raw output path, + /// including a dangling symlink, and any lstat error other than plain + /// absence: prior evidence is never truncated (WebRTC opens the path "wb"). + public static func checkOutputPath(_ path: String) throws { + var status = stat() + if lstat(path, &status) == 0 { + throw ScreenSharingError.invalid( + "RTC event log output already exists (file, directory or symlink): \(path). Use a fresh report stem.") + } + guard errno == ENOENT else { + throw ScreenSharingError.invalid( + "RTC event log output path cannot be checked (\(String(cString: strerror(errno)))): \(path).") + } + } + + /// Start-time exclusive ownership: creates the raw output as an owned empty + /// file with O_CREAT|O_EXCL (fails on any existing entry, including a + /// dangling symlink) so the start API may overwrite only this reserved + /// file. Nothing is ever deleted. + public static func reserveOutputPath(_ path: String) throws { + let descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR) + guard descriptor >= 0 else { + throw ScreenSharingError.invalid( + "RTC event log output could not be reserved exclusively (\(String(cString: strerror(errno)))): \(path).") + } + _ = Darwin.close(descriptor) // the POSIX close, not this class's close(measuredSeconds:reason:) + } + + /// The measurement loop's sleep: at most `tickInterval`, at most the + /// remaining duration, and no later than the next requested boundary. + public static func sleepSeconds( + measured: Double, remaining: Double, nextBoundary: Double?, tickInterval: Double = 1 + ) -> Double { + var seconds = min(tickInterval, remaining) + if let nextBoundary, nextBoundary > measured { seconds = min(seconds, nextBoundary - measured) } + return max(0.001, seconds) + } + + /// The next measured second at which a tick has work: the requested begin + /// while waiting, the requested end while started, nil once final. + public func nextBoundarySeconds() -> Double? { + lock.withLock { + switch state { + case .waiting: window.beginSeconds + case .started: window.endSeconds + case .notInitialized, .startRefused, .startFailed, .stopped, .stopSkipped, .closedWithoutStart: nil + } + } + } + + /// One measurement tick. Makes at most one API call per tick: the start at + /// the first tick at or after the requested begin, the stop at the first + /// tick at or after the requested end. A first opportunity that is already + /// past the end never starts. + public func tick(measuredSeconds: Double) { + lock.withLock { + guard !final else { return } + switch state { + case .waiting where measuredSeconds >= window.endSeconds: + state = .closedWithoutStart + closeReason = + "first start opportunity at \(measuredSeconds) s was already past the requested end \(window.endSeconds) s; no start call" + final = true + case .waiting where measuredSeconds >= window.beginSeconds: + do { + try Self.reserveOutputPath(path) + reservation = "reserved: owned empty file created O_EXCL immediately before the start API" + } catch { + reservation = "refused: \(error.localizedDescription)" + state = .startRefused + closeReason = "output reservation failed at \(measuredSeconds) s; start API not invoked" + final = true + return + } + startAttemptedAt = measuredSeconds + startBefore = boundaries.clock() + let returned = boundaries.start(path, maxSizeBytes) + startAfter = boundaries.clock() + startReturned = returned + if returned { + state = .started + } else { + state = .startFailed + closeReason = "start returned false at \(measuredSeconds) s; no stop call" + final = true + } + case .started where measuredSeconds >= window.endSeconds: + stopLocked(measuredSeconds: measuredSeconds, reason: "requested window end") + default: + break + } + } + } + + /// Normal completion: stops a started log exactly once; a log never started is closed without a call. + public func finish(measuredSeconds: Double) { close(measuredSeconds: measuredSeconds, reason: "normal completion") } + + /// Failure, cancellation or early close: the same once-only rules; repeated calls change nothing. + public func closeEarly(measuredSeconds: Double?, reason: String) { + close(measuredSeconds: measuredSeconds, reason: "early close: " + reason) + } + + private func close(measuredSeconds: Double?, reason: String) { + lock.withLock { + guard !final else { return } + switch state { + case .started: + stopLocked(measuredSeconds: measuredSeconds, reason: reason) + case .waiting: + state = .closedWithoutStart + closeReason = reason + "; no start call was made" + final = true + case .notInitialized, .startRefused, .startFailed, .stopped, .stopSkipped, .closedWithoutStart: + break + } + } + } + + private func stopLocked(measuredSeconds: Double?, reason: String) { + stopAttemptedAt = measuredSeconds + stopBefore = boundaries.clock() + stopInvoked = boundaries.stop() + stopAfter = boundaries.clock() + if stopInvoked { + state = .stopped + closeReason = reason + } else { + state = .stopSkipped + closeReason = reason + "; the stop boundary reported the native API was not invoked (peer already closed)" + } + final = true + } + + public var record: Record { + lock.withLock { + Record( + kind: Self.kind, role: role.rawValue, clock: "CACurrentMediaTime nanoseconds (Int64)", path: path, + maxSizeBytes: maxSizeBytes, + requestedBeginSeconds: window.beginSeconds, requestedDurationSeconds: window.durationSeconds, + requestedEndSeconds: window.endSeconds, state: state, final: final, outputReservation: reservation, + startApiInvoked: startBefore != nil, + startAttemptedAtMeasuredSeconds: startAttemptedAt, startCallBeforeNs: startBefore, startCallAfterNs: startAfter, + startReturned: startReturned, stopAttemptedAtMeasuredSeconds: stopAttemptedAt, stopCallBeforeNs: stopBefore, + stopCallAfterNs: stopAfter, stopAttempted: stopBefore != nil, stopApiInvoked: stopInvoked, + stoppedSuccessfulStart: state == .stopped, closeReason: closeReason, notes: Self.notes(for: role)) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingScrollAccumulator.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingScrollAccumulator.swift new file mode 100644 index 000000000..9f2f106ca --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingScrollAccumulator.swift @@ -0,0 +1,16 @@ +import Foundation + +/// Preserve fractional trackpad movement when sending integral pixel deltas. +public struct ScreenSharingScrollAccumulator { + private var x = 0.0 + private var y = 0.0 + public init() {} + public mutating func add(x: Double, y: Double) -> (x: Int32, y: Int32) { + guard x.isFinite, y.isFinite else { return (0, 0) } + self.x = min(4096, max(-4096, self.x + x)) + self.y = min(4096, max(-4096, self.y + y)) + let dx = Int32(self.x); let dy = Int32(self.y) + self.x -= Double(dx); self.y -= Double(dy) + return (dx, dy) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceIdle.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceIdle.swift new file mode 100644 index 000000000..3bfb2ae42 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceIdle.swift @@ -0,0 +1,428 @@ +import Foundation + +/// Host capture activity, shared by the capture queue (submissions), +/// VideoToolbox's output queue (encoded output) and the main actor +/// (evaluation). Trailing packet loss before an idle desktop is invisible to +/// RTP sequence feedback, so the host announces idle once with the identity +/// of its newest encoded content. Content that was submitted but never +/// encoded (admission, a WebRTC bandwidth pause, or an adaptation drop) is +/// re-offered: four times at the idle threshold, then at a slow bounded +/// rate until it is encoded, a newer capture arrives or the peer closes. +/// Older content is never announced as current. There is no periodic +/// re-encode: a re-offer costs an encode only when WebRTC accepts it. +final class ScreenSharingSourceIdleMonitor: @unchecked Sendable { + enum Evaluation: Equatable, Sendable { + case inactive + case wait(untilNs: Int64) + case resubmit + case idle(latestEncodedNs: Int64) + } + + /// Product default, promoted after the brief-loss measurements: 100 ms of + /// capture silence announces idle. The former 500 ms remains an explicit + /// diagnostic override. + static let defaultThresholdNs: Int64 = 100_000_000 + static let legacyThresholdNs: Int64 = 500_000_000 + static let maximumQuickResubmissions = 4 + static let slowResubmissionIntervalNs: Int64 = 2_000_000_000 + let thresholdNs: Int64 + private let slowIntervalNs: Int64 + private let lock = NSLock() + private var lastSubmittedNs: Int64? + private var lastSubmissionAtNs: Int64? + private var lastEncodedNs: Int64? + private var armed = false + private var resubmissions = 0 + private var stopped = false + + init( + thresholdNs: Int64 = ScreenSharingSourceIdleMonitor.defaultThresholdNs, + slowIntervalNs: Int64 = ScreenSharingSourceIdleMonitor.slowResubmissionIntervalNs + ) { + precondition(thresholdNs > 0 && slowIntervalNs > 0) + self.thresholdNs = thresholdNs + // The slow phase never re-offers faster than the quick phase. + self.slowIntervalNs = max(slowIntervalNs, thresholdNs) + } + + var slowResubmissionIntervalNs: Int64 { slowIntervalNs } + + /// Re-offers since the latest capture; at least the quick maximum means the + /// host is in its slow recovery phase. + var resubmissionCount: Int { lock.withLock { resubmissions } } + + /// Returns true only on the idle-to-active transition, which schedules one + /// evaluation loop. Later submissions during activity cost a lock only. + func recordSubmission(timestampNs: Int64, nowNs: Int64) -> Bool { + lock.withLock { + guard !stopped else { return false } + lastSubmittedNs = timestampNs + lastSubmissionAtNs = nowNs + resubmissions = 0 + guard !armed else { return false } + armed = true + return true + } + } + + /// Output delivered to WebRTC; discarded or failed output is never recorded. + func recordEncoded(timestampNs: Int64) { + lock.withLock { + if let lastEncodedNs, timestampNs <= lastEncodedNs { return } + lastEncodedNs = timestampNs + } + } + + var latestEncodedNs: Int64? { lock.withLock { lastEncodedNs } } + var latestSubmittedNs: Int64? { lock.withLock { lastSubmittedNs } } + + /// Evaluations performed (each is one wake of the notifier loop). + var evaluationCount: Int { lock.withLock { evaluations } } + private var evaluations = 0 + + func evaluate(nowNs: Int64) -> Evaluation { + lock.withLock { + evaluations += 1 + guard armed, !stopped, let submittedAt = lastSubmissionAtNs, let submitted = lastSubmittedNs else { + armed = false + return .inactive + } + let interval = resubmissions < Self.maximumQuickResubmissions ? thresholdNs : slowIntervalNs + if nowNs < submittedAt + interval { return .wait(untilNs: submittedAt + interval) } + if let encoded = lastEncodedNs, encoded >= submitted { + armed = false + return .idle(latestEncodedNs: encoded) + } + resubmissions += 1 + lastSubmissionAtNs = nowNs + return .resubmit + } + } + + func stop() { + lock.withLock { + stopped = true + armed = false + } + } +} + +/// Runs only while capture is active: one wake per threshold and none while +/// idle. The capture queue activates it once per activity period. A notice +/// that cannot be sent yet is retained (newest wins) until the channel opens. +@MainActor +final class ScreenSharingSourceIdleNotifier { + private let monitor: ScreenSharingSourceIdleMonitor + private let evaluated: () -> Void + private let resubmit: () -> Void + private let notify: (Int64) -> Bool + private let nowNs: @Sendable () -> Int64 + private let sleep: @Sendable (Duration) async throws -> Void + private var task: Task? + private var pendingNotice: Int64? + private var closed = false + + init( + monitor: ScreenSharingSourceIdleMonitor, evaluated: @escaping () -> Void = {}, + resubmit: @escaping () -> Void, notify: @escaping (Int64) -> Bool, + nowNs: @escaping @Sendable () -> Int64 = { ScreenSharingMetrics.nowNs }, + sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) } + ) { + self.monitor = monitor; self.evaluated = evaluated; self.resubmit = resubmit; self.notify = notify + self.nowNs = nowNs; self.sleep = sleep + } + + var hasPendingNotice: Bool { pendingNotice != nil } + + func activate() { + guard !closed, task == nil else { return } + task = Task { @MainActor [weak self] in await self?.run() } + } + + /// Retry a retained notice, e.g. when the channel becomes available. + func flush() { + guard !closed, let pendingNotice, notify(pendingNotice) else { return } + self.pendingNotice = nil + } + + private func run() async { + defer { task = nil } + while !Task.isCancelled, !closed { + evaluated() + switch monitor.evaluate(nowNs: nowNs()) { + case .inactive: + return + case .wait(let untilNs): + do { try await sleep(.nanoseconds(max(0, untilNs - nowNs()))) } catch { return } + case .resubmit: + resubmit() + case .idle(let latestEncodedNs): + pendingNotice = latestEncodedNs + flush() + } + } + } + + @discardableResult + func close() -> Task? { + closed = true + pendingNotice = nil + let pending = task + task = nil + pending?.cancel() + return pending + } +} + +/// Handles of a peer's cancelled owned tasks, retained until they complete so +/// that every waiter — concurrent or repeated — joins the same completions. +@MainActor +final class ScreenSharingOwnedWork { + private var handles: [Task]? + + /// Nil until `close(with:)`; then the number of retained handles. + var count: Int? { handles?.count } + + func close(with tasks: [Task]) { + guard handles == nil else { return } + handles = tasks + } + + /// Awaits every retained handle. Returns the number awaited, or nil when the + /// work was never closed (the call then returns immediately). + func join() async -> Int? { + guard let handles else { return nil } + for task in handles { await task.value } + return handles.count + } +} + +/// Viewer record of the newest decoded content identity and the host's +/// announced target. The target survives recovery: a delayed older keyframe +/// can complete the keyframe request without delivering the announced content. +final class ScreenSharingDeliveryAudit: @unchecked Sendable { + private let lock = NSLock() + private var latestDecodedNs: Int64? + private var targetNs: Int64? + private var recovering = false + private var targetMetAtNs: Int64? + /// Bounded recent decode times so a target that arrived before its notice + /// still has a first-decode time (diagnostic only; one second at 60 fps). + private var recentDecodes: [(identity: Int64, atNs: Int64)] = [] + static let recentDecodeCapacity = 64 + + var latestDecodedTimestampNs: Int64? { lock.withLock { latestDecodedNs } } + var pendingTargetNs: Int64? { lock.withLock { targetNs } } + /// Viewer clock at which the decoder first delivered the announced target + /// (or newer content), whether before the notice, during grace or after recovery. + var targetMetAtTimestampNs: Int64? { lock.withLock { targetMetAtNs } } + + func decoded(sourceTimestampNs: Int64, nowNs: Int64 = ScreenSharingMetrics.nowNs) { + lock.withLock { + if let latestDecodedNs, sourceTimestampNs <= latestDecodedNs { return } + latestDecodedNs = sourceTimestampNs + if recentDecodes.count == Self.recentDecodeCapacity { recentDecodes.removeFirst() } + recentDecodes.append((sourceTimestampNs, nowNs)) + if let targetNs, sourceTimestampNs >= targetNs, targetMetAtNs == nil { targetMetAtNs = nowNs } + } + } + + /// True when the announced content (or newer) has already been decoded. + func noticed(latestTimestampNs: Int64) -> Bool { + lock.withLock { + recovering = false + targetMetAtNs = nil + if let latestDecodedNs, latestDecodedNs >= latestTimestampNs { + targetNs = nil + targetMetAtNs = recentDecodes.first { $0.identity >= latestTimestampNs }?.atNs + return true + } + targetNs = latestTimestampNs + return false + } + } + + /// True when the target is still unmet after the grace period; recovery + /// then starts and the target remains until it is met or abandoned. + func verify() -> Bool { + lock.withLock { + guard let targetNs, !recovering else { return false } + if let latestDecodedNs, latestDecodedNs >= targetNs { + self.targetNs = nil + return false + } + recovering = true + return true + } + } + + /// After a keyframe completed recovery: nil when nothing was awaited, false + /// when the target is now met (and cleared), true when it is still missing. + func recoveryFinished() -> Bool? { + lock.withLock { + guard let targetNs, recovering else { return nil } + if let latestDecodedNs, latestDecodedNs >= targetNs { + self.targetNs = nil + recovering = false + return false + } + return true + } + } + + /// Before a retry: true while the target is still missing; a target met + /// meanwhile (for example by a delta frame) is cleared instead. + func targetMissing() -> Bool { + lock.withLock { + guard let targetNs else { return false } + if let latestDecodedNs, latestDecodedNs >= targetNs { + self.targetNs = nil + recovering = false + return false + } + return true + } + } +} + +/// Waits one grace period for frames still in the jitter buffer, then asks +/// for a keyframe only if the announced content never arrived. Optionally +/// (diagnostic experiment; the product default allows no extension) the wait +/// extends by further grace windows while strictly newer content keeps +/// arriving, up to a fixed cap, and recovery is requested once progress +/// stalls; duplicated or older frames never extend it. When recovery +/// completes without that content (a delayed older keyframe), it asks again +/// after an exponentially growing, capped delay, until the content arrives +/// or the peer closes; stale content is never silently accepted. A newer +/// notice replaces pending work; no timer runs otherwise. +@MainActor +final class ScreenSharingDeliveryVerifier { + enum Outcome: Equatable, Sendable { + case verified + case verifiedAfterGrace + case graceExtended + case refresh + case recovered + case retry + /// A scheduled retry ran and found the target still missing; a refresh follows. + case retryExecuted + } + + /// Product defaults, promoted after the brief-loss measurements: a 100 ms + /// grace extended up to four more windows while newer content keeps + /// arriving (at most 500 ms after the notice). The former fixed 500 ms grace + /// with no extension remains an explicit diagnostic override. + static let defaultGrace: Duration = .milliseconds(100) + static let defaultGraceExtensions = 4 + static let legacyGrace: Duration = .milliseconds(500) + static let legacyGraceExtensions = 0 + static let maximumGraceExtensions = 10 + nonisolated static let initialRetryDelayMs: Int64 = 100 + nonisolated static let maximumRetryDelayMs: Int64 = 5000 + private let audit: ScreenSharingDeliveryAudit + private let grace: Duration + private let graceExtensions: Int + private let refresh: () -> Void + private let report: (Outcome) -> Void + private let sleep: @Sendable (Duration) async throws -> Void + private var task: Task? + private var retryTask: Task? + private var revision: UInt64 = 0 + private var retries = 0 + private var closed = false + + /// Delay before retry number `retry` (1-based): 100 ms doubling to 5 s. + nonisolated static func retryDelay(_ retry: Int) -> Duration { + .milliseconds(min(initialRetryDelayMs << min(max(retry - 1, 0), 10), maximumRetryDelayMs)) + } + + init( + audit: ScreenSharingDeliveryAudit, grace: Duration = ScreenSharingDeliveryVerifier.defaultGrace, + graceExtensions: Int = ScreenSharingDeliveryVerifier.defaultGraceExtensions, + refresh: @escaping () -> Void, report: @escaping (Outcome) -> Void, + sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) } + ) { + self.audit = audit; self.grace = grace; self.refresh = refresh; self.report = report + self.graceExtensions = min(max(graceExtensions, 0), Self.maximumGraceExtensions) + self.sleep = sleep + } + + func noticed(latestTimestampNs: Int64) { + guard !closed else { return } + task?.cancel() + task = nil + retryTask?.cancel() + retryTask = nil + retries = 0 + if audit.noticed(latestTimestampNs: latestTimestampNs) { + report(.verified) + return + } + let sleep = sleep + let grace = grace + let extensions = graceExtensions + let target = latestTimestampNs + var mark = audit.latestDecodedTimestampNs + task = Task { @MainActor [weak self] in + var extended = 0 + while true { + do { try await sleep(grace) } catch { return } + guard let self, !Task.isCancelled, !self.closed else { return } + let latest = self.audit.latestDecodedTimestampNs + // Strictly newer content still short of the target is progress; the + // audit ignores duplicates and older frames, so `latest` only advances. + guard extended < extensions, let latest, latest < target, latest != mark else { break } + mark = latest + extended += 1 + self.report(.graceExtended) + } + guard let self, !Task.isCancelled, !self.closed else { return } + self.task = nil + if self.audit.verify() { + self.report(.refresh) + self.refresh() + } else { + self.report(.verifiedAfterGrace) + } + } + } + + /// Keyframe recovery state from the decoder signal, in revision order. + func recoveryChanged(_ event: ScreenSharingRefreshSignal.Event) { + guard !closed, event.revision > revision else { return } + revision = event.revision + guard !event.needed, let missing = audit.recoveryFinished() else { return } + guard missing else { + retries = 0 + report(.recovered) + return + } + retries += 1 + report(.retry) + let sleep = sleep + let delay = Self.retryDelay(retries) + retryTask?.cancel() + retryTask = Task { @MainActor [weak self] in + do { try await sleep(delay) } catch { return } + guard let self, !Task.isCancelled, !self.closed else { return } + self.retryTask = nil + if self.audit.targetMissing() { + self.report(.retryExecuted) + self.refresh() + } else { + self.report(.recovered) + } + } + } + + @discardableResult + func close() -> Task? { + closed = true + let pending = [task, retryTask].compactMap { $0 } + task = nil + retryTask = nil + for task in pending { task.cancel() } + guard !pending.isEmpty else { return nil } + return Task { for task in pending { await task.value } } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceMarker.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceMarker.swift new file mode 100644 index 000000000..d7831f04b --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingSourceMarker.swift @@ -0,0 +1,72 @@ +import Foundation + +/// Every encoded frame carries its content identity (the capture timestamp) +/// in a user-data SEI NAL unit, so the viewer can confirm that it decoded the +/// host's newest picture. WebRTC's RTP timestamps gain a random per-session +/// offset and frame counts diverge whenever the viewer's jitter buffer drops a +/// late frame; the marker is exact. Its bytes never include zero, so no +/// start-code emulation can occur, and the decoder strips it before +/// VideoToolbox sees the frame. +enum ScreenSharingSourceMarker { + /// uuid_iso_iec_11578 identifying Codevisor frame markers; no zero bytes. + static let identifier: [UInt8] = [ + 0x9c, 0x2e, 0x7b, 0x41, 0xd5, 0x6a, 0x48, 0x1f, 0xb7, 0x93, 0x5e, 0xc4, 0x2d, 0x8a, 0x61, 0xf3, + ] + private static let userDataUnregistered: UInt8 = 5 + /// Seven payload bits per byte with the high bit set: 70 bits, never zero. + /// The leading group is always 0x80 for a canonical non-negative value. + private static let valueByteCount = 10 + private static let payloadSize = identifier.count + valueByteCount + private static let trailingBits: UInt8 = 0x80 + + /// H.264 SEI (nal_ref_idc 0, type 6); HEVC prefix SEI (type 39, layer 0, TID 1). + static func header(for codec: ScreenSharingVideoCodec) -> [UInt8] { + codec == .h264 ? [0x06] : [0x4e, 0x01] + } + + static func nalUnit(timestampNs: Int64, codec: ScreenSharingVideoCodec) -> Data { + var unit = Data(header(for: codec)) + unit.append(userDataUnregistered) + unit.append(UInt8(payloadSize)) + unit.append(contentsOf: identifier) + let value = UInt64(bitPattern: timestampNs) + for index in (0..> (7 * index)) & 0x7f)) + } + unit.append(trailingBits) + return unit + } + + /// A Codevisor marker of any length, so malformed markers are still stripped. + static func isMarker(_ unit: Data, codec: ScreenSharingVideoCodec) -> Bool { + let header = header(for: codec) + let identifierEnd = header.count + 2 + identifier.count + guard unit.count >= identifierEnd else { return false } + let bytes = [UInt8](unit.prefix(identifierEnd)) + return Array(bytes[0.. Int64? { + let header = header(for: codec) + let valueStart = header.count + 2 + identifier.count + let expectedCount = valueStart + valueByteCount + 1 + for unit in units where unit.count == expectedCount && isMarker(unit, codec: codec) { + let bytes = [UInt8](unit) + guard bytes[header.count + 1] == UInt8(payloadSize), bytes[expectedCount - 1] == trailingBits, + bytes[valueStart] == 0x80 + else { continue } + var value: UInt64 = 0 + var valid = true + for byte in bytes[(valueStart + 1)..<(valueStart + valueByteCount)] { + guard byte & 0x80 != 0 else { valid = false; break } + value = (value << 7) | UInt64(byte & 0x7f) + } + guard valid, value <= UInt64(Int64.max) else { continue } + return Int64(value) + } + return nil + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoCodec.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoCodec.swift new file mode 100644 index 000000000..18957c62b --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoCodec.swift @@ -0,0 +1,62 @@ +import CoreMedia +import CoreVideo +import Foundation + +/// H.264 remains the product default. HEVC variants are explicit diagnostic +/// choices until transport recovery and hardware compatibility are validated. +public enum ScreenSharingVideoCodec: String, Sendable, CaseIterable { + case h264 + case hevc + case hevc444 + + var mediaType: CMVideoCodecType { self == .h264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC } + var payloadName: String { self == .h264 ? "H264" : "H265" } + var decodedPixelFormat: OSType { + self == .hevc444 ? kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + } + /// BGRA preserves all source chroma for the hardware Main444 encoder. + public var capturePixelFormat: OSType { + self == .hevc444 ? kCVPixelFormatType_32BGRA : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + } + var sdpParameters: [String: String] { + if self == .h264 { + return ["profile-level-id": "42e034", "packetization-mode": "1", "level-asymmetry-allowed": "1"] + } + return ["profile-id": self == .hevc444 ? "4" : "1", "tier-flag": "0", "level-id": "153"] + } +} + +/// Parse the hvcC fields needed to reject silent chroma/depth fallback. +struct ScreenSharingHEVCFormat: Equatable { + let profile: Int + let chroma: Int + let lumaDepth: Int + let chromaDepth: Int + + init(configuration: Data) throws { + let bytes = [UInt8](configuration) + guard bytes.count >= 23, bytes[0] == 1 else { + throw ScreenSharingError.invalid("Invalid HEVC decoder configuration.") + } + profile = Int(bytes[1] & 31) + chroma = Int(bytes[16] & 3) + lumaDepth = 8 + Int(bytes[17] & 7) + chromaDepth = 8 + Int(bytes[18] & 7) + } + + func validate(for codec: ScreenSharingVideoCodec) throws { + guard codec != .h264, profile == (codec == .hevc444 ? 4 : 1), + chroma == (codec == .hevc444 ? 3 : 1), lumaDepth == 8, chromaDepth == 8 + else { throw ScreenSharingError.invalid("HEVC output does not match the negotiated profile, chroma or bit depth.") } + } + + static func read(_ description: CMFormatDescription) throws -> Self { + let atoms = + CMFormatDescriptionGetExtension( + description, extensionKey: kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms) as? [String: Any] + guard let data = atoms?["hvcC"] as? Data else { + throw ScreenSharingError.invalid("Missing HEVC decoder configuration.") + } + return try Self(configuration: data) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoConfiguration.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoConfiguration.swift new file mode 100644 index 000000000..405b9ba59 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoConfiguration.swift @@ -0,0 +1,128 @@ +import CoreVideo +import Foundation + +public enum ScreenSharingError: Error, LocalizedError, Sendable { + case unavailable(String) + case invalid(String) + case codec(String, Int32) + + public var errorDescription: String? { + switch self { + case .unavailable(let message), .invalid(let message): message + case .codec(let operation, let status): "\(operation) failed (\(status))." + } + } +} + +public struct ScreenSharingVideoConfiguration: Sendable, Equatable, Codable { + public let width: Int + public let height: Int + public let framesPerSecond: Int + public let bitrate: Int + + public init(width: Int = 1920, height: Int = 1080, framesPerSecond: Int = 60, bitrate: Int = 12_000_000) throws { + guard (64...3840).contains(width), (64...2160).contains(height), width.isMultiple(of: 2), height.isMultiple(of: 2), + (1...60).contains(framesPerSecond), (100_000...80_000_000).contains(bitrate) + else { throw ScreenSharingError.invalid("Unsupported video dimensions, frame rate or bitrate.") } + self.width = width + self.height = height + self.framesPerSecond = framesPerSecond + self.bitrate = bitrate + } + + public init(from decoder: any Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + try self.init( + width: values.decode(Int.self, forKey: .width), height: values.decode(Int.self, forKey: .height), + framesPerSecond: values.decode(Int.self, forKey: .framesPerSecond), + bitrate: values.decode(Int.self, forKey: .bitrate)) + } +} + +/// Immutable ownership of a decoded CVPixelBuffer; retaining it prevents its +/// pool from recycling the storage before Metal has finished reading it. +public struct ScreenSharingVideoFrame: @unchecked Sendable { + public let pixelBuffer: CVPixelBuffer + public let timestampNs: Int64 + public let rtpTimestamp: UInt32 + /// Receiver-local Core Animation clock, recorded at the native RTC renderer. + /// Never compare this with a sender timestamp or a remote clock. + public let receivedAtSeconds: Double? + /// Content identity: the capture timestamp of the buffer's content. Equal to + /// `timestampNs` for a fresh capture, retained across refreshes, carried + /// in-band to the viewer. Compare only with host values. + public let sourceTimestampNs: Int64? + /// Receiver-only diagnostic identity read from the decoded buffer's audit + /// attachment at the RTC renderer callback; nil whenever the audit is off. + public let deliveryAuditIdentity: ScreenSharingFrameDeliveryAudit.Identity? + + public init( + pixelBuffer: CVPixelBuffer, timestampNs: Int64, rtpTimestamp: UInt32 = 0, + receivedAtSeconds: Double? = nil, sourceTimestampNs: Int64? = nil, + deliveryAuditIdentity: ScreenSharingFrameDeliveryAudit.Identity? = nil + ) { + self.pixelBuffer = pixelBuffer + self.timestampNs = timestampNs + self.rtpTimestamp = rtpTimestamp + self.receivedAtSeconds = receivedAtSeconds + self.sourceTimestampNs = sourceTimestampNs + self.deliveryAuditIdentity = deliveryAuditIdentity + } +} + +public struct ScreenSharingEncodedFrame: Sendable { + public let data: Data + public let timestampNs: Int64 + public let rtpTimestamp: UInt32 + public let width: Int + public let height: Int + public let isKeyFrame: Bool + /// Content identity written into the frame's marker. + public var sourceTimestampNs: Int64? = nil +} + +/// A one-element mailbox. A slow renderer retains the newest frame, never an +/// unbounded backlog. GPU in-flight ownership is managed by the renderer. +public final class ScreenSharingFrameMailbox: @unchecked Sendable { + private let lock = NSLock() + private var frame: ScreenSharingVideoFrame? + private var replacements = 0 + private var onAvailable: (@Sendable () -> Void)? + + public init() {} + + /// Newest-frame policy unchanged: returns the frame that was replaced (if + /// any) so a receiver-only audit can record the replacement; callers that + /// ignore it drop it exactly as before. + @discardableResult + public func put(_ value: ScreenSharingVideoFrame) -> ScreenSharingVideoFrame? { + let (notify, replaced) = lock.withLock { + let previous = frame + if previous != nil { replacements += 1 } + frame = value + return (previous == nil ? onAvailable : nil, previous) + } + notify?() + return replaced + } + + /// Notify only the empty-to-full transition, bounding scheduled render work. + public func onFrameAvailable(_ callback: (@Sendable () -> Void)?) { + let occupied = lock.withLock { + onAvailable = callback; return frame != nil + } + if occupied { callback?() } + } + + public func take() -> ScreenSharingVideoFrame? { + lock.withLock { + defer { frame = nil } + return frame + } + } + + public func clear() { lock.withLock { frame = nil } } + public var droppedFrames: Int { lock.withLock { replacements } } + /// Non-mutating ownership observation (diagnostic). + public var isHolding: Bool { lock.withLock { frame != nil } } +} diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoRefresh.swift b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoRefresh.swift new file mode 100644 index 000000000..47a4b6829 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/ScreenSharingVideoRefresh.swift @@ -0,0 +1,227 @@ +import Foundation + +enum ScreenSharingVideoRefreshMessage: Sendable, Equatable { + /// Viewer to host: encode the newest source frame as a keyframe. + case keyframe + /// Host to viewer: capture went idle; the newest encoded frame carries this + /// source timestamp. Sent once per idle transition over the reliable channel. + case sourceIdle(latestTimestampNs: Int64) + + func encoded() -> Data { + switch self { + case .keyframe: + return Data([1]) + case .sourceIdle(let latestTimestampNs): + var data = Data([2]) + withUnsafeBytes(of: latestTimestampNs.bigEndian) { data.append(contentsOf: $0) } + return data + } + } + + static func decode(_ data: Data) throws -> Self { + let bytes = [UInt8](data) + switch (bytes.first, bytes.count) { + case (1, 1): + return .keyframe + case (2, 9): + let value = bytes[1...].reduce(UInt64(0)) { ($0 << 8) | UInt64($1) } + guard value <= UInt64(Int64.max) else { throw ScreenSharingError.invalid("Invalid idle notice.") } + return .sourceIdle(latestTimestampNs: Int64(value)) + default: + throw ScreenSharingError.invalid("Invalid video refresh request.") + } + } +} + +/// Coalesces decoder notifications before scheduling work on the main actor. +/// Revisions let the receiver reject notifications delivered out of order. +final class ScreenSharingRefreshSignal: @unchecked Sendable { + struct Event: Sendable { + let revision: UInt64 + let needed: Bool + } + + private let lock = NSLock() + private var needed = false + private var revision: UInt64 = 0 + private var generation: UInt64 = 0 + private var decoderReset = false + private var closed = false + private var callback: (@Sendable (Event) -> Void)? + + func onChange(_ callback: @escaping @Sendable (Event) -> Void) { + let pending: Event? = lock.withLock { + guard !closed else { return nil } + self.callback = callback + return needed ? Event(revision: revision, needed: true) : nil + } + if let pending { callback(pending) } + } + var keyframeGeneration: UInt64 { lock.withLock { generation } } + func request(resetDecoder: Bool = false) { change(to: true, resetDecoder: resetDecoder) } + /// A verified delivery shortfall needs recovery only when none is pending; it + /// must not invalidate a replacement keyframe that is already in flight. + @discardableResult + func requestUnlessPending() -> Bool { + let notification: ((@Sendable (Event) -> Void)?, Event)? = lock.withLock { + guard !closed, !needed else { return nil } + generation &+= 1 + needed = true + revision &+= 1 + return (callback, Event(revision: revision, needed: true)) + } + guard let (callback, event) = notification else { return false } + callback?(event) + return true + } + func decodedKeyframe(generation: UInt64) { change(to: false, completedGeneration: generation) } + func consumeDecoderReset() -> Bool { + lock.withLock { + let result = decoderReset; decoderReset = false; return result + } + } + func close() { + lock.withLock { + closed = true; callback = nil + } + } + + private func change(to value: Bool, resetDecoder: Bool = false, completedGeneration: UInt64? = nil) { + let notification: ((@Sendable (Event) -> Void), Event)? = lock.withLock { + guard !closed else { return nil } + if value { + generation &+= 1 + decoderReset = decoderReset || resetDecoder + } else if completedGeneration != generation || decoderReset { + return nil + } + guard needed != value else { return nil } + needed = value + revision &+= 1 + guard let callback else { return nil } + return (callback, Event(revision: revision, needed: value)) + } + if let (callback, event) = notification { callback(event) } + } +} + +/// A request made on the peer actor is consumed on WebRTC's encoder queue. +final class ScreenSharingEncoderRefreshRequest: @unchecked Sendable { + private let lock = NSLock() + private var pending = false + func request() { lock.withLock { pending = true } } + func consume() -> Bool { + lock.withLock { + let result = pending; pending = false; return result + } + } +} + +struct ScreenSharingRefreshRateLimit { + static let intervalNs: Int64 = 100_000_000 + private var lastAllowed: Int64? + + mutating func allow(nowNs: Int64) -> Bool { + guard nowNs >= 0 else { return false } + if let lastAllowed, nowNs < lastAllowed || nowNs - lastAllowed < Self.intervalNs { return false } + lastAllowed = nowNs + return true + } +} + +/// Retries only while keyframe recovery is pending. Normal idle video has no timer. +@MainActor +final class ScreenSharingRefreshRequester { + private let send: () -> Bool + private let available: () -> Bool + private let nowNs: @Sendable () -> Int64 + private let sleep: @Sendable (Duration) async throws -> Void + private var task: Task? + private var revision: UInt64 = 0 + private var needed = false + private var closed = false + private var rateLimit = ScreenSharingRefreshRateLimit() + + init( + available: @escaping () -> Bool, send: @escaping () -> Bool, + nowNs: @escaping @Sendable () -> Int64 = { ScreenSharingMetrics.nowNs }, + sleep: @escaping @Sendable (Duration) async throws -> Void = { try await Task.sleep(for: $0) } + ) { + self.available = available; self.send = send; self.nowNs = nowNs; self.sleep = sleep + } + + func update(_ event: ScreenSharingRefreshSignal.Event) { + guard !closed, event.revision > revision else { return } + revision = event.revision + needed = event.needed + guard needed else { task?.cancel(); task = nil; return } + wake() + guard task == nil else { return } + let sleep = sleep + task = Task { @MainActor [weak self] in + do { + while !Task.isCancelled { + try await sleep(.milliseconds(100)) + guard !Task.isCancelled, let self, !self.closed, self.needed else { return } + self.wake() + } + } catch {} + } + } + + func wake() { + guard !closed, needed, available(), rateLimit.allow(nowNs: nowNs()) else { return } + _ = send() + } + + @discardableResult + func close() -> Task? { + closed = true + needed = false + let pending = task + task = nil + pending?.cancel() + return pending + } +} + +/// Owns at most one capture buffer. Two independent orders: content follows +/// actual capture timestamps (duplicates and out-of-order captures are +/// rejected), while timestamps submitted to WebRTC stay strictly increasing +/// across refreshes, so a synthetic refresh never discards a newer capture +/// that arrives late. Clearing the cache releases its surface immediately. +struct ScreenSharingRefreshFrameStore { + private var frame: ScreenSharingVideoFrame? + private var lastCapturedNs: Int64? + private var lastSubmittedNs: Int64? + + /// The frame to submit for a newer capture: same buffer and content + /// identity, submission timestamp above every earlier submission. + mutating func capture(_ frame: ScreenSharingVideoFrame) -> ScreenSharingVideoFrame? { + guard frame.timestampNs >= 0 else { return nil } + if let lastCapturedNs, frame.timestampNs <= lastCapturedNs { return nil } + guard let submission = nextSubmission(atLeast: frame.timestampNs) else { return nil } + self.frame = frame + lastCapturedNs = frame.timestampNs + lastSubmittedNs = submission + return ScreenSharingVideoFrame( + pixelBuffer: frame.pixelBuffer, timestampNs: submission, sourceTimestampNs: frame.timestampNs) + } + + mutating func refresh(nowNs: Int64) -> ScreenSharingVideoFrame? { + guard let frame, nowNs >= 0, let submission = nextSubmission(atLeast: nowNs) else { return nil } + lastSubmittedNs = submission + return ScreenSharingVideoFrame( + pixelBuffer: frame.pixelBuffer, timestampNs: submission, sourceTimestampNs: frame.timestampNs) + } + + mutating func clear() { frame = nil } + + var isHolding: Bool { frame != nil } + + private func nextSubmission(atLeast value: Int64) -> Int64? { + guard let lastSubmittedNs else { return value } + guard lastSubmittedNs < Int64.max else { return nil } + return max(value, lastSubmittedNs + 1) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/NALUnitBitstreamTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/NALUnitBitstreamTests.swift new file mode 100644 index 000000000..4cde85521 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/NALUnitBitstreamTests.swift @@ -0,0 +1,37 @@ +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct NALUnitBitstreamTests { + @Test func mixedStartCodesRoundTrip() throws { + let frame = Data([0, 0, 0, 1, 0x67, 0xaa, 0, 0, 1, 0x68, 0xbb, 0, 0, 0, 1, 0x65, 0xcc]) + let units = try NALUnitBitstream.nalUnits(frame) + #expect(units == [Data([0x67, 0xaa]), Data([0x68, 0xbb]), Data([0x65, 0xcc])]) + let converted = try NALUnitBitstream.annexB(NALUnitBitstream.lengthPrefixed(units)) + #expect(try NALUnitBitstream.nalUnits(converted) == units) + } + + @Test func rejectsTruncatedAndEmptyUnits() { + for bytes: [UInt8] in [[0, 0, 0, 5, 0x65], [0, 0, 0, 0], [0, 0, 1]] { + #expect(throws: (any Error).self) { try NALUnitBitstream.annexB(Data(bytes)) } + } + #expect(throws: (any Error).self) { try NALUnitBitstream.nalUnits(Data([0, 0, 1])) } + #expect(throws: (any Error).self) { try NALUnitBitstream.nalUnits(Data([0x65, 0xaa])) } + } + + @Test func rejectsExcessiveNALCountAndEmptyFrame() { + let manyUnits = Data(Array(repeating: [UInt8(0), 0, 1, 0x65], count: 1025).flatMap { $0 }) + #expect(throws: (any Error).self) { try NALUnitBitstream.nalUnits(manyUnits) } + #expect(throws: (any Error).self) { try NALUnitBitstream.annexB(Data()) } + } + + @Test func acceptsEmulationPreventionBytes() throws { + let nal = Data([0x65, 0, 0, 3, 1, 0xaa]) + #expect(try NALUnitBitstream.nalUnits(Data([0, 0, 0, 1]) + nal) == [nal]) + } + + @Test func rejectsInvalidConfigurationOnDecode() { + let invalid = Data(#"{"width":1921,"height":1080,"framesPerSecond":60,"bitrate":12000000}"#.utf8) + #expect(throws: (any Error).self) { try JSONDecoder().decode(ScreenSharingVideoConfiguration.self, from: invalid) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingAdaptiveQualityTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingAdaptiveQualityTests.swift new file mode 100644 index 000000000..52a07f2af --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingAdaptiveQualityTests.swift @@ -0,0 +1,38 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingAdaptiveQualityTests { + @Test func sustainedShortagePreservesTextBeforeReducingResolution() throws { + var quality = ScreenSharingAdaptiveQuality(configuration: try .init()) + #expect(quality.update(availableBitrate: 2_000_000, now: 0) == nil) + #expect(quality.update(availableBitrate: 2_000_000, now: 1.999) == nil) + let slowerValue = quality.update(availableBitrate: 2_000_000, now: 2) + let slower = try #require(slowerValue) + #expect(slower.width == 1920 && slower.height == 1080 && slower.framesPerSecond == 30) + #expect(quality.update(availableBitrate: 2_000_000, now: 3) == nil) + let smallerValue = quality.update(availableBitrate: 2_000_000, now: 5) + let smaller = try #require(smallerValue) + #expect(smaller.width == 1440 && smaller.height == 810 && smaller.framesPerSecond == 30) + #expect(quality.update(availableBitrate: 1_000_000, now: 6) == nil) + let minimumValue = quality.update(availableBitrate: 1_000_000, now: 8) + let minimum = try #require(minimumValue) + #expect(minimum.width == 960 && minimum.height == 540 && minimum.framesPerSecond == 20) + #expect(quality.update(availableBitrate: 100_000, now: 100) == nil) + } + + @Test func recoveryNeedsContinuousHeadroomAndMissingSamplesResetTheDecision() throws { + var quality = ScreenSharingAdaptiveQuality(configuration: try .init(width: 1512, height: 982)) + _ = quality.update(availableBitrate: 1_000_000, now: 0) + _ = quality.update(availableBitrate: 1_000_000, now: 2) + #expect(quality.update(availableBitrate: 12_000_000, now: 3) == nil) + #expect(quality.update(availableBitrate: nil, now: 10) == nil) + #expect(quality.update(availableBitrate: 12_000_000, now: 18) == nil) + #expect(quality.update(availableBitrate: 12_000_000, now: 32.999) == nil) + let restoredValue = quality.update(availableBitrate: 12_000_000, now: 33) + let restored = try #require(restoredValue) + #expect(restored.width == 1512 && restored.height == 982 && restored.framesPerSecond == 60) + #expect(quality.update(availableBitrate: .nan, now: 100) == nil) + #expect(quality.update(availableBitrate: .infinity, now: 100) == nil) + #expect(quality.update(availableBitrate: -1, now: 100) == nil) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingBufferTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingBufferTests.swift new file mode 100644 index 000000000..3c5de10d4 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingBufferTests.swift @@ -0,0 +1,80 @@ +import CoreVideo +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingBufferTests { + @Test func cadenceSeparatesClocksAndRebasesAfterTimestampRegression() throws { + let metrics = ScreenSharingMetrics() + metrics.event("capture", atNanoseconds: 10_000_000) + metrics.event("capture", atNanoseconds: 26_000_000) + metrics.event("capture", atNanoseconds: 26_000_000) + metrics.event("capture", atNanoseconds: -1) + metrics.event("capture", atNanoseconds: 2_000_000) + metrics.event("capture", atNanoseconds: 35_000_000) + metrics.event("decode", atNanoseconds: 1_000_000_000) + metrics.event("decode", atNanoseconds: 1_010_000_000) + let capture = try #require(metrics.snapshot().timings["capture"]) + #expect(capture.count == 3) + #expect(capture.p50Ms == 16) + #expect(capture.maximumMs == 33) + #expect(metrics.snapshot().timings["decode"]?.p50Ms == 10) + } + + @Test func slowConsumerReceivesNewestFrameAndReleasesQueuedFrame() throws { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 64, 64, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + let buffer = try #require(pixel) + let mailbox = ScreenSharingFrameMailbox() + for timestamp in 1...100 { + mailbox.put(ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: Int64(timestamp))) + } + #expect(mailbox.droppedFrames == 99) + #expect(mailbox.take()?.timestampNs == 100) + #expect(mailbox.take() == nil) + mailbox.put(ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: 101)) + mailbox.clear() + #expect(mailbox.take() == nil) + #expect(mailbox.droppedFrames == 99) + } + + @Test func metricsBoundHistoryAndRejectInvalidDurations() throws { + let metrics = ScreenSharingMetrics() + for value in 0..<2000 { metrics.observe("encode", milliseconds: Double(value)) } + metrics.observe("encode", milliseconds: .infinity) + metrics.observe("encode", milliseconds: .nan) + metrics.observe("encode", milliseconds: -1) + let timing = try #require(metrics.snapshot().timings["encode"]) + #expect(timing.count == 1800) + #expect(timing.p50Ms == 1100) + #expect(timing.p95Ms == 1910) + #expect(timing.maximumMs == 1999) + } + + @Test func rendererNotificationsAreBoundedUntilMailboxIsDrained() throws { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 64, 64, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + let buffer = try #require(pixel) + let mailbox = ScreenSharingFrameMailbox() + let notifications = ScreenSharingMetrics() + mailbox.onFrameAvailable { notifications.increment("available") } + for timestamp in 1...100 { + mailbox.put(ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: Int64(timestamp))) + } + #expect(notifications.snapshot().counters["available"] == 1) + #expect(mailbox.take()?.timestampNs == 100) + mailbox.put(ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: 101)) + #expect(notifications.snapshot().counters["available"] == 2) + mailbox.onFrameAvailable(nil) + mailbox.clear() + mailbox.put(ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: 102)) + #expect(notifications.snapshot().counters["available"] == 2) + mailbox.onFrameAvailable { notifications.increment("available") } + #expect(notifications.snapshot().counters["available"] == 3) + } + + @Test(arguments: [0, 63, 1921, 3842]) + func rejectsUnsupportedWidths(_ width: Int) { + #expect(throws: (any Error).self) { try ScreenSharingVideoConfiguration(width: width) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureBufferCopyTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureBufferCopyTests.swift new file mode 100644 index 000000000..1c98dc7bb --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingCaptureBufferCopyTests.swift @@ -0,0 +1,51 @@ +import CoreVideo +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingCaptureBufferCopyTests { + private func source(width: Int = 64) throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + let status = CVPixelBufferCreate( + nil, width, 64, kCVPixelFormatType_32BGRA, + [kCVPixelBufferIOSurfacePropertiesKey: [:]] as CFDictionary, &pixel) + #expect(status == kCVReturnSuccess) + let source = try #require(pixel) + #expect(CVPixelBufferLockBaseAddress(source, []) == kCVReturnSuccess) + defer { CVPixelBufferUnlockBaseAddress(source, []) } + let bytes = try #require(CVPixelBufferGetBaseAddress(source)).assumingMemoryBound(to: UInt8.self) + for y in 0..<64 { + for x in 0..? + + func apply(_ request: ScreenSharingCaptureIntervalRequest) async throws { + self.request = request + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + entered.signal() + } + } + + func finish() { + let pending = continuation + continuation = nil + pending?.resume() + } + } + + @MainActor private final class Generation { var value = 0 } + + private func initialCapture() async throws -> (ScreenSharingCapture, ScreenSharingMetrics) { + let capture = ScreenSharingCapture(captureIntervalFPS: 120) + let metrics = ScreenSharingMetrics() + try await capture.applyIntervalUpdate( + configuration: .init(), override: 120, metrics: metrics, + apply: { _ in }, isCurrent: { true }) + return (capture, metrics) + } + + @Test func validationFailureNeverCallsTheStreamOrChangesTheRequest() async throws { + let (capture, metrics) = try await initialCapture() + let before = metrics.snapshot().labels + var calls = 0 + await #expect(throws: ScreenSharingError.self) { + try await capture.applyIntervalUpdate( + configuration: .init(framesPerSecond: 60), override: 30, metrics: metrics, + apply: { _ in calls += 1 }, isCurrent: { true }) + } + #expect(calls == 0) + #expect(capture.requestState.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels == before) + } + + @Test func aThrownStreamUpdatePreservesTheOriginalErrorRequestAndLabels() async throws { + let (capture, metrics) = try await initialCapture() + let before = metrics.snapshot().labels + var calls = 0 + var generationChecks = 0 + await #expect(throws: Failure.streamUpdate) { + try await capture.applyIntervalUpdate( + configuration: .init(framesPerSecond: 30), override: nil, metrics: metrics, + apply: { request in + calls += 1 + #expect(request.requestedFramesPerSecond == 30) + throw Failure.streamUpdate + }, + isCurrent: { + generationChecks += 1; return true + }) + } + #expect(calls == 1 && generationChecks == 0) + #expect(capture.requestState.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels == before) + } + + @Test func aHeldSuccessfulUpdateCommitsOnlyAfterItsApplyReturns() async throws { + let (capture, metrics) = try await initialCapture() + let before = metrics.snapshot().labels + let held = HeldApply() + let update = Task { @MainActor in + try await capture.applyIntervalUpdate( + configuration: .init(framesPerSecond: 30), override: nil, metrics: metrics, + apply: held.apply, isCurrent: { true }) + } + await held.entered.wait() + #expect(held.request?.requestedFramesPerSecond == 30) + #expect(capture.requestState.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels == before) + held.finish() + try await update.value + #expect(capture.requestState.overrideFramesPerSecond == nil) + #expect(metrics.snapshot().labels["captureRequestedMinimumFrameIntervalFPS"] == "30") + #expect(metrics.snapshot().labels["captureRequestedFrameIntervalOverride"] == "none") + } + + @Test func aSuccessfulApplyFromAnOldGenerationCannotCommit() async throws { + let (capture, metrics) = try await initialCapture() + let before = metrics.snapshot().labels + let held = HeldApply() + let generation = Generation() + let startedGeneration = generation.value + let update = Task { @MainActor in + try await capture.applyIntervalUpdate( + configuration: .init(framesPerSecond: 30), override: nil, metrics: metrics, + apply: held.apply, isCurrent: { generation.value == startedGeneration }) + } + await held.entered.wait() + generation.value += 1 + held.finish() + await #expect(throws: CancellationError.self) { try await update.value } + #expect(capture.requestState.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels == before) + } + } +#endif diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingClipboardTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingClipboardTests.swift new file mode 100644 index 000000000..94e689a39 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingClipboardTests.swift @@ -0,0 +1,144 @@ +import Foundation +import Testing +@testable import CodevisorScreenSharing + +@MainActor +struct ScreenSharingClipboardTests { + @Test func explicitRoundTripsPreserveUnicodeAcrossChunkBoundariesAndBoundTheQueue() throws { + let fixture = ClipboardFixture() + let value = String(repeating: "é👩🏽‍💻日本語\n", count: 1500) + fixture.viewer.sendText(value) + #expect(fixture.viewer.isBusy) + fixture.drain() + #expect(fixture.hostText == value) + #expect(fixture.maximumQueued <= 1) + #expect(!fixture.viewer.isBusy && !fixture.host.isBusy) + #expect(fixture.viewerResults.last! == nil) + fixture.hostText = "Remote clipboard ✓" + fixture.viewer.requestText() + fixture.drain() + #expect(fixture.viewerText == fixture.hostText) + #expect(fixture.viewerResults.count == 2) + } + + @Test func viewerRejectsUnrequestedClipboardAndNeverChangesItsPasteboard() { + let fixture = ClipboardFixture() + fixture.host.sendText("unsolicited") + fixture.drain() + #expect(fixture.viewerText == "local") + #expect(fixture.hostResults.last! != nil) + #expect(!fixture.host.isBusy) + } + + @Test func lateChunksCannotCompleteAfterCancellationOrTimeout() { + let fixture = ClipboardFixture() + fixture.viewer.requestText() + fixture.deliverOne() + fixture.deliverOne() + fixture.viewer.cancel() + fixture.drain() + #expect(fixture.viewerText == "local") + #expect(!fixture.host.isBusy && !fixture.viewer.isBusy) + + fixture.viewer.requestText() + fixture.time = 9.999; fixture.viewer.tick() + #expect(fixture.viewer.isBusy) + fixture.time = 10; fixture.viewer.tick() + #expect(!fixture.viewer.isBusy) + fixture.drain() + #expect(fixture.viewerText == "local") + } + + @Test func invalidOrderOversizeAndInvalidUTF8NeverWritePartialData() { + var sent: [ScreenSharingClipboardMessage] = [] + var written: [String] = [] + let host = ScreenSharingClipboardTransfer( + send: { + sent.append($0); return true + }, + canReceiveUnsolicited: { true }, read: { "" }, write: { written.append($0) }) + let id = UUID() + host.receive(.begin(id: id, bytes: 65537)) + #expect(!host.isBusy) + host.receive(.begin(id: id, bytes: 2)) + host.receive(.chunk(id: id, index: 1, data: Data([65, 66]))) + host.receive(.end(id: id)) + #expect(written.isEmpty && !host.isBusy) + host.receive(.begin(id: id, bytes: 2)) + host.receive(.chunk(id: id, index: 0, data: Data([255, 255]))) + host.receive(.end(id: id)) + #expect(written.isEmpty && !host.isBusy) + } + + @Test func losingAuthorizationBeforeCompletionCannotWriteClipboard() { + var authorized = true + var written = false + let host = ScreenSharingClipboardTransfer( + send: { _ in true }, canReceiveUnsolicited: { authorized }, + read: { "" }, write: { _ in written = true }) + let id = UUID() + host.receive(.begin(id: id, bytes: 1)) + host.receive(.chunk(id: id, index: 0, data: Data([65]))) + authorized = false + host.receive(.end(id: id)) + #expect(!written && !host.isBusy) + } + + @Test func writeErrorsAndClosedChannelsEndTheTransfer() { + let fixture = ClipboardFixture() + fixture.rejectWrite = true + fixture.viewer.sendText("new") + fixture.drain() + #expect(fixture.hostText == "remote") + #expect(fixture.viewerResults.last! == "Fixture clipboard changed") + fixture.connected = false + fixture.viewer.requestText() + #expect(!fixture.viewer.isBusy) + #expect(fixture.viewerResults.last!?.contains("unavailable") == true) + } + + @Test func wirePayloadStaysBelowChannelLimitForLargestChunk() throws { + let message = ScreenSharingClipboardMessage.chunk(id: UUID(), index: 31, data: Data(repeating: 255, count: 2048)) + let data = try message.encoded() + #expect(data.count < 4096) + #expect(try ScreenSharingClipboardMessage.decode(data) == message) + #expect(throws: (any Error).self) { try ScreenSharingClipboardMessage.decode(Data(repeating: 0, count: 4097)) } + } +} + +@MainActor +private final class ClipboardFixture { + var time = 0.0 + var hostText = "remote" + var viewerText = "local" + var rejectWrite = false + var connected = true + var maximumQueued = 0 + var viewerResults: [String?] = [] + var hostResults: [String?] = [] + var queue: [(host: Bool, message: ScreenSharingClipboardMessage)] = [] + lazy var viewer = make(host: false) + lazy var host = make(host: true) + private func make(host: Bool) -> ScreenSharingClipboardTransfer { + let transfer = ScreenSharingClipboardTransfer( + send: { [unowned self] in + guard connected else { return false } + queue.append((host: !host, message: $0)); maximumQueued = max(maximumQueued, queue.count); return true + }, now: { [unowned self] in time }, canReceiveUnsolicited: { host }, + read: { [unowned self] in host ? hostText : viewerText }, + write: { [unowned self] text in + if rejectWrite { throw ScreenSharingError.invalid("Fixture clipboard changed") } + if host { hostText = text } else { viewerText = text } + }) + transfer.onFinished = { [unowned self] error in + if host { hostResults.append(error) } else { viewerResults.append(error) } + } + return transfer + } + func deliverOne() { + guard !queue.isEmpty else { return } + let next = queue.removeFirst() + if next.host { host.receive(next.message) } else { viewer.receive(next.message) } + } + func drain() { while !queue.isEmpty { deliverOne() } } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingControlMessageTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingControlMessageTests.swift new file mode 100644 index 000000000..e761f9b8a --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingControlMessageTests.swift @@ -0,0 +1,81 @@ +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingControlMessageTests { + @Test func wireFormatRejectsUnsupportedVersionsOversizeAndInvalidInput() throws { + let id = UUID() + let message = ScreenSharingControlMessage.input( + lease: id, sequence: 2, event: .button(.init(x: 0.1, y: 0.9), button: 2, down: true, clicks: 2, modifiers: 9)) + #expect(try ScreenSharingControlMessage.decode(message.encoded()) == message) + let data = try message.encoded() + var object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + object["version"] = 2 + let unsupported = try JSONSerialization.data(withJSONObject: object) + #expect(throws: (any Error).self) { try ScreenSharingControlMessage.decode(unsupported) } + #expect(throws: (any Error).self) { try ScreenSharingControlMessage.decode(Data(repeating: 0, count: 4097)) } + let invalid = ScreenSharingControlMessage.input( + lease: id, sequence: 1, event: .key(code: 65535, down: true, repeatKey: false, modifiers: 0)) + #expect(throws: (any Error).self) { try ScreenSharingControlMessage.decode(invalid.encoded()) } + #expect(!ScreenSharingInputEvent.move(.init(x: .nan, y: 0), modifiers: 0).isValid) + #expect(!ScreenSharingInputEvent.scroll(.init(x: 0, y: 0), x: .min, y: 0, modifiers: 0).isValid) + #expect(!ScreenSharingInputEvent.text(String(repeating: "x", count: 1025)).isValid) + } + + @Test func geometryExcludesLetterboxingAndClampsOnlyAnActiveDrag() { + func pointer(_ x: Double, _ y: Double, clamp: Bool = false) -> ScreenSharingPointer? { + ScreenSharingVideoGeometry.pointer( + x: x, y: y, surfaceWidth: 1000, surfaceHeight: 1000, + videoWidth: 1920, videoHeight: 1080, fit: true, clamp: clamp) + } + #expect(pointer(500, 500) == .init(x: 0.5, y: 0.5)) + #expect(pointer(500, 100) == nil) + #expect(pointer(-10, 500) == nil) + #expect(pointer(-10, 500, clamp: true) == .init(x: 0, y: 0.5)) + #expect(pointer(500, 100, clamp: true) == .init(x: 0.5, y: 0)) + #expect( + ScreenSharingVideoGeometry.pointer( + x: 100, y: 100, surfaceWidth: 0, surfaceHeight: 100, + videoWidth: 100, videoHeight: 100, fit: true) == nil) + } + + @Test func incomingControlHasBoundedBytesAndMessageCountEvenForEmptyPackets() { + var inbox = ScreenSharingControlInbox() + #expect(inbox.enqueue(Data()) == true) + for _ in 1..<256 { #expect(inbox.enqueue(Data()) == false) } + #expect(inbox.enqueue(Data()) == false) + let overflow = inbox.drain() + #expect(overflow.0.count == 256 && overflow.1) + inbox.stop() + #expect(inbox.enqueue(Data([1])) == false) + #expect(inbox.drain().0.isEmpty) + + var bytes = ScreenSharingControlInbox() + for index in 0..<16 { #expect(bytes.enqueue(Data(repeating: UInt8(index), count: 4096)) == (index == 0)) } + let batch = bytes.drain() + #expect(!batch.1) + #expect(batch.0.map { $0.first! } == Array(UInt8(0)..<16)) + #expect(bytes.enqueue(Data(repeating: 1, count: 4097)) == true) + #expect(bytes.drain().1) + } + + @Test func fractionalTrackpadMovementIsPreservedAndInvalidDeltasDoNotContaminateIt() { + var scroll = ScreenSharingScrollAccumulator() + #expect(scroll.add(x: 0.4, y: -0.4) == (0, 0)) + #expect(scroll.add(x: 0.4, y: -0.4) == (0, 0)) + #expect(scroll.add(x: .infinity, y: 0) == (0, 0)) + #expect(scroll.add(x: 0.4, y: -0.4) == (1, -1)) + #expect(scroll.add(x: 100_000, y: -100_000) == (4096, -4096)) + } + + @Test func actualSizeMapsVideoPixelsInsideTheCenteredDocument() { + #expect( + ScreenSharingVideoGeometry.pointer( + x: 500, y: 300, surfaceWidth: 2000, surfaceHeight: 1200, + videoWidth: 1000, videoHeight: 600, fit: false) == .init(x: 0, y: 0)) + #expect( + ScreenSharingVideoGeometry.pointer( + x: 1500, y: 900, surfaceWidth: 2000, surfaceHeight: 1200, + videoWidth: 1000, videoHeight: 600, fit: false) == .init(x: 1, y: 1)) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDecoderRecoveryCheckTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDecoderRecoveryCheckTests.swift new file mode 100644 index 000000000..ec8df7bf9 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDecoderRecoveryCheckTests.swift @@ -0,0 +1,49 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingDecoderRecoveryCheckTests { + @Test func encoderFaultIsArmedOnlyAtTheDecoderResetEdge() { + let decoder = ScreenSharingDecoderRecoveryCheck() + let outputDrop = ScreenSharingEncoderDropCheck() + decoder.arm(afterFrames: 2) { + // Reentry also verifies the notification runs outside the state lock. + #expect(decoder.inspect(keyFrame: false, nowNs: 2) == .rejectDelta) + outputDrop.arm() + } + #expect(!outputDrop.consume()) + #expect(decoder.inspect(keyFrame: true, nowNs: 0) == .accept) + #expect(!outputDrop.consume()) + #expect(decoder.inspect(keyFrame: false, nowNs: 1) == .reset) + #expect(outputDrop.consume()) + #expect(!outputDrop.consume()) + #expect(decoder.inspect(keyFrame: true, nowNs: 3) == .recovered(milliseconds: 0.000002)) + #expect(!outputDrop.consume()) + } + + @Test func unarmedDecoderDoesNotDiscardFrames() { + let check = ScreenSharingDecoderRecoveryCheck() + #expect(check.inspect(keyFrame: false, nowNs: 0) == .accept) + #expect(check.inspect(keyFrame: true, nowNs: 1) == .accept) + } + + @Test func lossRequiresANewKeyframeAndOccursOnlyOnce() { + let check = ScreenSharingDecoderRecoveryCheck() + check.arm(afterFrames: 3) + #expect(check.inspect(keyFrame: true, nowNs: 0) == .accept) + #expect(check.inspect(keyFrame: false, nowNs: 10_000_000) == .accept) + #expect(check.inspect(keyFrame: false, nowNs: 20_000_000) == .reset) + #expect(check.inspect(keyFrame: false, nowNs: 30_000_000) == .rejectDelta) + #expect(check.inspect(keyFrame: false, nowNs: 40_000_000) == .rejectDelta) + #expect(check.inspect(keyFrame: true, nowNs: 70_000_000) == .recovered(milliseconds: 50)) + #expect(check.inspect(keyFrame: false, nowNs: 80_000_000) == .accept) + #expect(check.inspect(keyFrame: true, nowNs: 90_000_000) == .accept) + } + + @Test func aKeyframeChosenForTheFaultCannotAlsoSatisfyRecovery() { + let check = ScreenSharingDecoderRecoveryCheck() + check.arm(afterFrames: 1) + #expect(check.inspect(keyFrame: true, nowNs: 0) == .reset) + #expect(check.inspect(keyFrame: false, nowNs: 1) == .rejectDelta) + #expect(check.inspect(keyFrame: true, nowNs: 2_000_000) == .recovered(milliseconds: 2)) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDiagnosticProfileTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDiagnosticProfileTests.swift new file mode 100644 index 000000000..543cafdac --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDiagnosticProfileTests.swift @@ -0,0 +1,400 @@ +import Foundation +import Testing + +@testable import CodevisorScreenSharing + +/// Stage 3t: the default-OFF experimental product profile and the single WebRTC initialization boundary. +/// Every test owns its own `ScreenSharingFieldTrials` instance with an injected apply closure: the real +/// process-global trials are never touched here, and no test depends on another's order. +@Suite struct ScreenSharingDiagnosticProfileTests { + + // MARK: profile resolution + + @Test func theProfileIsOffUnlessTheVariableNamesItExactly() throws { + #expect(try ScreenSharingDiagnosticProfile.resolve(environment: [:]) == nil) + #expect(try ScreenSharingDiagnosticProfile.resolve(environment: ["OTHER": "paced15-worker"]) == nil) + #expect( + try ScreenSharingDiagnosticProfile.resolve( + environment: [ScreenSharingDiagnosticProfile.environmentKey: ""]) == nil) + let on = try ScreenSharingDiagnosticProfile.resolve( + environment: [ScreenSharingDiagnosticProfile.environmentKey: "paced15-worker"]) + #expect(on == ScreenSharingDiagnosticProfile.paced15Worker) + } + + @Test func anUnknownValueFailsInsteadOfSelectingTheCandidate() { + for raw in [ + "paced15", "paced15-worker ", " paced15-worker", "Paced15-Worker", "PACED15-WORKER", "1", "true", + "off", "default", "paced35-worker", + ] { + #expect(throws: ScreenSharingError.self) { + try ScreenSharingDiagnosticProfile.resolve( + environment: [ScreenSharingDiagnosticProfile.environmentKey: raw]) + } + } + } + + @Test func theProfileMapIsExactAndLevelAware() { + let profile = ScreenSharingDiagnosticProfile.paced15Worker + #expect(profile.name == "paced15-worker") + #expect(profile.fieldTrials == ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"]) + #expect(profile.playoutExperimentLabel == "WebRTC-ForcePlayoutDelay min_ms:1,max_ms:15") + #expect(profile.trialSelection.playoutExperimentLabel == profile.playoutExperimentLabel) + #expect((profile.renderOnArrival, profile.maximumDrawableCount, profile.offMainPreparation) == (true, 2, true)) + // The capture request is the override only at level 0; every lower level requests the video rate. + #expect(profile.captureIntervalFPS(adaptiveLevel: 0) == 120) + for level in 1...3 { #expect(profile.captureIntervalFPS(adaptiveLevel: level) == nil) } + } + + @Test func theLevelAwareRequestStaysValidAsAdaptationLowersTheVideoRate() throws { + let profile = ScreenSharingDiagnosticProfile.paced15Worker + // Level 0: 60 fps video with the 120 override is a valid request. + let atZero = try ScreenSharingCaptureIntervalRequest( + videoFramesPerSecond: 60, overrideFramesPerSecond: profile.captureIntervalFPS(adaptiveLevel: 0)) + #expect(atZero.requestedFramesPerSecond == 120 && atZero.isOverride) + // Lower levels: nil keeps the video rate. A 120 request would still VALIDATE at 30 fps (the rule is only + // "at least the video rate, at most 120"), so dropping it below level 0 is this profile's deliberate choice — + // asking for four times the video rate while adaptation is shedding load is not what the profile means. + for (level, videoRate) in [(1, 30), (2, 30), (3, 20)] { + let request = try ScreenSharingCaptureIntervalRequest( + videoFramesPerSecond: videoRate, overrideFramesPerSecond: profile.captureIntervalFPS(adaptiveLevel: level)) + #expect(request.requestedFramesPerSecond == videoRate && !request.isOverride) + } + #expect(try ScreenSharingCaptureIntervalRequest(videoFramesPerSecond: 30, overrideFramesPerSecond: 120).isOverride) + // The genuinely invalid direction is a request BELOW the video rate, which the single validated path rejects. + #expect(throws: ScreenSharingError.self) { + try ScreenSharingCaptureIntervalRequest(videoFramesPerSecond: 60, overrideFramesPerSecond: 30) + } + } + + // MARK: installation state machine + + /// Records every applied dictionary in order; the optional gate lets a test hold the apply open. + private final class Recorder: @unchecked Sendable { + private let lock = NSLock() + private var applied: [[String: String]] = [] + var onApply: (@Sendable ([String: String]) -> Void)? + var count: Int { lock.lock(); defer { lock.unlock() }; return applied.count } + var last: [String: String]? { lock.lock(); defer { lock.unlock() }; return applied.last } + func record(_ trials: [String: String]) { + lock.lock(); applied.append(trials); lock.unlock() + onApply?(trials) + } + } + + @Test func theFirstSelectionIsAppliedOnceAndFixesTheProcess() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + #expect(trials.installed == nil) + let installed = try trials.install(profile: .paced15Worker) + #expect(installed.name == "paced15-worker") + #expect(recorder.count == 1 && recorder.last == ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"]) + #expect(trials.installed == installed) + // Re-installing the same request changes nothing and applies nothing further. + #expect(try trials.install(profile: .paced15Worker) == installed) + #expect(recorder.count == 1) + // ensureInstalled accepts what is in force rather than pinning the default over it. + #expect(trials.ensureInstalled() == installed) + #expect(recorder.count == 1) + } + + @Test func provenanceNamesNeverCreateAFalseConflict() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + let first = try trials.install( + .init(name: "probe options", trials: ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"])) + // Same native dictionary asked for under the product profile's name: idempotent, and the FIRST provenance wins. + let second = try trials.install(profile: .paced15Worker) + #expect(second == first && second.name == "probe options") + #expect(recorder.count == 1) + } + + @Test func aDifferentTrialMapFailsBeforeAnyPeerExists() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + try trials.install(profile: .paced15Worker) + #expect(throws: ScreenSharingError.self) { + try trials.install(.init(name: "probe options", trials: ["WebRTC-ForcePlayoutDelay": "min_ms:0,max_ms:0"])) + } + #expect(throws: ScreenSharingError.self) { try trials.install(.default) } + #expect(recorder.count == 1 && trials.installed?.name == "paced15-worker") + } + + @Test func installingTheDefaultAlsoFixesTheProcessAgainstALaterProfile() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + let installed = trials.ensureInstalled() + #expect(installed == .default && installed.playoutExperimentLabel == nil) + #expect(recorder.count == 1 && recorder.last == [:]) // the empty apply is still a decision + #expect(throws: ScreenSharingError.self) { try trials.install(profile: .paced15Worker) } + #expect(trials.installed == .default && recorder.count == 1) + } + + // MARK: ordering — readiness is not published until the apply returns + + @Test func readinessIsNotVisibleWhileTheApplyIsStillRunning() throws { + let recorder = Recorder() + let observed = Box(ScreenSharingFieldTrials.Selection.default) // non-nil so a missed write fails the test + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + recorder.onApply = { [weak trials] _ in + // Inside the apply, on the applying thread: the selection must NOT be observable yet. + observed.value = trials?.installed + } + try trials.install(profile: .paced15Worker) + #expect(observed.value == nil) + #expect(trials.installed?.name == "paced15-worker") + } + + @Test func aSecondCallerBlocksAtTheWaitingBoundaryUntilTheApplyReturns() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + let events = EventLog() + let applyStarted = DispatchSemaphore(value: 0) + let releaseApply = DispatchSemaphore(value: 0) + // Signalled by whichever happens first: the second caller REACHING the wait, or the second caller returning. + let secondProgressed = DispatchSemaphore(value: 0) + let secondReturned = DispatchSemaphore(value: 0) + let firstReturned = DispatchSemaphore(value: 0) + + // The internal observer fires exactly where a caller is about to block on the apply in progress. + trials.onWaitingForApply = { + events.append("secondReachedWait") + secondProgressed.signal() + } + recorder.onApply = { _ in + events.append("applyStarted") + applyStarted.signal() + releaseApply.wait() // the test controls exactly when the apply finishes + events.append("applyFinished") + } + DispatchQueue.global().async { + _ = try? trials.install(profile: .paced15Worker) + events.append("firstReturned") + firstReturned.signal() + } + applyStarted.wait() // the apply is in flight and has NOT returned + DispatchQueue.global().async { + let selection = trials.ensureInstalled() + events.append("secondReturned(\(selection.name))") + secondProgressed.signal() // in case it returned without ever waiting (the regression this test targets) + secondReturned.signal() + } + secondProgressed.wait() // proceed only once the second caller has actually reached the wait, or returned early + // Readiness must not have been published: a caller that returned here would have used an unapplied selection. + #expect(events.entries.contains("secondReachedWait")) + #expect(!events.entries.contains { $0.hasPrefix("secondReturned") }) + releaseApply.signal() + firstReturned.wait() + secondReturned.wait() + let recorded = events.entries + let applyFinished = try #require(recorded.firstIndex(of: "applyFinished")) + let second = try #require(recorded.firstIndex(of: "secondReturned(paced15-worker)")) + let reachedWait = try #require(recorded.firstIndex(of: "secondReachedWait")) + #expect(reachedWait < applyFinished && applyFinished < second) + #expect(recorded.first == "applyStarted") + #expect(recorder.count == 1) // single flight: the second caller never applied anything + } + + /// Supporting evidence only: a broad concurrent smoke over the same boundary. The ordering guarantee itself is + /// pinned by `aSecondCallerBlocksAtTheWaitingBoundaryUntilTheApplyReturns`. + @Test func concurrentCallersApplyExactlyOnceAndAgreeOnTheResult() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + let results = EventLog() + let group = DispatchGroup() + for index in 0..<8 { + DispatchQueue.global().async(group: group) { + // Identical native map, eight different provenance names, plus ensureInstalled callers. + if index % 2 == 0 { + let selection = try? trials.install( + .init(name: "caller \(index)", trials: ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"])) + results.append(selection?.trials["WebRTC-ForcePlayoutDelay"] ?? "threw") + } else { + results.append(trials.ensureInstalled().trials["WebRTC-ForcePlayoutDelay"] ?? "default") + } + } + } + group.wait() + #expect(recorder.count == 1) // single flight, whichever caller won + let entries = results.entries + #expect(entries.count == 8) + let installed = try #require(trials.installed) + let inForce = installed.trials["WebRTC-ForcePlayoutDelay"] ?? "default" + // Nobody observed a half-installed state: every caller either agrees with what is in force, or -- only when an + // `ensureInstalled` caller won and fixed the default first -- was refused for asking a different map. + #expect(entries.allSatisfy { $0 == inForce || $0 == "threw" }) + if entries.contains("threw") { #expect(inForce == "default") } + if inForce != "default" { #expect(entries.allSatisfy { $0 == inForce }) } + } + + // MARK: real wiring boundaries + + @Test func theProbeOptionsSelectionCoversEveryExistingTrialCombination() throws { + typealias Selection = ScreenSharingFieldTrials.Selection + // The exact factory `ProbeOptions.fieldTrialSelection` forwards to. + let none = Selection.probeOptions(jitterWindowFrames: nil, lowLatencyPlayout: false, playoutDelayBoundsMs: nil) + #expect(none.trials.isEmpty && none.name == "default") + #expect( + Selection.probeOptions(jitterWindowFrames: nil, lowLatencyPlayout: true, playoutDelayBoundsMs: nil).trials + == ["WebRTC-ForcePlayoutDelay": "min_ms:0,max_ms:0"]) + for bounds in [(min: 1, max: 15), (min: 1, max: 35), (min: 0, max: 500)] { + #expect( + Selection.probeOptions(jitterWindowFrames: nil, lowLatencyPlayout: false, playoutDelayBoundsMs: bounds).trials + == ["WebRTC-ForcePlayoutDelay": "min_ms:\(bounds.min),max_ms:\(bounds.max)"]) + } + let jitter = Selection.probeOptions( + jitterWindowFrames: 90, lowLatencyPlayout: false, playoutDelayBoundsMs: nil) + #expect(jitter.trials == ["WebRTC-JitterEstimatorConfig": "max_frame_size_percentile:0.95,frame_size_window:90"]) + let both = Selection.probeOptions(jitterWindowFrames: 90, lowLatencyPlayout: true, playoutDelayBoundsMs: nil) + #expect(both.trials.count == 2 && both.trials["WebRTC-ForcePlayoutDelay"] == "min_ms:0,max_ms:0") + #expect(both.name == "probe options") + // The probe's paced15 selection is exactly the product profile's native map, so the two roles never conflict. + #expect( + Selection.probeOptions(jitterWindowFrames: nil, lowLatencyPlayout: false, playoutDelayBoundsMs: (min: 1, max: 15)) + .installs(ScreenSharingDiagnosticProfile.paced15Worker.trialSelection)) + } + + /// Covers what the bootstrap itself does: it pins a selection and publishes labels derived only from the INSTALLED + /// map. It does not exercise a factory: `ScreenSharingPeer.init` has no injection point (it is bound to `.process`) + /// precisely so that no fake initializer can authorize a real RTC factory, and the ordering of bootstrap-before- + /// factory inside `init` is therefore SOURCE-REVIEWED, not exercised here. + @Test @MainActor func theBootstrapPinsASelectionAndLabelsOnlyWhatIsInstalled() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + try trials.install(profile: .paced15Worker) + let metrics = ScreenSharingMetrics() + let installed = ScreenSharingPeer.bootstrapTrials(trials, publishingInto: metrics) + #expect(installed.trials == ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"]) + // After the bootstrap returns, the applied selection is readable — what `init` relies on before building anything. + #expect(trials.installed == installed) + #expect(metrics.snapshot().labels["playoutExperiment"] == "WebRTC-ForcePlayoutDelay min_ms:1,max_ms:15") + #expect(metrics.snapshot().labels["fieldTrialProvenance"] == "paced15-worker") + #expect(recorder.count == 1) // the bootstrap pinned the existing selection, it did not re-apply + + // With nothing installed, the bootstrap pins the default and publishes NO playout label (absence preserved). + let plainRecorder = Recorder() + let plainTrials = ScreenSharingFieldTrials(apply: { plainRecorder.record($0) }) + let plainMetrics = ScreenSharingMetrics() + #expect(ScreenSharingPeer.bootstrapTrials(plainTrials, publishingInto: plainMetrics) == .default) + #expect(plainMetrics.snapshot().labels["playoutExperiment"] == nil) + #expect(plainMetrics.snapshot().labels["fieldTrialProvenance"] == "default") + // and a profile request after that bootstrap fails instead of claiming to be active. + #expect(throws: ScreenSharingError.self) { try plainTrials.install(profile: .paced15Worker) } + } + + @Test @MainActor func provenanceAndActiveProfileAreSeparateFacts() throws { + let recorder = Recorder() + let trials = ScreenSharingFieldTrials(apply: { recorder.record($0) }) + // The probe (or another role) installed the identical map first, so provenance is NOT the profile name. + try trials.install(.init(name: "probe options", trials: ["WebRTC-ForcePlayoutDelay": "min_ms:1,max_ms:15"])) + let metrics = ScreenSharingMetrics() + ScreenSharingPeer.bootstrapTrials(trials, publishingInto: metrics) + let labels = metrics.snapshot().labels + #expect(labels["fieldTrialProvenance"] == "probe options") // first installer's provenance, preserved + #expect(labels["playoutExperiment"] == "WebRTC-ForcePlayoutDelay min_ms:1,max_ms:15") // actual installed map + // The product roles publish "active" from the validated profile whose settings they wire, which install(profile:) + // has just proved compatible with the installed map — a different map would have thrown instead. + #expect( + try trials.install(profile: .paced15Worker).installs(ScreenSharingDiagnosticProfile.paced15Worker.trialSelection)) + } + + // NOT covered by a test, deliberately: that `ScreenSharingDiagnosticProfile.process()` parses once per process and + // replays a cached configuration failure. It is a `static let` holding a `Result`, so exercising the caching would + // mean either reading the live process environment (which an external configuration could change, and which proves + // nothing about caching) or adding production machinery purely to observe Swift's static initialization. The pure + // parser above is fully covered; the caching is reported as SOURCE-REVIEWED. + + // MARK: capture request state — the PURE value type only + // + // Scope, stated exactly: this covers `ScreenSharingCaptureRequestState` in isolation — that `validated` refuses an + // out-of-range request without touching state, and that `commit` is what moves the stored request and its labels + // together. It does NOT exercise the update transaction, a thrown apply, or a stale generation: those live in + // `ScreenSharingCapture.applyIntervalUpdate`, which this file never calls. Root's separate + // `ScreenSharingCaptureUpdateTransactionTests.swift` invokes that real transaction. + + @Test func theRequestStateRefusesAnInvalidRequestAndMovesStateAndLabelsOnlyOnCommit() throws { + let metrics = ScreenSharingMetrics() + var state = ScreenSharingCaptureRequestState(overrideFramesPerSecond: 120) + let level0 = try ScreenSharingVideoConfiguration(width: 1920, height: 1080, framesPerSecond: 60) + // Successful update at level 0: validate, (stream update succeeds), commit. + let first = try state.validated(video: level0, override: 120) + state.commit(override: 120, request: first, metrics: metrics) + #expect(state.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels["captureRequestedMinimumFrameIntervalFPS"] == "120") + #expect(metrics.snapshot().labels["captureRequestedFrameIntervalOverride"] == "120") + + // Validation failure (override below the new video rate): nothing is committed, nothing is relabelled. + let level1 = try ScreenSharingVideoConfiguration(width: 1280, height: 720, framesPerSecond: 30) + #expect(throws: ScreenSharingError.self) { try state.validated(video: level1, override: 20) } + #expect(state.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels["captureRequestedMinimumFrameIntervalFPS"] == "120") + + // Validating alone changes nothing: the stored request and labels still describe the last COMMITTED request. + // (Whether the transaction reaches commit after a thrown or stale apply is not decided here — see the note above.) + let pending = try state.validated(video: level1, override: nil) + #expect(pending.requestedFramesPerSecond == 30 && !pending.isOverride) + #expect(state.overrideFramesPerSecond == 120) + #expect(metrics.snapshot().labels["captureRequestedMinimumFrameIntervalFPS"] == "120") + + // Only a commit moves the stored request and its telemetry together. + state.commit(override: nil, request: pending, metrics: metrics) + #expect(state.overrideFramesPerSecond == nil) + #expect(metrics.snapshot().labels["captureRequestedMinimumFrameIntervalFPS"] == "30") + #expect(metrics.snapshot().labels["captureRequestedFrameIntervalOverride"] == "none") + } +} + +/// Single mutable cell shared across threads. +private final class Box: @unchecked Sendable { + private let lock = NSLock() + private var stored: Value? + init(_ value: Value?) { stored = value } + var value: Value? { + get { lock.lock(); defer { lock.unlock() }; return stored } + set { lock.lock(); stored = newValue; lock.unlock() } + } +} + +/// Append-only ordered log shared across threads. +private final class EventLog: @unchecked Sendable { + private let lock = NSLock() + private var values: [String] = [] + func append(_ value: String) { lock.lock(); values.append(value); lock.unlock() } + var entries: [String] { lock.lock(); defer { lock.unlock() }; return values } +} + +/// The renderer selection the product viewer forwards for this profile, exercised on the real initializer. +@Suite @MainActor struct ScreenSharingDiagnosticProfileRendererTests { + @Test func theProfileSelectsTheWorkerPathAndTheDefaultKeepsTheProductRenderer() throws { + let profile = ScreenSharingDiagnosticProfile.paced15Worker + let profiled = ScreenSharingMetrics() + let worker = try ScreenSharingMetalView( + mailbox: ScreenSharingFrameMailbox(), metrics: profiled, renderOnArrival: profile.renderOnArrival, + maximumDrawableCount: profile.maximumDrawableCount, offMainPreparation: profile.offMainPreparation) + var labels = profiled.snapshot().labels + #expect(labels["renderPreparation"] == "off-main serial worker (diagnostic)") + #expect(labels["drawableAcquisitionPath"] == "CAMetalLayer.nextDrawable on render worker, default timeout") + #expect(labels["maximumDrawableCount"] == "2") + #expect(labels["displaySync"] == "enabled") // no unsynced option belongs to this profile + #expect(labels["frameSelection"] == "before drawable acquisition") + worker.stop() + #expect(profiled.snapshot().labels["rendererStopped"] == "true") // terminal stop preserved + worker.stop() // idempotent + + // Default (profile OFF): the product renderer is untouched — display-link drive, three drawables, main actor. + let plain = ScreenSharingMetrics() + let standard = try ScreenSharingMetalView(mailbox: ScreenSharingFrameMailbox(), metrics: plain) + labels = plain.snapshot().labels + #expect(labels["renderPreparation"] == "main actor (MTKView)") + #expect(labels["drawableAcquisitionPath"] == "MTKView.currentDrawable on main actor") + #expect(labels["maximumDrawableCount"] == "3") + standard.stop() + } + + // NOT covered here: the initializer's refusals (off-main without arrival rendering, a drawable count outside 2...3). + // Attempting them in this test bundle aborted the process with an uncaught NSException whose stack ended in + // `-[MTKView dealloc]`. That abort was observed interactively while writing Stage 3t and was NOT captured to any + // file, so no crash log exists to cite and it has not been reproduced. The cause is UNRESOLVED — the stack showed + // where the abort surfaced, which is not by itself evidence of an Apple allocation-contract defect. It is a + // pre-existing failure-path limitation of testing this initializer, unchanged by Stage 3t, reported rather than + // papered over; no renderer change is made for it here. +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDrawTimestampRecordTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDrawTimestampRecordTests.swift new file mode 100644 index 000000000..89167068e --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingDrawTimestampRecordTests.swift @@ -0,0 +1,40 @@ +import Testing + +@testable import CodevisorScreenSharing + +@Suite struct ScreenSharingDrawTimestampRecordTests { + @Test func recordsFirstDrawStartPerCodeSkipsRepeatsAndTruncatesAtTheLimit() { + var record = ScreenSharingDrawTimestampRecord(limit: 3) + let first = record.record(code: 0, startedAtSeconds: 100.000) + let redraw = record.record(code: 0, startedAtSeconds: 100.004) // redraw of the same code: first start kept + let second = record.record(code: 1, startedAtSeconds: 100.017) + let skipped = record.record(code: 3, startedAtSeconds: 100.051) // skipped code 2 is simply absent + #expect(first && !redraw && second && skipped) + #expect(!record.truncated) + let overLimit = record.record(code: 4, startedAtSeconds: 100.067) // limit reached + #expect(!overLimit) + #expect(record.truncated && record.samples.count == 3) + #expect(record.samples[0] == .init(code: 0, startedAtSeconds: 100.000)) + #expect(record.samples.map(\.code) == [0, 1, 3]) + } + + @Test func codeTimeMappingFollowsTheWorkloadSequenceAndAPausedWorkloadAddsNoEntries() { + // The code drawn at time t is floor((t - start) * fps); a frozen (paused) workload keeps redrawing + // the same code, which must not add samples. Draw starts are placed mid-slot so the expected + // codes do not depend on floating-point rounding at slot boundaries. + var sequence = ScreenSharingWorkloadSequence(framesPerSecond: 60, startedAtSeconds: 200) + var record = ScreenSharingDrawTimestampRecord() + for tick in 0..<5 { + let t = 200 + (Double(tick) + 0.5) / 60 // mid-slot draw start: floor is exactly `tick` + record.record(code: sequence.drawn(atSeconds: t), startedAtSeconds: t) + } + #expect(record.samples.map(\.code) == [0, 1, 2, 3, 4]) + sequence.freezeAtLastDrawn(atSeconds: 200.09) + for tick in 5..<10 { + let t = 200 + (Double(tick) + 0.5) / 60 // mid-slot draw start: floor is exactly `tick` + record.record(code: sequence.drawn(atSeconds: t), startedAtSeconds: t) + } + #expect(record.samples.count == 5 && record.samples.last?.code == 4 && !record.truncated) + #expect(ScreenSharingDrawTimestampRecord.defaultLimit == 20_000) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingEncoderDropLogTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingEncoderDropLogTests.swift new file mode 100644 index 000000000..34c0e0bfc --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingEncoderDropLogTests.swift @@ -0,0 +1,40 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingEncoderDropLogTests { + @Test func retainsOnlyExpectedNumericCounters() { + let message = + "(video_stream_encoder.cc:1686): Number of frames: captured 900, dropped (due to congestion window pushback) 12, dropped (due to encoder blocked) 210, interval_ms 60000\n" + #expect( + ScreenSharingEncoderDropLog.counters(in: message) == [ + "rtcLoggedInputFrames": 900, "rtcCongestionWindowDrops": 12, "rtcEncoderQueueDrops": 210, + ]) + } + + @Test(arguments: [ + "ICE candidate with address and credentials", + "Number of frames: captured 900, unknown counter 12, dropped (due to encoder blocked) 210, interval_ms 60000", + "Number of frames: captured -1, dropped (due to congestion window pushback) 0, dropped (due to encoder blocked) 0, interval_ms 60000", + "Number of frames: captured overflow, dropped (due to congestion window pushback) 0, dropped (due to encoder blocked) 0, interval_ms 60000", + ]) + func ignoresOtherOrMalformedMessages(_ message: String) { + #expect(ScreenSharingEncoderDropLog.counters(in: message).isEmpty) + } + + @Test func dropDiagnosticsKeepOnlyPinnedDropMessagesWithoutAddresses() { + let message = + "(video_stream_encoder.cc:1643): Same/old NTP timestamp (5 <= 5) for incoming frame. Dropping. this 0x7f8a1b2c" + #expect( + ScreenSharingEncoderDropLog.dropDiagnostic(in: message) + == "(video_stream_encoder.cc:1643): Same/old NTP timestamp (5 <= 5) for incoming frame. Dropping. this 0x…") + #expect(ScreenSharingEncoderDropLog.dropDiagnostic(in: "Dropping frame. Too large for target bitrate.") != nil) + #expect( + ScreenSharingEncoderDropLog.dropDiagnostic( + in: + "Number of frames: captured 60, dropped (due to congestion window pushback) 0, dropped (due to encoder blocked) 0, interval_ms 5000" + ) == nil) + #expect(ScreenSharingEncoderDropLog.dropDiagnostic(in: "") == nil) + let long = "Drop Frame: " + String(repeating: "x", count: 400) + #expect(ScreenSharingEncoderDropLog.dropDiagnostic(in: long)?.count == 160) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingFrameDeliveryAuditTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingFrameDeliveryAuditTests.swift new file mode 100644 index 000000000..42ba85922 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingFrameDeliveryAuditTests.swift @@ -0,0 +1,225 @@ +import CoreVideo +import Foundation +import QuartzCore +import Testing + +@testable import CodevisorScreenSharing + +/// The receiver-only frame-delivery audit: injected clock, window edges, +/// capacity/overflow, identity across decoder generations and reused buffers, +/// duplicates/missing/out-of-order/late events, scalar-only records. +struct ScreenSharingFrameDeliveryAuditTests { + private final class Clock: @unchecked Sendable { + private let lock = NSLock() + private var value: Int64 = 0 + private(set) var reads = 0 + func set(_ ns: Int64) { lock.withLock { value = ns } } + var read: @Sendable () -> Int64 { + { [self] in + self.lock.withLock { + self.reads += 1; return self.value + } + } + } + } + + private func audit( + begin: Double = 1, duration: Double = 2, capacity: Int = 64 + ) throws -> (ScreenSharingFrameDeliveryAudit, Clock) { + let clock = Clock() + let audit = ScreenSharingFrameDeliveryAudit( + window: try .init(beginSeconds: begin, durationSeconds: duration), capacity: capacity, clock: clock.read) + return (audit, clock) + } + + private func makeBuffer() throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + return try #require(pixel) + } + + @Test func windowEdgesOriginAndCapacityAreExact() throws { + let (audit, clock) = try audit(begin: 1, duration: 2, capacity: 3) + clock.set(500) + audit.record(.decoderInput, .init(sequence: 1, generation: 1), rtpTimestamp: 10) // before the origin is known + audit.start(originNs: 1_000_000_000) + audit.start(originNs: 5) // a second start is ignored + // 1 ns before the window + clock.set(1_999_999_999); audit.record(.decoderInput, .init(sequence: 1, generation: 1), rtpTimestamp: 10) + // begin is inclusive + clock.set(2_000_000_000); audit.record(.decoderInput, .init(sequence: 2, generation: 1), rtpTimestamp: 11) + // last ns inside + clock.set(3_999_999_999); audit.record(.vtOutput, .init(sequence: 2, generation: 1), rtpTimestamp: 11) + // end is exclusive + clock.set(4_000_000_000); audit.record(.rtcRendererCallback, .init(sequence: 2, generation: 1), rtpTimestamp: 11) + clock.set(3_000_000_000) + audit.record(.selection, .init(sequence: 2, generation: 1), rtpTimestamp: 11) + audit.record(.submission, .init(sequence: 2, generation: 1), rtpTimestamp: 11) // capacity 3 reached here + let s = audit.snapshot() + #expect(s.originNs == 1_000_000_000 && s.windowNs == [2_000_000_000, 4_000_000_000]) + #expect((s.beforeOrigin, s.outsideWindowBefore, s.outsideWindowAfter, s.recorded, s.overflow) == (1, 1, 1, 3, 1)) + #expect(s.firstEventNs == 2_000_000_000 && s.lastEventNs == 3_999_999_999) // true min/max, not first/last append + #expect(s.stages == [1, 2, 4] && s.sequences == [2, 2, 2] && s.rtpTimestamps == [11, 11, 11]) + #expect( + s.preallocatedRecordBytes == 3 * ScreenSharingFrameDeliveryAudit.recordByteStride && s.recordByteStride == 32) + #expect(s.seenDictionaryEntries == 1) + } + + @Test func inputEpochsAndConfiguredEventsFollowTheRealInputConfigureOutputOrder() throws { + let (audit, clock) = try audit() + audit.start(originNs: 0); clock.set(1_500_000_000) + // first frame: input (epoch 0) -> its decode creates the VT session -> output carries epoch 0 + let a = audit.decoderInput(rtpTimestamp: 100) + #expect(a == .init(sequence: 1, generation: 0)) + #expect(audit.decoderConfigured(a, rtpTimestamp: 100) == 1) + audit.record(.vtOutput, a, rtpTimestamp: 100) + // frames decoded by that session carry epoch 1 + let b = audit.decoderInput(rtpTimestamp: 101) + #expect(b == .init(sequence: 2, generation: 1)) + // a reset: the triggering frame was input in epoch 1, the new session is event 2, later inputs are epoch 2 + let c = audit.decoderInput(rtpTimestamp: 102) + #expect(audit.decoderConfigured(c, rtpTimestamp: 102) == 2 && c.generation == 1) + let d = audit.decoderInput(rtpTimestamp: 103) + #expect(d == .init(sequence: 4, generation: 2)) // the global sequence never restarts + let s = audit.snapshot() + #expect(s.decoderGenerations == 2 && s.sequenceRange == [1, 4] && s.perStage["decoderConfigured"] == 2) + let configured = zip(s.stages, zip(s.sequences, s.valueNs)).filter { $0.0 == 12 }.map { $0.1 } + #expect(configured.map { $0.0 } == [1, 3] && configured.map { $0.1 } == [1, 2]) + } + + @Test func theBridgeStampClearsAStaleIdentityOnAReusedBuffer() throws { + let buffer = try makeBuffer() + #expect(ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer) == nil) + let a = ScreenSharingFrameDeliveryAudit.Identity(sequence: 1, generation: 0) + let b = ScreenSharingFrameDeliveryAudit.Identity(sequence: 2, generation: 0) + ScreenSharingFrameDeliveryAudit.stamp(a, on: buffer) + #expect(ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer) == a) + ScreenSharingFrameDeliveryAudit.stamp(nil, on: buffer) // an output without identity clears the reused buffer + #expect(ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer) == nil) + ScreenSharingFrameDeliveryAudit.stamp(b, on: buffer) + #expect(ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer) == b) + ScreenSharingFrameDeliveryAudit.removeIdentity(from: buffer) + #expect(ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer) == nil) + // Not exercised here: the disabled decoder/bridge path. That guarantee is the source's nil guards — + // ScreenSharingDecoder's VT output `if let audit = context.audit { stamp…; record… }` and + // ScreenSharingPeerRenderer's `if let audit { readIdentity…; record… }` — which never call stamp/read + // when the audit is nil; those private paths need a decoded frame and are not unit-testable without media. + } + + @Test func outOfOrderSuppliedTimestampsKeepAppendOrderWithTrueMinMaxBounds() throws { + let (audit, _) = try audit(begin: 0, duration: 10) + audit.start(originNs: 0) + let id = ScreenSharingFrameDeliveryAudit.Identity(sequence: 1, generation: 0) + audit.record(.submission, id, rtpTimestamp: 1, atNs: 2_000_000_000) + audit.record(.selection, id, rtpTimestamp: 1, atNs: 1_000_000_000) // appended later, earlier stamp + audit.record(.gpuCompletionCallback, id, rtpTimestamp: 1, valueNs: 1, atNs: 3_000_000_000) + let s = audit.snapshot() + #expect(s.atNs == [2_000_000_000, 1_000_000_000, 3_000_000_000]) // append order preserved + #expect(s.firstEventNs == 1_000_000_000 && s.lastEventNs == 3_000_000_000) // true bounds + } + + @Test func anAuditConstructedWithTheDefaultClockStampsOnCACurrentMediaTimeInNanoseconds() throws { + // default clock + let audit = ScreenSharingFrameDeliveryAudit(window: try .init(beginSeconds: 0, durationSeconds: 10)) + let before = Int64(CACurrentMediaTime() * 1_000_000_000) + let now = audit.now() + audit.start(originNs: before) + audit.record(.selection, .init(sequence: 1, generation: 0), rtpTimestamp: 1) // stamped by the default clock + let after = Int64(CACurrentMediaTime() * 1_000_000_000) + let s = audit.snapshot() + #expect(before <= now && now <= after) + #expect(s.atNs.count == 1 && before <= s.atNs[0] && s.atNs[0] <= after) + #expect(ScreenSharingFrameDeliveryAudit.defaultClock() >= after) + let (injected, clock) = try self.audit() + clock.set(123) + #expect(injected.now() == 123) // an injected clock replaces the default entirely + } + + @Test func mailboxReplacementIsObservableAsAScalarEvent() throws { + let (audit, clock) = try audit(begin: 0, duration: 10) + audit.start(originNs: 0); clock.set(1_000) + let mailbox = ScreenSharingFrameMailbox() + let first = ScreenSharingVideoFrame( + pixelBuffer: try makeBuffer(), timestampNs: 1, rtpTimestamp: 11, + deliveryAuditIdentity: .init(sequence: 1, generation: 0)) + #expect(mailbox.put(first) == nil) + let replaced = mailbox.put( + .init( + pixelBuffer: try makeBuffer(), timestampNs: 2, rtpTimestamp: 12, + deliveryAuditIdentity: .init(sequence: 2, generation: 0))) + // policy unchanged: newest wins, replacement counted + #expect(replaced?.rtpTimestamp == 11 && mailbox.droppedFrames == 1) + audit.record(.rtcRendererCallback, first.deliveryAuditIdentity, rtpTimestamp: 11) + audit.record(.mailboxReplaced, replaced?.deliveryAuditIdentity, rtpTimestamp: replaced?.rtpTimestamp ?? 0) + let s = audit.snapshot() + #expect(s.perStage["mailboxReplaced"] == 1 && s.coverage["absent.selectionAfterRendererCallback"] == 1) + #expect(s.coverage["absent.selectionAfterRendererCallback.mailboxReplaced"] == 1) // the absence is explained + #expect(s.truncation["missingIdentityEvents"] == 0 && s.truncation["overflow"] == 0) + } + + @Test func duplicatesMissingIdentitiesOutOfOrderAndLateEventsAreDeclaredNotPaired() throws { + let (audit, clock) = try audit() + audit.start(originNs: 0); clock.set(1_500_000_000) + let id = audit.decoderInput(rtpTimestamp: 7) + // out of order: presented before submission, zero presented time + audit.record(.presentedResult, id, rtpTimestamp: 7, valueNs: 0) + audit.record(.submission, id, rtpTimestamp: 7) + audit.record(.submission, id, rtpTimestamp: 7) // duplicate stage for the same identity + audit.record(.rtcRendererCallback, nil, rtpTimestamp: 8) // a frame whose buffer carried no identity + audit.close() + audit.record(.gpuCompletionCallback, id, rtpTimestamp: 7, valueNs: 1) // late after close + _ = audit.decoderInput(rtpTimestamp: 9) // identity is still assigned but the event is late + let s = audit.snapshot() + #expect( + s.recorded == 5 && s.duplicateEvents == 1 && s.missingIdentityEvents == 1 && s.lateAfterClose == 2 + && s.closedAtNs == 1_500_000_000) + #expect( + s.coverage["presentedResultZero"] == 1 && s.coverage["absent.completionCallbackAfterSubmission"] == 1 + && s.coverage["framesWithIdentity"] == 1) + #expect(s.coverage["reached.presentedResult"] == 1 && s.coverage["absent.vtOutputAfterDecoderInput"] == 1) + #expect(s.truncation["lateAfterClose"] == 2 && s.truncation["missingIdentityEvents"] == 1) + // missing identity stored as sequence 0, never paired + #expect(s.sequences.contains(0) && s.perStage["rtcRendererCallback"] == 1) + #expect(audit.isClosed) + } + + @Test func explicitTimestampsAreUsedWithoutReadingTheClock() throws { + let (audit, clock) = try audit() + audit.start(originNs: 0) + let before = clock.reads + audit.record(.acquisitionBegin, .init(sequence: 1, generation: 1), rtpTimestamp: 1, atNs: 1_200_000_000) + audit.record(.acquisitionEnd, .init(sequence: 1, generation: 1), rtpTimestamp: 1, valueNs: 1, atNs: 1_210_000_000) + #expect(clock.reads == before) + let s = audit.snapshot() + #expect(s.atNs == [1_200_000_000, 1_210_000_000] && s.valueNs == [0, 1]) + } + + @Test func recordsHoldScalarsOnlyAndBuffersAreReleased() throws { + let (audit, clock) = try audit() + audit.start(originNs: 0); clock.set(1_500_000_000) + weak var weakBuffer: CVPixelBuffer? + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + let id = audit.decoderInput(rtpTimestamp: 3) + ScreenSharingFrameDeliveryAudit.attach(id, to: buffer) + audit.record(.vtOutput, ScreenSharingFrameDeliveryAudit.readIdentity(from: buffer), rtpTimestamp: 3) + let frame = ScreenSharingVideoFrame( + pixelBuffer: buffer, timestampNs: 1, rtpTimestamp: 3, deliveryAuditIdentity: id) + audit.record(.rtcRendererCallback, frame.deliveryAuditIdentity, rtpTimestamp: frame.rtpTimestamp) + } + #expect(weakBuffer == nil) // three recorded events, no retained buffer + #expect(audit.snapshot().recorded == 3) + } + + @Test func frameIdentityDefaultsToNilWhenTheAuditIsOff() throws { + let frame = ScreenSharingVideoFrame(pixelBuffer: try makeBuffer(), timestampNs: 1) + #expect(frame.deliveryAuditIdentity == nil) + #expect(throws: ScreenSharingError.self) { + try ScreenSharingFrameDeliveryAudit.Window(beginSeconds: -1, durationSeconds: 1) + } + #expect(throws: ScreenSharingError.self) { + try ScreenSharingFrameDeliveryAudit.Window(beginSeconds: 0, durationSeconds: 0) + } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingHEVCFormatTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingHEVCFormatTests.swift new file mode 100644 index 000000000..dc3dd0c3a --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingHEVCFormatTests.swift @@ -0,0 +1,65 @@ +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingHEVCFormatTests { + private func configuration(profile: UInt8 = 4, chroma: UInt8 = 3, luma: UInt8 = 0, color: UInt8 = 0) -> Data { + var bytes = [UInt8](repeating: 0, count: 23) + bytes[0] = 1 + bytes[1] = 0x20 | profile + bytes[16] = 0xfc | chroma + bytes[17] = 0xf8 | luma + bytes[18] = 0xf8 | color + return Data(bytes) + } + + @Test func validatesExplicitEightBitChromaWithoutInferringFromProfile() throws { + let full = try ScreenSharingHEVCFormat(configuration: configuration()) + try full.validate(for: .hevc444) + #expect(full.profile == 4 && full.chroma == 3 && full.lumaDepth == 8 && full.chromaDepth == 8) + let main = try ScreenSharingHEVCFormat(configuration: configuration(profile: 1, chroma: 1)) + try main.validate(for: .hevc) + #expect(throws: (any Error).self) { try main.validate(for: .hevc444) } + #expect(throws: (any Error).self) { try full.validate(for: .hevc) } + #expect(throws: (any Error).self) { try full.validate(for: .h264) } + } + + @Test func rejectsSilentChromaAndDepthFallback() throws { + for bytes in [ + configuration(chroma: 1), configuration(chroma: 2), configuration(profile: 1), + configuration(luma: 2), configuration(color: 2), + ] { + let actual = try ScreenSharingHEVCFormat(configuration: bytes) + #expect(throws: (any Error).self) { try actual.validate(for: .hevc444) } + } + } + + @Test func rejectsTruncationAndUnknownConfigurationVersion() { + for length in 0..<23 { + #expect(throws: (any Error).self) { + try ScreenSharingHEVCFormat(configuration: configuration().prefix(length)) + } + } + var unknown = configuration() + unknown[0] = 2 + #expect(throws: (any Error).self) { try ScreenSharingHEVCFormat(configuration: unknown) } + } + + @Test func rejectsMalformedHEVCHeadersBeforeCreatingHardwareDecoder() { + let decoder = ScreenSharingDecoder(metrics: ScreenSharingMetrics(), codec: .hevc444) { _ in + Issue.record("Malformed NAL must not produce a decoded frame.") + } + for header: [UInt8] in [[0x40], [0xc0, 1], [0x40, 0]] { + let frame = ScreenSharingEncodedFrame( + data: Data([0, 0, 0, 1] + header), timestampNs: 0, + rtpTimestamp: 0, width: 64, height: 64, isKeyFrame: true) + #expect(throws: (any Error).self) { try decoder.decode(frame) } + } + } + + @Test func preservesHEVCParameterAndSliceNALsAcrossFramingConversion() throws { + let units = [Data([0x40, 1, 0xaa]), Data([0x42, 1, 0xbb]), Data([0x44, 1, 0xcc]), Data([0x26, 1, 0xdd])] + let bytes = try NALUnitBitstream.annexB(NALUnitBitstream.lengthPrefixed(units)) + #expect(try NALUnitBitstream.nalUnits(bytes) == units) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingICETests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingICETests.swift new file mode 100644 index 000000000..b4487e1ad --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingICETests.swift @@ -0,0 +1,32 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingICETests { + @Test(arguments: [ + "stun:example.test:3478", "stuns:example.test:5349", "turn:[::1]:3478?transport=udp", + "turn:example.test?transport=tcp", "turns:example.test:443?transport=tcp", + ]) + func validServers(_ url: String) throws { + #expect(ScreenSharingICEServer.validURL(url)) + _ = try ScreenSharingICEServer(urls: [url], username: "fixture", credential: "fixture") + } + @Test(arguments: [ + "https://example.test", "turn://example.test", "turn:user:pass@example.test", "turn:example.test/path", + "turn:example.test:0", "turn:example.test:65536", "turn:example.test?transport=anything", + "turns:example.test?transport=udp", "stun:example.test?transport=tcp", "turn:example.test\n", + ]) + func invalidServers(_ url: String) { + #expect(!ScreenSharingICEServer.validURL(url)) + #expect(throws: (any Error).self) { + try ScreenSharingICEServer(urls: [url], username: "fixture", credential: "fixture") + } + } + @Test func relayPolicyRequiresUsableBoundedCredentials() throws { + #expect(throws: (any Error).self) { try ScreenSharingICEServer(urls: ["turn:example.test"]) } + #expect(throws: (any Error).self) { try ScreenSharingICEConfiguration(relayOnly: true) } + let stun = try ScreenSharingICEServer(urls: ["stun:example.test"]) + #expect(throws: (any Error).self) { try ScreenSharingICEConfiguration(servers: [stun], relayOnly: true) } + #expect(throws: (any Error).self) { try ScreenSharingICEConfiguration(servers: Array(repeating: stun, count: 9)) } + #expect(try ScreenSharingICEConfiguration().servers.isEmpty) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingKeyframeRequestTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingKeyframeRequestTests.swift new file mode 100644 index 000000000..3a05397f0 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingKeyframeRequestTests.swift @@ -0,0 +1,93 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingKeyframeRequestTests { + @Test func ordinaryFramesDoNotForceKeyframes() { + var request = ScreenSharingKeyframeRequest() + #expect(!request.isPending) + let noUnrequestedAttempt = request.beginAttempt() == nil + #expect(noUnrequestedAttempt) + let noUnsolicitedRetry = !request.complete(nil, producedKeyFrame: true) + #expect(noUnsolicitedRetry) + } + + @Test func requestSurvivesUntilAdmissionAndUsableOutput() throws { + var request = ScreenSharingKeyframeRequest() + request.request() + // An admission drop does not begin an attempt or acknowledge the request. + #expect(request.isPending) + let attemptCandidate = request.beginAttempt() + let attempt = try #require(attemptCandidate) + #expect(request.isPending) + let noParallelAttempt = request.beginAttempt() == nil + #expect(noParallelAttempt) + let noRetryAfterSuccess = !request.complete(attempt, producedKeyFrame: true) + #expect(noRetryAfterSuccess) + #expect(!request.isPending) + let noAttemptAfterSuccess = request.beginAttempt() == nil + #expect(noAttemptAfterSuccess) + } + + @Test func droppedOrUnusableOutputRetriesWithoutAnotherTransportRequest() throws { + var request = ScreenSharingKeyframeRequest() + request.request() + let firstCandidate = request.beginAttempt() + let first = try #require(firstCandidate) + let retryRequired = request.complete(first, producedKeyFrame: false) + #expect(retryRequired) + #expect(request.isPending) + let retryCandidate = request.beginAttempt() + let retry = try #require(retryCandidate) + #expect(retry != first) + let retrySucceeded = !request.complete(retry, producedKeyFrame: true) + #expect(retrySucceeded) + #expect(!request.isPending) + } + + @Test func earlierKeyframeCannotConsumeARequestMadeDuringEncoding() throws { + var request = ScreenSharingKeyframeRequest() + request.request() + let firstCandidate = request.beginAttempt() + let first = try #require(firstCandidate) + request.request() + let newRequestWaitsForOutput = request.beginAttempt() == nil + #expect(newRequestWaitsForOutput) + request.complete(first, producedKeyFrame: true) + #expect(request.isPending) + let nextCandidate = request.beginAttempt() + let next = try #require(nextCandidate) + request.complete(next, producedKeyFrame: true) + #expect(!request.isPending) + } + + @Test func requestsBeforeAdmissionCoalesceIntoOneAttempt() throws { + var request = ScreenSharingKeyframeRequest() + request.request() + request.request() + let attemptCandidate = request.beginAttempt() + let attempt = try #require(attemptCandidate) + request.complete(attempt, producedKeyFrame: true) + #expect(!request.isPending) + let noRedundantAttempt = request.beginAttempt() == nil + #expect(noRedundantAttempt) + } + + @Test func lateCompletionCannotAcknowledgeOrReleaseARetry() throws { + var request = ScreenSharingKeyframeRequest() + request.request() + let firstCandidate = request.beginAttempt() + let first = try #require(firstCandidate) + request.complete(first, producedKeyFrame: false) + let retryCandidate = request.beginAttempt() + let retry = try #require(retryCandidate) + let lateSuccessIgnored = !request.complete(first, producedKeyFrame: true) + #expect(lateSuccessIgnored) + let lateFailureIgnored = !request.complete(first, producedKeyFrame: false) + #expect(lateFailureIgnored) + #expect(request.isPending) + let retryStillInFlight = request.beginAttempt() == nil + #expect(retryStillInFlight) + request.complete(retry, producedKeyFrame: true) + #expect(!request.isPending) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowCaptureTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowCaptureTests.swift new file mode 100644 index 000000000..7c6f848a3 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowCaptureTests.swift @@ -0,0 +1,150 @@ +import Testing + +@testable import CodevisorScreenSharing + +@Suite struct ScreenSharingOwnedWindowSelectionTests { + typealias Selection = ScreenSharingOwnedWindowSelection + + @Test func selectsOnlyTheExactWindowOwnedByThisProcess() { + let candidates = [ + Selection.Candidate(windowID: 10, owningProcessID: 500), Selection.Candidate(windowID: 11, owningProcessID: 500), + Selection.Candidate(windowID: 12, owningProcessID: 777), + ] + #expect(Selection.select(windowID: 11, processID: 500, from: candidates) == .success(candidates[1])) + #expect( + Selection.select(windowID: 12, processID: 500, from: candidates) + == .failure(.owningProcessMismatch(reported: 777))) + #expect(Selection.select(windowID: 99, processID: 500, from: candidates) == .failure(.windowNotListed)) + #expect(Selection.select(windowID: 10, processID: 500, from: []) == .failure(.windowNotListed)) + } + + @Test func refusesUnreportedOwnerAndDuplicateListings() { + let unreported = [Selection.Candidate(windowID: 10, owningProcessID: nil)] + #expect(Selection.select(windowID: 10, processID: 500, from: unreported) == .failure(.owningProcessNotReported)) + let duplicated = [ + Selection.Candidate(windowID: 10, owningProcessID: 500), Selection.Candidate(windowID: 10, owningProcessID: 500), + ] + #expect(Selection.select(windowID: 10, processID: 500, from: duplicated) == .failure(.ambiguous(count: 2))) + } +} + +@Suite struct ScreenSharingCaptureCallbackAccountingTests { + typealias Accounting = ScreenSharingCaptureCallbackAccounting + + @Test func onlyCompleteSamplesWithAnImageAreDelivered() { + let metrics = ScreenSharingMetrics() + #expect(Accounting.record(valid: true, rawStatus: 0, hasImage: true, metrics: metrics)) + #expect(!Accounting.record(valid: true, rawStatus: 0, hasImage: false, metrics: metrics)) + for status in [1, 2, 3, 4, 5] { + #expect(!Accounting.record(valid: true, rawStatus: status, hasImage: true, metrics: metrics)) + } + #expect(!Accounting.record(valid: false, rawStatus: 0, hasImage: true, metrics: metrics)) + #expect(!Accounting.record(valid: true, rawStatus: nil, hasImage: true, metrics: metrics)) + #expect(!Accounting.record(valid: true, rawStatus: 42, hasImage: true, metrics: metrics)) + let counters = metrics.snapshot().counters + #expect(counters["captureCallbacksComplete"] == 2) + #expect(counters["captureSamplesWithoutImage"] == 1) + #expect(counters["captureCallbacksIdle"] == 1 && counters["captureCallbacksBlank"] == 1) + #expect(counters["captureCallbacksSuspended"] == 1 && counters["captureCallbacksStarted"] == 1) + #expect(counters["captureCallbacksStopped"] == 1) + #expect(counters["captureSamplesInvalid"] == 1 && counters["captureSamplesWithoutStatus"] == 1) + #expect(counters["captureCallbacksOtherStatus"] == 1) + #expect(metrics.snapshot().labels["captureLatestStatus"] == "other(42)") + // No counter name mentions a pool drop: absence of callbacks is not measured here. + #expect(counters.keys.allSatisfy { !$0.lowercased().contains("drop") }) + } +} + +@Suite struct ScreenSharingOwnedWorkloadLifecycleTests { + typealias Lifecycle = ScreenSharingOwnedWorkloadLifecycle + + @Test func boundariesMustHappenInOrderAndCaptureStopsBeforeTheWindowCloses() throws { + var lifecycle = Lifecycle() + #expect(lifecycle.state == .created) + #expect(throws: Lifecycle.Refusal(transition: .startCapture, state: .created)) { + try lifecycle.apply(.startCapture, atNs: 1) + } + try lifecycle.apply(.show, atNs: 1) + #expect(throws: Lifecycle.Refusal(transition: .pauseWorkload, state: .shown)) { + try lifecycle.apply(.pauseWorkload, atNs: 2) + } + try lifecycle.apply(.ready, atNs: 2) + try lifecycle.apply(.startCapture, atNs: 3) + #expect(throws: Lifecycle.Refusal(transition: .close, state: .capturing)) { try lifecycle.apply(.close, atNs: 4) } + try lifecycle.apply(.pauseWorkload, atNs: 4) + #expect(throws: Lifecycle.Refusal(transition: .pauseWorkload, state: .workloadPaused)) { + try lifecycle.apply(.pauseWorkload, atNs: 5) + } + #expect(!lifecycle.completedInOrder) + try lifecycle.apply(.stopCapture, atNs: 5) + #expect(throws: Lifecycle.Refusal(transition: .stopCapture, state: .captureStopped)) { + try lifecycle.apply(.stopCapture, atNs: 6) + } + try lifecycle.apply(.close, atNs: 6) + #expect(lifecycle.state == .closed && lifecycle.completedInOrder) + #expect( + lifecycle.timestampsNs == [.show: 1, .ready: 2, .startCapture: 3, .pauseWorkload: 4, .stopCapture: 5, .close: 6]) + } + + @Test func captureMayStopWithoutAPauseAndTheWindowNeverClosesFirst() throws { + var lifecycle = Lifecycle() + try lifecycle.apply(.show, atNs: 1); try lifecycle.apply(.ready, atNs: 2); + try lifecycle.apply(.startCapture, atNs: 3) + try lifecycle.apply(.stopCapture, atNs: 4) + try lifecycle.apply(.close, atNs: 4) + #expect(lifecycle.completedInOrder) + var early = Lifecycle() + try early.apply(.show, atNs: 1); try early.apply(.ready, atNs: 2) + #expect(throws: Lifecycle.Refusal(transition: .close, state: .ready)) { try early.apply(.close, atNs: 3) } + } + + @Test func cleanupBeforeReadinessOrAfterAFailedStartAbandonsTheWindowWithoutClaimingAStop() throws { + // Shown, readiness never reached (no first draw / not visible). + var shown = Lifecycle() + try shown.apply(.show, atNs: 1) + #expect(shown.cleanUp(captureStopCompleted: false, atNs: 2) == .abandoned(from: .shown)) + #expect(shown.state == .abandoned && shown.abandonedFrom == .shown && !shown.completedInOrder) + #expect(shown.timestampsNs[.stopCapture] == nil && shown.timestampsNs[.close] == nil) + // Ready, then SCStream start / ownership selection failed: captureStarted + // was never applied. Even a stray "stop completed" claim cannot fabricate + // a capture stop from this state. + var failedStart = Lifecycle() + try failedStart.apply(.show, atNs: 1); try failedStart.apply(.ready, atNs: 2) + #expect(failedStart.cleanUp(captureStopCompleted: true, atNs: 3) == .abandoned(from: .ready)) + #expect(failedStart.timestampsNs[.stopCapture] == nil && !failedStart.completedInOrder) + // Nothing shown: nothing to clean. + var never = Lifecycle() + #expect(never.cleanUp(captureStopCompleted: false, atNs: 1) == .neverShown) + #expect(never.state == .created) + } + + @Test func cleanupDuringAnActiveRunCreditsTheStopOnlyWithCompletionEvidence() throws { + // Error mid-run, stream stop completed (capture recorded completion). + var stopped = Lifecycle() + try stopped.apply(.show, atNs: 1); try stopped.apply(.ready, atNs: 2); try stopped.apply(.startCapture, atNs: 3) + #expect(stopped.cleanUp(captureStopCompleted: true, atNs: 4) == .closedAfterCaptureStop) + #expect(stopped.state == .closed && stopped.completedInOrder) + #expect(stopped.timestampsNs[.stopCapture] == 4 && stopped.timestampsNs[.close] == 4) + // Error mid-run, stop did not complete: abandoned from capturing, no stop claimed. + var unstopped = Lifecycle() + try unstopped.apply(.show, atNs: 1); try unstopped.apply(.ready, atNs: 2); + try unstopped.apply(.startCapture, atNs: 3) + #expect(unstopped.cleanUp(captureStopCompleted: false, atNs: 4) == .abandoned(from: .capturing)) + #expect(unstopped.timestampsNs[.stopCapture] == nil && !unstopped.completedInOrder) + // Paused workload, stop completed: closes in order. + var paused = Lifecycle() + try paused.apply(.show, atNs: 1); try paused.apply(.ready, atNs: 2); try paused.apply(.startCapture, atNs: 3) + try paused.apply(.pauseWorkload, atNs: 4) + #expect(paused.cleanUp(captureStopCompleted: true, atNs: 5) == .closedAfterCaptureStop) + #expect(paused.completedInOrder) + // Already stopped by the normal path, then cleanup: closes; a second cleanup is a no-op. + var normal = Lifecycle() + try normal.apply(.show, atNs: 1); try normal.apply(.ready, atNs: 2); try normal.apply(.startCapture, atNs: 3) + try normal.apply(.stopCapture, atNs: 4) + #expect(normal.cleanUp(captureStopCompleted: true, atNs: 5) == .closedAfterCaptureStop) + #expect(normal.cleanUp(captureStopCompleted: true, atNs: 6) == .alreadyFinished(.closed)) + #expect(throws: Lifecycle.Refusal(transition: .close, state: .abandoned)) { try unstopped.apply(.close, atNs: 7) } + var fresh = Lifecycle() + #expect(throws: Lifecycle.Refusal(transition: .abandon, state: .created)) { try fresh.apply(.abandon, atNs: 1) } + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowObservationTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowObservationTests.swift new file mode 100644 index 000000000..c29bdc83f --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowObservationTests.swift @@ -0,0 +1,101 @@ +import Testing + +@testable import CodevisorScreenSharing + +@Suite struct ScreenSharingFirstObservationTests { + @Test func firstNonZeroTickIsRecordedOncePerMetricWithInitialAndFinalValues() { + var observation = ScreenSharingFirstObservation(names: [ + "anyCallback", "completeStatus", "capturedFrames", "decodedFrames", + ]) + #expect(observation.record(elapsedSeconds: 0, values: [:]) == []) + #expect(observation.record(elapsedSeconds: 1.02, values: ["anyCallback": 3]) == ["anyCallback"]) + // A complete status alone is not an accepted frame: capturedFrames stays unobserved. + #expect( + observation.record(elapsedSeconds: 2.05, values: ["anyCallback": 60, "completeStatus": 57]) == ["completeStatus"]) + #expect( + observation.record( + elapsedSeconds: 3.1, + values: ["anyCallback": 120, "completeStatus": 117, "capturedFrames": 110, "decodedFrames": 100]) == [ + "capturedFrames", "decodedFrames", + ]) + #expect( + observation.record( + elapsedSeconds: 4.1, + values: ["anyCallback": 180, "completeStatus": 177, "capturedFrames": 170, "decodedFrames": 160]) == []) + let any = observation.metrics["anyCallback"]! + #expect( + any.initialValue == 0 && any.finalValue == 180 && any.firstObservedTick == 1 && any.firstObservedAtSeconds == 1.02 + && any.valueWhenFirstObserved == 3) + #expect(observation.metrics["capturedFrames"]?.firstObservedTick == 3) + #expect(observation.ticks == 5) + let summary = observation.summary + #expect(summary["completeStatus"]?["firstObservedAtSeconds"] == "2.05") + #expect(summary["capturedFrames"]?["resolution"] == ScreenSharingFirstObservation.resolution) + } + + @Test func neverObservedStaysExplicitAndCountersAreNotAccumulated() { + var observation = ScreenSharingFirstObservation(names: ["anyCallback", "capturedFrames"]) + for tick in 0..<5 { + observation.record(elapsedSeconds: Double(tick), values: ["anyCallback": 0, "capturedFrames": 0]) + } + let summary = observation.summary + #expect(summary["anyCallback"]?["firstObservedAtSeconds"] == "never observed") + #expect(summary["anyCallback"]?["firstObservedTick"] == "never observed") + #expect(summary["capturedFrames"]?["initialValue"] == "0" && summary["capturedFrames"]?["finalValue"] == "0") + #expect(observation.metrics.count == 2) // bounded: one entry per metric, no per-tick history + var unrecorded = ScreenSharingFirstObservation(names: ["x"]) + #expect(unrecorded.summary["x"]?["initialValue"] == "not recorded") + unrecorded.record(elapsedSeconds: 0, values: [:]) + #expect(unrecorded.summary["x"]?["initialValue"] == "0") + } +} + +@Suite struct ScreenSharingOwnedWindowGeometryTests { + typealias Geometry = ScreenSharingOwnedWindowGeometry + + @Test func cocoaFrameConvertsToTopLeftUsingTheMainDisplayHeight() { + // The run's window: Cocoa (0, 540, 960, 540) on a 1080-point main display → top-left (0, 0, 960, 540). + let frame = Geometry.Rect(x: 0, y: 540, width: 960, height: 540) + #expect( + Geometry.cocoaToTopLeft(frame, mainDisplayHeight: 1080) == Geometry.Rect(x: 0, y: 0, width: 960, height: 540)) + // A different main display height gives a different answer: the conversion is only valid with the real height. + #expect( + Geometry.cocoaToTopLeft(frame, mainDisplayHeight: 900) == Geometry.Rect(x: 0, y: -180, width: 960, height: 540)) + } + + @Test func ownWindowRecordRequiresTheExactNumberAndPidAndKeepsEveryOtherStateExplicit() { + let mine = Geometry.WindowListEntry( + number: 12829, ownerPID: 55188, bounds: Geometry.Rect(x: 8, y: 4, width: 944, height: 532), layer: 0, + isOnscreen: true, alpha: 1) + let other = Geometry.WindowListEntry( + number: 12272, ownerPID: 1, bounds: nil, layer: nil, isOnscreen: nil, alpha: nil) + let unreported = Geometry.WindowListEntry( + number: 12829, ownerPID: nil, bounds: Geometry.Rect(x: 1, y: 1, width: 2, height: 2), layer: 0, isOnscreen: true, + alpha: 1) + #expect(Geometry.ownWindow(in: [other, mine], number: 12829, pid: 55188) == .found(mine)) + #expect(Geometry.ownWindow(in: [other], number: 12829, pid: 55188) == .absent) + #expect(Geometry.ownWindow(in: nil, number: 12829, pid: 55188) == .queryUnavailable) + // Exact ID with an unreported owner: not "absent", and its fields are not persisted. + #expect(Geometry.ownWindow(in: [other, unreported], number: 12829, pid: 55188) == .ownerUnreported) + #expect(Geometry.ownWindow(in: [other, mine], number: 12829, pid: 1) == .ownerMismatch(reportedPID: 55188)) + #expect(Geometry.ownWindow(in: [mine, mine], number: 12829, pid: 55188) == .duplicate(count: 2)) + } +} + +@Suite struct ScreenSharingCallbackTotalTests { + @Test func completeSampleWithoutImageCountsOnceNotTwice() { + typealias A = ScreenSharingCaptureCallbackAccounting + // A valid complete sample without an image increments BOTH complete and missingImage. + let metrics = ScreenSharingMetrics() + A.record(valid: true, rawStatus: 0, hasImage: false, metrics: metrics) + #expect(A.callbackTotal(counters: metrics.snapshot().counters) == 1) + A.record(valid: true, rawStatus: 0, hasImage: true, metrics: metrics) // complete with image + A.record(valid: true, rawStatus: 1, hasImage: true, metrics: metrics) // idle + A.record(valid: false, rawStatus: 0, hasImage: true, metrics: metrics) // invalid + A.record(valid: true, rawStatus: nil, hasImage: true, metrics: metrics) // missing status + A.record(valid: true, rawStatus: 42, hasImage: true, metrics: metrics) // other status + let counters = metrics.snapshot().counters + #expect(counters["captureSamplesWithoutImage"] == 1 && counters["captureCallbacksComplete"] == 2) + #expect(A.callbackTotal(counters: counters) == 6) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowSessionTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowSessionTests.swift new file mode 100644 index 000000000..c12d9cfd1 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnedWindowSessionTests.swift @@ -0,0 +1,268 @@ +import Testing +import CodevisorTestSupport + +@testable import CodevisorScreenSharing + +@Suite struct ScreenSharingFirstDrawGateTests { + @Test @MainActor func signalResumesAPendingWaitOnce() async throws { + let clock = TestClock() + let gate = ScreenSharingFirstDrawGate() + let waiting = TestSignal() + let waiter = Task { @MainActor in + waiting.signal() + try await gate.wait(timeout: .seconds(10)) { try await clock.sleep(for: $0) } + } + await waiting.wait() + await clock.waitForSleep(.seconds(10)) + gate.signal() + try await waiter.value + #expect(gate.isSignalled && gate.failure == nil && gate.resumeCount == 1) + #expect(clock.pendingCount == 0) // the deadline was cancelled on exit + // A later wait returns immediately; a repeated signal is harmless. + gate.signal() + try await gate.wait(timeout: .seconds(1)) { try await clock.sleep(for: $0) } + #expect(gate.resumeCount == 1) + } + + /// Ordering check (added while diagnosing the 11:18 UTC stalled suite run; NOT a reproduction of it): + /// the signal is issued before the waiter's task gets its first main-actor turn, so no continuation + /// exists yet. The main-actor-isolated gate must observe the signal on entry and return without ever + /// registering a continuation that nobody resumes. + @Test @MainActor func signalBeforeTheWaiterRegistersIsObservedOnEntry() async throws { + let clock = TestClock() + let gate = ScreenSharingFirstDrawGate() + let waiter = Task { @MainActor in + try await gate.wait(timeout: .seconds(10)) { try await clock.sleep(for: $0) } + } + gate.signal() // same main-actor turn as the task creation: the waiter has not run yet + try await waiter.value + #expect(gate.isSignalled && gate.failure == nil) + #expect(gate.resumeCount == 0) // nothing was pending: the wait returned on entry, no continuation was ever stored + #expect(clock.pendingCount == 0 && clock.requestCount(.seconds(10)) == 0) // no deadline was even started + } + + @Test @MainActor func timeoutFailsThePendingWaitOnceAndBlocksLaterWaits() async { + let clock = TestClock() + let gate = ScreenSharingFirstDrawGate() + let waiter = Task { @MainActor in + try await gate.wait(timeout: .seconds(10)) { try await clock.sleep(for: $0) } + } + await clock.waitForSleep(.seconds(10)) + clock.advance(by: .seconds(10)) + await #expect(throws: ScreenSharingFirstDrawGate.Failure.timedOut) { try await waiter.value } + #expect(gate.failure == .timedOut && gate.resumeCount == 1) + gate.signal() // a late first draw cannot revive readiness + #expect(!gate.isSignalled) + await #expect(throws: ScreenSharingFirstDrawGate.Failure.timedOut) { + try await gate.wait(timeout: .seconds(1)) { try await clock.sleep(for: $0) } + } + } + + @Test @MainActor func cancellationResumesThePendingWaitOnceAndCancelsTheDeadline() async { + let clock = TestClock() + let gate = ScreenSharingFirstDrawGate() + let waiter = Task { @MainActor in + try await gate.wait(timeout: .seconds(10)) { try await clock.sleep(for: $0) } + } + await clock.waitForSleep(.seconds(10)) + waiter.cancel() + await #expect(throws: ScreenSharingFirstDrawGate.Failure.cancelled) { try await waiter.value } + #expect(gate.failure == .cancelled && gate.resumeCount == 1) + #expect(clock.pendingCount == 0) + await #expect(throws: ScreenSharingFirstDrawGate.Failure.cancelled) { + try await gate.wait(timeout: .seconds(1)) { try await clock.sleep(for: $0) } + } + } +} + +@Suite struct ScreenSharingFirstDrawGateTeardownTests { + @Test @MainActor func teardownResolvesAHeldWaitOnceWithoutTaskCancellationAndBlocksLaterWaits() async { + let clock = TestClock() + let gate = ScreenSharingFirstDrawGate() + let waiter = Task { @MainActor in + try await gate.wait(timeout: .seconds(10)) { try await clock.sleep(for: $0) } + } + await clock.waitForSleep(.seconds(10)) + gate.teardown() + await #expect(throws: ScreenSharingFirstDrawGate.Failure.tornDown) { try await waiter.value } + #expect(gate.failure == .tornDown && gate.resumeCount == 1 && clock.pendingCount == 0) + gate.signal() + #expect(!gate.isSignalled) + gate.teardown() // idempotent + await #expect(throws: ScreenSharingFirstDrawGate.Failure.tornDown) { + try await gate.wait(timeout: .seconds(1)) { try await clock.sleep(for: $0) } + } + #expect(gate.resumeCount == 1) + } +} + +@Suite struct ScreenSharingOwnedWindowSessionTests { + typealias Session = ScreenSharingOwnedWindowSession + struct Boom: Error {} + + @MainActor + final class Fake { + var shown = 0 + var hidden = 0 + var captureStarts = 0 + var stops = 0 + var stopThrows = false + var tick: Int64 = 0 + lazy var session = Session( + show: { self.shown += 1 }, hide: { self.hidden += 1 }, + stopCapture: { + self.stops += 1 + if self.stopThrows { throw Boom() } + }, + now: { + self.tick += 1 + return self.tick + }) + } + + @Test @MainActor func normalPathClosesInOrderAfterASuccessfulStop() async throws { + let fake = Fake() + try await fake.session.start(ready: {}, startCapture: { fake.captureStarts += 1 }) + #expect(fake.session.lifecycle.state == .capturing && fake.shown == 1 && fake.captureStarts == 1) + try fake.session.pauseWorkload {} + #expect(await fake.session.stopCapture()) + #expect(fake.stops == 1 && fake.session.finish() == .closedAfterCaptureStop) + #expect(fake.session.completedInOrder && fake.hidden == 1 && fake.session.stopSucceeded == true) + #expect(fake.session.finish() == .closedAfterCaptureStop && fake.hidden == 1) // idempotent + } + + @Test @MainActor func cancellationWhileReadinessIsHeldAbandonsTheShownWindowAndNeverStartsCapture() async { + let fake = Fake() + let held = TestSignal() + let entered = TestSignal() + let task = Task { @MainActor in + try await fake.session.start( + ready: { + entered.signal() + // Readiness returns NORMALLY after the cancellation: only the + // session's own check may prevent the capture start. + await held.wait() + }, + startCapture: { fake.captureStarts += 1 }) + } + await entered.wait() + task.cancel() + held.signal() + await #expect(throws: CancellationError.self) { try await task.value } + #expect(fake.captureStarts == 0 && fake.stops == 0) + #expect(fake.session.lifecycle.state == .abandoned && fake.session.lifecycle.abandonedFrom == .shown) + #expect(fake.hidden == 1 && !fake.session.completedInOrder) + #expect( + fake.session.lifecycle.timestampsNs[.ready] == nil && fake.session.lifecycle.timestampsNs[.stopCapture] == nil) + #expect(fake.session.startFailure != nil) + } + + @Test @MainActor func cancellationAfterAHeldCaptureStartSucceededStopsTheStreamBeforeHiding() async { + let fake = Fake() + let held = TestSignal() + let entered = TestSignal() + let task = Task { @MainActor in + try await fake.session.start( + ready: {}, + startCapture: { + entered.signal() + await held.wait() // the stream start completes after the cancellation + fake.captureStarts += 1 + }) + } + await entered.wait() + task.cancel() + held.signal() + await #expect(throws: CancellationError.self) { try await task.value } + // The stream had started, so it was really stopped before the window hid. + #expect(fake.captureStarts == 1 && fake.stops == 1 && fake.hidden == 1) + #expect(fake.session.stopSucceeded == true && fake.session.lifecycle.state == .closed) + let stamps = fake.session.lifecycle.timestampsNs + #expect(stamps[.startCapture]! < stamps[.stopCapture]! && stamps[.stopCapture]! <= stamps[.close]!) + #expect(fake.session.startFailure?.contains("Cancellation") == true) + } + + @Test @MainActor func cancellationAfterAHeldCaptureStartWithAFailingStopIsAbandonedNotClosed() async { + let fake = Fake() + fake.stopThrows = true + let held = TestSignal() + let entered = TestSignal() + let task = Task { @MainActor in + try await fake.session.start( + ready: {}, + startCapture: { + entered.signal(); await held.wait() + }) + } + await entered.wait() + task.cancel() + held.signal() + await #expect(throws: CancellationError.self) { try await task.value } + #expect(fake.stops == 1 && fake.hidden == 1) + #expect(fake.session.stopSucceeded == false && fake.session.stopFailure != nil) + #expect(fake.session.lifecycle.abandonedFrom == .capturing && !fake.session.completedInOrder) + #expect((fake.session.record["cleanup"] as? String)?.hasPrefix("abandoned") == true) + #expect(fake.session.record["stopSucceeded"] as? String == "false") + } + + @Test @MainActor func failedCaptureStartAbandonsFromReadyWithoutAStopClaim() async { + let fake = Fake() + await #expect(throws: Boom.self) { + try await fake.session.start( + ready: {}, + startCapture: { + fake.captureStarts += 1; throw Boom() + }) + } + #expect(fake.captureStarts == 1 && fake.hidden == 1 && fake.stops == 0) + #expect(fake.session.lifecycle.abandonedFrom == .ready && fake.session.stopSucceeded == nil) + #expect(!fake.session.completedInOrder) + // finish() is idempotent: it keeps the first outcome and never hides twice. + #expect(fake.session.finish() == .abandoned(from: .ready) && fake.hidden == 1) + #expect(await fake.session.stopCapture() == false && fake.stops == 0) + } + + @Test @MainActor func failedStreamStopIsPreservedAndNeverBecomesAnInOrderClose() async throws { + let fake = Fake() + fake.stopThrows = true + try await fake.session.start(ready: {}, startCapture: { fake.captureStarts += 1 }) + #expect(await fake.session.stopCapture() == false && fake.stops == 1) + #expect(fake.session.stopSucceeded == false && fake.session.stopFailure != nil) + #expect(fake.session.lifecycle.state == .capturing) // no stopCapture transition was applied + #expect(fake.session.finish() == .abandoned(from: .capturing)) + #expect(!fake.session.completedInOrder && fake.hidden == 1) + #expect(fake.session.record["stopSucceeded"] as? String == "false") + } + + @Test @MainActor func repeatedCleanupAfterAFailedStopNeverReplacesTheFailure() async throws { + // The runtime path: normal-end stop fails → run throws → stop() asks the + // session to stop again. The second closure would succeed trivially (the + // capture already cleared its stream); it must not run or be believed. + let fake = Fake() + fake.stopThrows = true + try await fake.session.start(ready: {}, startCapture: { fake.captureStarts += 1 }) + #expect(await fake.session.stopCapture() == false && fake.stops == 1) + fake.stopThrows = false + #expect(await fake.session.stopCapture() == false) + #expect(fake.stops == 1 && fake.session.stopRetriesRefused == 1) + #expect(fake.session.stopSucceeded == false && fake.session.stopFailure != nil) + #expect(fake.session.lifecycle.state == .capturing) + #expect(fake.session.finish() == .abandoned(from: .capturing)) + #expect(!fake.session.completedInOrder && fake.hidden == 1) + // Cleanup again (top-level catch after a failed run): unchanged evidence. + #expect(await fake.session.stopCapture() == false && fake.stops == 1) + #expect(fake.session.finish() == .abandoned(from: .capturing) && fake.hidden == 1) + let record = fake.session.record + #expect(record["stopSucceeded"] as? String == "false" && record["stopRetriesRefused"] as? Int == 1) + #expect( + (record["cleanup"] as? String)?.hasPrefix("abandoned") == true && record["completedInOrder"] as? Bool == false) + } + + @Test @MainActor func failureBeforeTheWindowIsShownNeedsNoCleanup() async { + let broken = Session(show: { throw Boom() }, hide: {}, stopCapture: {}) + await #expect(throws: Boom.self) { try await broken.start(ready: {}, startCapture: {}) } + #expect(broken.lifecycle.state == .abandoned && broken.lifecycle.abandonedFrom == .shown && broken.hideCount == 1) + let never = Session(show: {}, hide: {}, stopCapture: {}) + #expect(never.finish() == .neverShown && never.hideCount == 0) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnershipTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnershipTests.swift new file mode 100644 index 000000000..bf005b80d --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingOwnershipTests.swift @@ -0,0 +1,231 @@ +import CodevisorTestSupport +import CoreVideo +import Foundation +import Testing +@testable import CodevisorScreenSharing +@preconcurrency import WebRTC + +/// Buffer lifetime at the boundaries this package owns. Weak references to +/// the CoreVideo buffers observe actual release; a nil refresh result alone +/// would not distinguish the cache's owner from WebRTC or VideoToolbox owners. +struct ScreenSharingOwnershipTests { + private func makeBuffer(width: Int = 64) throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, width, 64, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + return try #require(pixel) + } + + @Test func cacheReplacementAndClearReleaseThePreviousBufferOnly() throws { + var store = ScreenSharingRefreshFrameStore() + weak var weakA: CVPixelBuffer? + weak var weakB: CVPixelBuffer? + try autoreleasepool { + let a = try makeBuffer() + weakA = a + #expect(store.capture(.init(pixelBuffer: a, timestampNs: 10)) != nil) + } + #expect(weakA != nil && store.isHolding) + try autoreleasepool { + let b = try makeBuffer() + weakB = b + // Replacing the cached frame releases A while B stays owned by the store. + #expect(store.capture(.init(pixelBuffer: b, timestampNs: 20)) != nil) + } + #expect(weakA == nil && weakB != nil) + // A refresh hands out a new reference to the cached buffer without transferring ownership. + try autoreleasepool { + let refreshed = store.refresh(nowNs: 30) + let handed = try #require(refreshed) + #expect(handed.pixelBuffer === weakB) + } + #expect(weakB != nil) + store.clear() + #expect(weakB == nil && !store.isHolding) + // Ordering state survives the release: older captures stay rejected, newer ones submit later. + #expect(store.capture(.init(pixelBuffer: try makeBuffer(), timestampNs: 20)) == nil) + #expect(store.capture(.init(pixelBuffer: try makeBuffer(), timestampNs: 21))?.timestampNs == 31) + } + + @Test func mailboxTakeTransfersOwnershipAndClearReleases() throws { + let mailbox = ScreenSharingFrameMailbox() + weak var weakA: CVPixelBuffer? + weak var weakB: CVPixelBuffer? + try autoreleasepool { + let a = try makeBuffer() + weakA = a + mailbox.put(.init(pixelBuffer: a, timestampNs: 1)) + } + #expect(weakA != nil) + try autoreleasepool { + let b = try makeBuffer() + weakB = b + mailbox.put(.init(pixelBuffer: b, timestampNs: 2)) + } + // A slow consumer keeps only the newest frame; the replaced one is released. + #expect(weakA == nil && weakB != nil) + try autoreleasepool { + let taken = try #require(mailbox.take()) + #expect(taken.pixelBuffer === weakB) + #expect(mailbox.take() == nil) + } + #expect(weakB == nil) + try autoreleasepool { + let c = try makeBuffer() + weakA = c + mailbox.put(.init(pixelBuffer: c, timestampNs: 3)) + } + mailbox.clear() + #expect(weakA == nil) + } + + @Test @MainActor func senderReleasesCacheOnConfigurationAndRejectsFramesAfterStop() throws { + // Same boundary as production: trials are pinned before this bare factory exists. + ScreenSharingFieldTrials.process.ensureInstalled() + let factory = RTCPeerConnectionFactory() + let source = factory.videoSource(forScreenCast: true) + let metrics = ScreenSharingMetrics() + let sender = ScreenSharingFrameSender( + source: source, metrics: metrics, idleMonitor: ScreenSharingSourceIdleMonitor()) + sender.configure(try ScreenSharingVideoConfiguration(width: 128, height: 64)) + weak var weakOld: CVPixelBuffer? + weak var weakNew: CVPixelBuffer? + weak var weakLate: CVPixelBuffer? + try autoreleasepool { + let old = try makeBuffer(width: 128) + weakOld = old + sender.push(.init(pixelBuffer: old, timestampNs: 10)) + } + // No sink is attached to the unnegotiated source, so only the cache owns the buffer. + #expect(weakOld != nil && sender.isHoldingCachedFrame) + #expect(metrics.snapshot().counters["capturedFrames"] == 1) + #expect( + metrics.snapshot().counters["refreshCacheFills"] == 1 + && metrics.snapshot().counters["refreshCacheReplacements"] == nil) + // A configuration transition releases cached content before accepting the new size. + sender.configure(try ScreenSharingVideoConfiguration(width: 64, height: 64)) + #expect(weakOld == nil && !sender.isHoldingCachedFrame) + #expect(metrics.snapshot().counters["refreshCacheReleases"] == 1) + try autoreleasepool { + let stale = try makeBuffer(width: 128) + weakLate = stale + sender.push(.init(pixelBuffer: stale, timestampNs: 20)) + } + // An old-size frame arriving after the transition is neither submitted nor retained. + #expect(weakLate == nil && metrics.snapshot().counters["captureTransitionDrops"] == 1) + try autoreleasepool { + let fresh = try makeBuffer(width: 64) + weakNew = fresh + sender.push(.init(pixelBuffer: fresh, timestampNs: 30)) + } + #expect(weakNew != nil && metrics.snapshot().counters["capturedFrames"] == 2) + #expect(metrics.snapshot().counters["refreshCacheFills"] == 2) + sender.stop() + #expect(weakNew == nil && !sender.isHoldingCachedFrame) + #expect(metrics.snapshot().counters["refreshCacheReleases"] == 2) + try autoreleasepool { + let late = try makeBuffer(width: 64) + weakLate = late + sender.push(.init(pixelBuffer: late, timestampNs: 40)) + sender.refreshLatest() + } + #expect(weakLate == nil && metrics.snapshot().counters["capturedFrames"] == 2) + #expect(metrics.snapshot().counters["refreshFrames"] == nil) + } + + @Test @MainActor func peerCloseReleasesCachedMediaAndRejectsLaterCaptureCallbacks() async throws { + let metrics = ScreenSharingMetrics() + let peer = try ScreenSharingPeer( + sending: true, configuration: try ScreenSharingVideoConfiguration(width: 64, height: 64), metrics: metrics) + weak var weakCached: CVPixelBuffer? + try autoreleasepool { + let cached = try makeBuffer(width: 64) + weakCached = cached + peer.frameSender.push(.init(pixelBuffer: cached, timestampNs: 10)) + } + #expect(weakCached != nil && peer.frameSender.isHoldingCachedFrame) + peer.close() + peer.close() + // Owned work is cancelled and the cache released; the buffer has no remaining owner. + #expect(weakCached == nil && !peer.frameSender.isHoldingCachedFrame) + weak var weakLate: CVPixelBuffer? + try autoreleasepool { + let late = try makeBuffer(width: 64) + weakLate = late + peer.frameSender.push(.init(pixelBuffer: late, timestampNs: 20)) + } + #expect(weakLate == nil && metrics.snapshot().counters["capturedFrames"] == 1) + #expect(!peer.mailbox.isHolding) + #expect(metrics.snapshot().counters["refreshCacheFills"] == 1) + #expect(metrics.snapshot().counters["refreshCacheReleases"] == 1) + } + + @Test @MainActor func closeBoundaryIsSharedByConcurrentWaitersAndAbsentBeforeClose() async throws { + let metrics = ScreenSharingMetrics() + let peer = try ScreenSharingPeer( + sending: true, configuration: try ScreenSharingVideoConfiguration(width: 64, height: 64), metrics: metrics) + #expect(await peer.awaitClosed() == nil) + // This peer-level check only establishes the boundary's shape on an + // unnegotiated peer: nil before close, one identical answer for concurrent + // waiters, idempotent afterwards. Activation crosses a main-actor hop, so + // the count here may be zero; the pending-handle join itself is proved + // through the peer's owned-work seam in the next test. + peer.frameSender.push(.init(pixelBuffer: try makeBuffer(), timestampNs: 10)) + #expect(metrics.snapshot().counters["capturedFrames"] == 1) + peer.close() + async let first = peer.awaitClosed() + async let second = peer.awaitClosed() + let counts = await [first, second] + #expect(counts[0] != nil && counts[0] == counts[1]) + #expect(await peer.awaitClosed() == counts[0]) + } + + @Test @MainActor func ownedWorkJoinIsSharedByConcurrentWaitersWhileAHandleIsStillPending() async { + // The seam ScreenSharingPeer.awaitClosed() delegates to. An early-clear + // implementation would answer one waiter with the handle count and the + // other with zero under every interleaving; both must answer the same. + let work = ScreenSharingOwnedWork() + #expect(await work.join() == nil && work.count == nil) + let gate = TestSignal() + let released = TestSignal() + let pending = Task { @MainActor in + await gate.wait() + released.signal() + } + work.close(with: [pending]) + work.close(with: []) + #expect(work.count == 1) + let entered = TestSignal() + let finished = TestSignal() + async let first: Int? = { + entered.signal() + let count = await work.join() + finished.signal() + return count + }() + async let second: Int? = { + entered.signal() + let count = await work.join() + finished.signal() + return count + }() + // Both waiters have called join() while the gate is still closed, so the + // join is suspended on the held handle: nothing has finished or released. + await entered.wait(for: 2) + #expect(finished.value == 0) + #expect(released.value == 0) + gate.signal() + let counts = await [first, second] + await finished.wait(for: 2) + #expect(released.value == 1) + #expect(counts == [1, 1]) + #expect(await work.join() == 1) + } + + @Test func mailboxOwnershipObservationDoesNotConsumeTheFrame() throws { + let mailbox = ScreenSharingFrameMailbox() + #expect(!mailbox.isHolding) + mailbox.put(.init(pixelBuffer: try makeBuffer(), timestampNs: 1)) + #expect(mailbox.isHolding && mailbox.isHolding) + #expect(mailbox.take() != nil && !mailbox.isHolding) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingPresentationMetricsTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingPresentationMetricsTests.swift new file mode 100644 index 000000000..08b4c6b83 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingPresentationMetricsTests.swift @@ -0,0 +1,17 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingPresentationMetricsTests { + @Test func skippedDrawablesAndRedrawsDoNotCountAsNewPresentedFrames() { + let metrics = ScreenSharingMetrics() + #expect(!metrics.recordPresentation(isNewFrame: true, presentedAt: 0, submittedAt: 1, receivedAt: 0.5)) + #expect(!metrics.recordPresentation(isNewFrame: false, presentedAt: 2, submittedAt: 1, receivedAt: 0.5)) + #expect(metrics.recordPresentation(isNewFrame: true, presentedAt: 2, submittedAt: 1.75, receivedAt: 1.5)) + let result = metrics.snapshot() + #expect(result.counters["presentationCallbacks"] == 2) + #expect(result.counters["unpresentedDrawables"] == 1) + #expect(result.counters["presentedFrames"] == 1) + #expect(result.timings["submissionToPresentation"]?.p50Ms == 250) + #expect(result.timings["receiverCallbackToPresentation"]?.p50Ms == 500) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRTCIntervalMetricsTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRTCIntervalMetricsTests.swift new file mode 100644 index 000000000..b605c89b2 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRTCIntervalMetricsTests.swift @@ -0,0 +1,36 @@ +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingRTCIntervalMetricsTests { + private func sample(_ id: String = "a", seconds: String, count: String) -> [String: String] { + ["inbound-rtp.\(id).jitterBufferDelay": seconds, "inbound-rtp.\(id).jitterBufferEmittedCount": count] + } + + @Test func computesIntervalInsteadOfLifetimeMeanAndWeightsStreams() throws { + var metrics = ScreenSharingRTCIntervalMetrics() + let first = sample(seconds: "10", count: "100").merging(sample("b", seconds: "5", count: "100")) { a, _ in a } + #expect(metrics.update(first).isEmpty) + let next = sample(seconds: "10.5", count: "150").merging(sample("b", seconds: "9.5", count: "250")) { a, _ in a } + #expect(try #require(metrics.update(next)["jitterBufferMeanMs"]) == 25) + #expect(metrics.update(next).isEmpty) + } + + @Test func rebaselinesResetMissingAndReplacedStreams() { + var metrics = ScreenSharingRTCIntervalMetrics() + _ = metrics.update(sample(seconds: "5", count: "100")) + #expect(metrics.update(sample(seconds: "1", count: "10")).isEmpty) + #expect(metrics.update(sample(seconds: "2", count: "20"))["jitterBufferMeanMs"] == 100) + #expect(metrics.update([:]).isEmpty) + #expect(metrics.update(sample(seconds: "3", count: "30")).isEmpty) + #expect(metrics.update(sample("new", seconds: "4", count: "40")).isEmpty) + } + + @Test(arguments: ["NaN", "inf", "-1", "invalid"]) + func rejectsInvalidTotalsAndRebaselines(_ invalid: String) { + var metrics = ScreenSharingRTCIntervalMetrics() + _ = metrics.update(sample(seconds: "1", count: "10")) + #expect(metrics.update(sample(seconds: invalid, count: "20")).isEmpty) + #expect(metrics.update(sample(seconds: "3", count: "30")).isEmpty) + #expect(metrics.update(sample(seconds: "4", count: "40"))["jitterBufferMeanMs"] == 100) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRenderCoordinatorTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRenderCoordinatorTests.swift new file mode 100644 index 000000000..3ba2f48f4 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRenderCoordinatorTests.swift @@ -0,0 +1,864 @@ +import CoreVideo +import Foundation +import Testing + +@testable import CodevisorScreenSharing + +/// Lifecycle and ownership of the renderer's scheduling state through the +/// real coordinator used by `ScreenSharingMetalView`, with a controlled +/// submission (completion/presentation delivered by the test) and a controlled +/// hop (queued main-actor work drained by the test). No Metal, no window. +@MainActor +struct ScreenSharingRenderCoordinatorTests { + /// Records the installed handlers; the test decides when the GPU "completes" + /// and when the drawable is "presented". + private final class ControlledSubmission: ScreenSharingRenderSubmission, @unchecked Sendable { + private let lock = NSLock() + private var completed: (@Sendable (Bool) -> Void)? + private var presented: (@Sendable (Double) -> Void)? + private var installedCompletion: (@Sendable (Bool) -> Void)? + private(set) var committed = false + func onCompleted(_ handler: @escaping @Sendable (Bool) -> Void) { + lock.withLock { + completed = handler + installedCompletion = handler + } + } + func onPresented(_ handler: @escaping @Sendable (Double) -> Void) { lock.withLock { presented = handler } } + func commit() { lock.withLock { committed = true } } + var holdsHandlers: Bool { lock.withLock { completed != nil || presented != nil } } + var holdsPresentationHandler: Bool { lock.withLock { presented != nil } } + func complete(_ success: Bool = true) { + let handler = lock.withLock { + let h = completed; completed = nil; return h + } + handler?(success) + } + func replayCompletion(_ success: Bool = true) -> Bool { + guard let handler = lock.withLock({ installedCompletion }) else { return false } + handler(success) + return true + } + func forgetInstalledCompletion() { lock.withLock { installedCompletion = nil } } + func present(at time: Double) { + let handler = lock.withLock { + let h = presented; presented = nil; return h + } + handler?(time) + } + } + + /// Queued main-actor work (arrival, completion, presentation hops), drained explicitly. + private final class WorkQueue: @unchecked Sendable { + private let lock = NSLock() + private var items: [@Sendable @MainActor () -> Void] = [] + var hop: ScreenSharingRenderCoordinator.Hop { { [self] work in self.lock.withLock { self.items.append(work) } } } + var pending: Int { lock.withLock { items.count } } + @MainActor func drain() -> Int { + let work = lock.withLock { + let w = items; items = []; return w + } + for item in work { item() } + return work.count + } + } + + /// Stand-in for the view's retained textures + frame (released at GPU completion). + private final class Retained: @unchecked Sendable { + let frame: ScreenSharingVideoFrame + init(_ frame: ScreenSharingVideoFrame) { self.frame = frame } + } + + @MainActor private struct Fixture { + let mailbox = ScreenSharingFrameMailbox() + let metrics = ScreenSharingMetrics() + let queue = WorkQueue() + let coordinator: ScreenSharingRenderCoordinator + let draws = Counter() + @MainActor final class Counter { var value = 0 } + init(renderOnArrival: Bool) { + coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: metrics, renderOnArrival: renderOnArrival, hop: queue.hop) + let draws = draws + coordinator.bind { draws.value += 1 } + } + func counter(_ name: String) -> Int { metrics.snapshot().counters[name] ?? 0 } + } + + private func makeBuffer() throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 64, 64, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + return try #require(pixel) + } + + private func frame(_ buffer: CVPixelBuffer, rtp: UInt32 = 7, receivedAt: Double? = 1.0) -> ScreenSharingVideoFrame { + .init(pixelBuffer: buffer, timestampNs: Int64(rtp), rtpTimestamp: rtp, receivedAtSeconds: receivedAt) + } + + /// Selection order is unchanged: nothing in flight → incoming first; the + /// incoming frame is cached; a redraw re-selects the cache as not-new; while + /// in flight nothing is taken and the mailbox keeps its frame. + @Test func selectionOrderAndCachingAreUnchanged() throws { + let f = Fixture(renderOnArrival: false) + #expect(f.coordinator.select() == nil) + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + let first = try #require(f.coordinator.select()) + #expect(first.isNewFrame && first.frame.rtpTimestamp == 1 && f.coordinator.isHoldingCachedFrame) + #expect(f.coordinator.select() == nil) // no redraw requested, nothing incoming + f.coordinator.setNeedsRedraw() + #expect(f.queue.pending == 0) // without redrawsOnDemand a redraw request schedules nothing (unchanged) + let redraw = try #require(f.coordinator.select()) + #expect(!redraw.isNewFrame && redraw.frame.rtpTimestamp == 1) + let submission = ControlledSubmission() + f.coordinator.commit( + submission, retaining: Retained(redraw.frame), frame: redraw.frame, isNewFrame: false, submittedAt: 2) + #expect(submission.committed && f.coordinator.inFlight) + f.mailbox.put(frame(try makeBuffer(), rtp: 2)) + #expect(f.coordinator.select() == nil && f.mailbox.isHolding) // in flight: the frame stays in the mailbox + submission.complete() + #expect(f.queue.drain() == 1 && !f.coordinator.inFlight) + #expect(f.draws.value == 0) // display-link drive: completion never requests a draw + let next = try #require(f.coordinator.select()) + #expect(next.isNewFrame && next.frame.rtpTimestamp == 2) + } + + @Test func stopReleasesTheCachedFrameAndMailboxWhileTheCoordinatorStaysAlive() throws { + let f = Fixture(renderOnArrival: true) + weak var weakCached: CVPixelBuffer? + weak var weakWaiting: CVPixelBuffer? + var sizes: [CGSize] = [] + f.coordinator.onFrameSize = { sizes.append($0) } + try autoreleasepool { + let cached = try makeBuffer() + weakCached = cached + f.mailbox.put(frame(cached, rtp: 1)) + _ = f.queue.drain() // arrival hop → one draw request + let selected = try #require(f.coordinator.select()) + f.coordinator.reportSize(CGSize(width: 64, height: 64)) + let submission = ControlledSubmission() + f.coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + submission.complete() + _ = f.queue.drain() + let waiting = try makeBuffer() + weakWaiting = waiting + f.mailbox.put(frame(waiting, rtp: 2)) // arrives after completion; queued, not yet drawn + } + #expect(f.draws.value == 2 && sizes == [CGSize(width: 64, height: 64)]) + #expect(weakCached != nil && weakWaiting != nil && f.coordinator.isHoldingCachedFrame && f.mailbox.isHolding) + f.coordinator.stop() + #expect(f.coordinator.stopped && !f.coordinator.isHoldingCachedFrame && !f.mailbox.isHolding) + #expect(weakCached == nil && weakWaiting == nil) + #expect(f.coordinator.onFrameSize == nil && f.coordinator.onPresented == nil) + #expect(f.metrics.snapshot().labels["rendererStopped"] == "true") + _ = f.queue.drain() // the queued arrival for frame 2 is inert + #expect(f.draws.value == 2) + } + + @Test func inFlightOwnershipIsHeldThroughStopAndReleasedOnlyAtActualCompletion() throws { + let f = Fixture(renderOnArrival: true) + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer)) + _ = f.queue.drain() + let selected = try #require(f.coordinator.select()) + f.coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + } + f.coordinator.stop() + #expect(f.coordinator.inFlight && weakBuffer != nil) // GPU still owns the buffer: stop must not release it + #expect(f.counter("renderedFrames") == 0) + submission.complete() + submission.forgetInstalledCompletion() + #expect(weakBuffer == nil && f.counter("renderedFrames") == 1) // released at completion, telemetry still counted + #expect(f.queue.drain() == 1 && !f.coordinator.inFlight) + #expect(f.draws.value == 1) // the completion hop does not restart the arrival drive after stop + } + + @Test func presentationHandlerHoldsOnlyScalarsAndIsSilentAfterStop() throws { + let f = Fixture(renderOnArrival: false) + var notified: [UInt32] = [] + f.coordinator.onPresented = { notified.append($0) } + weak var weakBuffer: CVPixelBuffer? + let first = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer, rtp: 41, receivedAt: 0.5)) + let selected = try #require(f.coordinator.select()) + f.coordinator.commit( + first, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + first.complete() + first.forgetInstalledCompletion() + _ = f.queue.drain() + } + #expect(f.coordinator.isHoldingCachedFrame && weakBuffer != nil) + // releases the cache; the un-invoked presentation handler is still held by the submission + f.coordinator.stop() + #expect(first.holdsPresentationHandler && weakBuffer == nil) + first.present(at: 1.5) + // real presentation is still recorded + #expect(f.counter("presentationCallbacks") == 1 && f.counter("presentedFrames") == 1) + #expect(f.queue.drain() == 1 && notified.isEmpty) // but the product is not notified after stop + + let g = Fixture(renderOnArrival: false) + var live: [UInt32] = [] + g.coordinator.onPresented = { live.append($0) } + weak var weakLive: CVPixelBuffer? + let second = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakLive = buffer + g.mailbox.put(frame(buffer, rtp: 42)) + let selected = try #require(g.coordinator.select()) + g.coordinator.commit( + second, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + second.complete() + second.forgetInstalledCompletion() + _ = g.queue.drain() + g.mailbox.put(frame(try makeBuffer(), rtp: 43)) + _ = try #require(g.coordinator.select()) // the cache moves on to frame 43 + } + #expect(second.holdsPresentationHandler && weakLive == nil) // the pending handler alone keeps no buffer alive + second.present(at: 2) + _ = g.queue.drain() + #expect(live == [42]) + second.present(at: 0) // a second delivery is impossible for a real drawable; the handler was consumed + #expect(g.counter("unpresentedDrawables") == 0) + } + + @Test func resizeDisplayLinkAndArrivalWorkAfterStopAreInertAndStopRepeats() throws { + let f = Fixture(renderOnArrival: true) + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + _ = f.queue.drain() + _ = try #require(f.coordinator.select()) + #expect(f.draws.value == 1) + f.coordinator.stop() + f.coordinator.setNeedsRedraw() // resize / fit change + #expect(f.coordinator.select() == nil) // display-link or arrival draw: nothing to render + f.mailbox.put(frame(try makeBuffer(), rtp: 2)) // late decoder delivery: no subscription left + #expect(f.queue.pending == 0 && f.mailbox.isHolding) + f.coordinator.arrivalScheduled() // an arrival hop queued before stop + #expect(f.coordinator.select() == nil && f.mailbox.isHolding && f.draws.value == 1) + f.coordinator.reportSize(CGSize(width: 1, height: 1)) + #expect(f.metrics.snapshot().labels["videoSize"] == nil) + f.coordinator.onPresented = { _ in } + f.coordinator.onFrameSize = { _ in } + #expect(f.coordinator.onPresented == nil && f.coordinator.onFrameSize == nil) + f.coordinator.bind { Issue.record("bind after stop must be ignored") } + f.coordinator.arrivalScheduled() + f.coordinator.stop() + #expect(f.coordinator.stopped && f.draws.value == 1 && f.counter("renderedFrames") == 0) + } + + /// The reentrant path: select → reportSize → the size callback closes the + /// renderer → the caller still reaches commit. Commit is refused before any + /// handler is installed, anything retained or anything submitted. + @Test func stopInsideTheSizeCallbackRefusesTheHeldSelectionAtCommit() throws { + let f = Fixture(renderOnArrival: true) + let coordinator = f.coordinator + coordinator.onFrameSize = { _ in coordinator.stop() } + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer)) + _ = f.queue.drain() + let selected = try #require(coordinator.select()) + coordinator.reportSize(CGSize(width: 64, height: 64)) + #expect(coordinator.stopped) + let committed = coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + #expect(!committed) + } + #expect(!submission.committed && !submission.holdsPresentationHandler && !coordinator.inFlight) + #expect(weakBuffer == nil) // nothing retained by a refused commit + submission.complete() + submission.present(at: 1) + #expect(f.queue.pending == 0 && f.counter("renderedFrames") == 0 && f.counter("presentationCallbacks") == 0) + } + + @Test func aSecondCommitWhileTheFirstIsPendingIsRefusedAndRetainsNothing() throws { + let f = Fixture(renderOnArrival: false) + let first = ControlledSubmission() + let second = ControlledSubmission() + weak var weakSecond: CVPixelBuffer? + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + let selected = try #require(f.coordinator.select()) + #expect( + f.coordinator.commit( + first, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1)) + try autoreleasepool { + let buffer = try makeBuffer() + weakSecond = buffer + let late = frame(buffer, rtp: 2) + #expect( + !f.coordinator.commit( + second, retaining: Retained(late), frame: late, isNewFrame: true, submittedAt: 2)) + } + #expect(weakSecond == nil && !second.committed && !second.holdsPresentationHandler) + #expect(f.coordinator.inFlight && first.committed) + first.complete() + #expect(f.queue.drain() == 1 && !f.coordinator.inFlight && f.counter("renderedFrames") == 1) + } + + /// Distinct order: the GPU completion and the presentation happen while the + /// renderer is live (their main-actor hops are queued), the renderer stops, + /// and only then the queued work runs — no draw, no product notification. + @Test func mainActorWorkQueuedBeforeStopIsInertWhenDrained() throws { + let f = Fixture(renderOnArrival: true) + var notified: [UInt32] = [] + f.coordinator.onPresented = { notified.append($0) } + let submission = ControlledSubmission() + f.mailbox.put(frame(try makeBuffer(), rtp: 9)) + _ = f.queue.drain() + #expect(f.draws.value == 1) + let selected = try #require(f.coordinator.select()) + f.coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + submission.complete() // live: queues the completion hop (would request a draw) + submission.present(at: 2) // live: queues the presentation hop (would notify) + #expect(f.queue.pending == 2 && f.coordinator.inFlight) + #expect(f.counter("renderedFrames") == 1 && f.counter("presentedFrames") == 1) + f.coordinator.stop() + #expect(f.queue.drain() == 2) + #expect(!f.coordinator.inFlight && f.draws.value == 1 && notified.isEmpty) + #expect(f.coordinator.select() == nil) + } + + @Test func stopBeforeAnyFrameAndACompletionErrorAreHandled() throws { + let f = Fixture(renderOnArrival: true) + f.coordinator.stop() + f.mailbox.put(frame(try makeBuffer())) + #expect(f.queue.pending == 0 && f.coordinator.select() == nil) + let g = Fixture(renderOnArrival: true) + g.mailbox.put(frame(try makeBuffer())) + _ = g.queue.drain() + let selected = try #require(g.coordinator.select()) + let submission = ControlledSubmission() + g.coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1) + submission.complete(false) + #expect(g.counter("renderErrors") == 1 && g.counter("renderedFrames") == 0) + // live arrival drive continues after an error + #expect(g.queue.drain() == 1 && !g.coordinator.inFlight && g.draws.value == 2) + } +} + +/// Off-main preparation through the same coordinator: one reserved slot, the +/// newest mailbox frame, main-actor callbacks, and stop/failure/stale release. +/// The preparer is a controlled gate; the hop is the explicit queue. +@MainActor +struct ScreenSharingRenderPreparationTests { + /// Scalars of a request, kept as history; the full request (with its frame) + /// is held only while the preparation is outstanding and released on finish. + private struct RequestSummary: Equatable { + let rtpTimestamp: UInt32 + let isNewFrame: Bool + let geometry: ScreenSharingRenderGeometry + } + + private final class ControlledPreparer: ScreenSharingRenderPreparer, @unchecked Sendable { + private let lock = NSLock() + private var outstanding: + [(request: ScreenSharingPreparationRequest, completion: @Sendable (ScreenSharingPreparedSubmission?) -> Void)] = + [] + private var history: [RequestSummary] = [] + func prepare( + _ request: ScreenSharingPreparationRequest, + completion: @escaping @Sendable (ScreenSharingPreparedSubmission?) -> Void + ) { + lock.withLock { + history.append( + .init(rtpTimestamp: request.frame.rtpTimestamp, isNewFrame: request.isNewFrame, geometry: request.geometry)) + outstanding.append((request, completion)) + } + } + var requestCount: Int { lock.withLock { history.count } } + var last: RequestSummary? { lock.withLock { history.last } } + var outstandingCount: Int { lock.withLock { outstanding.count } } + /// Finishes the oldest outstanding preparation with a result built from its + /// frame (or nil), then releases the request — the worker keeps nothing. + func finish(_ make: (ScreenSharingVideoFrame) -> ScreenSharingPreparedSubmission?) { + let entry = lock.withLock { outstanding.isEmpty ? nil : outstanding.removeFirst() } + guard let entry else { return } + let prepared = make(entry.request.frame) + entry.completion(prepared) + } + } + + private final class ControlledSubmission: ScreenSharingRenderSubmission, @unchecked Sendable { + private let lock = NSLock() + private var completed: (@Sendable (Bool) -> Void)? + private var presented: (@Sendable (Double) -> Void)? + private var installedCompletion: (@Sendable (Bool) -> Void)? + private(set) var committed = false + func onCompleted(_ handler: @escaping @Sendable (Bool) -> Void) { + lock.withLock { + completed = handler + installedCompletion = handler + } + } + func onPresented(_ handler: @escaping @Sendable (Double) -> Void) { lock.withLock { presented = handler } } + func commit() { lock.withLock { committed = true } } + var holdsHandlers: Bool { lock.withLock { completed != nil || presented != nil } } + var holdsPresentationHandler: Bool { lock.withLock { presented != nil } } + func complete(_ success: Bool = true) { + let handler = lock.withLock { + let h = completed; completed = nil; return h + } + handler?(success) + } + func replayCompletion(_ success: Bool = true) -> Bool { + guard let handler = lock.withLock({ installedCompletion }) else { return false } + handler(success) + return true + } + func forgetInstalledCompletion() { lock.withLock { installedCompletion = nil } } + func present(at time: Double) { + let handler = lock.withLock { + let h = presented; presented = nil; return h + } + handler?(time) + } + } + + private final class WorkQueue: @unchecked Sendable { + private let lock = NSLock() + private var items: [@Sendable @MainActor () -> Void] = [] + var hop: ScreenSharingRenderCoordinator.Hop { { [self] work in self.lock.withLock { self.items.append(work) } } } + var pending: Int { lock.withLock { items.count } } + @MainActor func drain() -> Int { + let work = lock.withLock { + let w = items; items = []; return w + } + for item in work { item() } + return work.count + } + } + + private final class Retained: @unchecked Sendable { + let frame: ScreenSharingVideoFrame + init(_ frame: ScreenSharingVideoFrame) { self.frame = frame } + } + + @MainActor private struct Fixture { + let mailbox = ScreenSharingFrameMailbox() + let metrics = ScreenSharingMetrics() + let queue = WorkQueue() + let preparer = ControlledPreparer() + let coordinator: ScreenSharingRenderCoordinator + let draws = Counter() + @MainActor final class Counter { var value = 0 } + static let size = CGSize(width: 1920, height: 1080) + let geometry = ScreenSharingRenderGeometry( + fitToWindow: false, clearColor: SIMD4(0.1, 0.2, 0.3, 1), drawableSize: size) + init() { + coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: metrics, renderOnArrival: true, redrawsOnDemand: true, hop: queue.hop) + let draws = draws + coordinator.bind { draws.value += 1 } + } + func counter(_ name: String) -> Int { metrics.snapshot().counters[name] ?? 0 } + /// What the view does on a draw request in the off-main mode (size from its backing store). + @discardableResult func drawRequested(size: CGSize = size) -> Bool { + coordinator.prepare( + with: preparer, + geometry: .init(fitToWindow: geometry.fitToWindow, clearColor: geometry.clearColor, drawableSize: size)) + } + /// A prepared result: the submission plus the retained frame (as the worker's TextureFrame would be). + func prepared(_ submission: ControlledSubmission) -> (ScreenSharingVideoFrame) -> ScreenSharingPreparedSubmission? { + { frame in .init(submission: submission, retained: Retained(frame), videoSize: CGSize(width: 64, height: 64)) } + } + } + + private func makeBuffer() throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 64, 64, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + return try #require(pixel) + } + + private func frame(_ buffer: CVPixelBuffer, rtp: UInt32 = 7, receivedAt: Double? = 1.0) -> ScreenSharingVideoFrame { + .init(pixelBuffer: buffer, timestampNs: Int64(rtp), rtpTimestamp: rtp, receivedAtSeconds: receivedAt) + } + + @Test func oneSlotNewestMailboxFrameAndMainActorCallbacks() throws { + let f = Fixture() + var sizes: [CGSize] = [] + var notified: [UInt32] = [] + f.coordinator.onFrameSize = { sizes.append($0) } + f.coordinator.onPresented = { notified.append($0) } + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + #expect(f.queue.drain() == 1 && f.draws.value == 1) + #expect(f.drawRequested()) + let request = try #require(f.preparer.last) + #expect(request.isNewFrame && request.rtpTimestamp == 1 && request.geometry == f.geometry) + #expect(request.geometry.drawableSize == Fixture.size) // the size travels with the request + #expect(f.coordinator.inFlight && f.coordinator.preparationToken != nil && f.coordinator.isHoldingCachedFrame) + // While the slot is held: a second draw request prepares nothing, commits are refused, + // arrivals only replace the mailbox's newest frame. + #expect(!f.drawRequested() && f.preparer.requestCount == 1) + let refused = ControlledSubmission() + let held = frame(try makeBuffer(), rtp: 99) + #expect(!f.coordinator.commit(refused, retaining: Retained(held), frame: held, isNewFrame: true, submittedAt: 1)) + f.mailbox.put(frame(try makeBuffer(), rtp: 2)) + f.mailbox.put(frame(try makeBuffer(), rtp: 3)) + _ = f.queue.drain() // the arrival hop for frame 2 requests a draw, which prepares nothing + #expect(f.preparer.requestCount == 1 && f.mailbox.isHolding && f.mailbox.droppedFrames == 1) + // The worker finishes; the result is committed on the main actor behind the guard. + let submission = ControlledSubmission() + f.preparer.finish(f.prepared(submission)) + #expect(f.queue.drain() == 1) + #expect(submission.committed && f.coordinator.inFlight && f.coordinator.preparationToken == nil) + #expect(sizes == [CGSize(width: 64, height: 64)]) + submission.present(at: 2) + _ = f.queue.drain() + #expect(notified == [1]) + // GPU completion — not presentation — releases the slot and drives the next arrival. + submission.complete() + _ = f.queue.drain() + #expect(!f.coordinator.inFlight && f.counter("renderedFrames") == 1) + #expect(f.drawRequested() && f.preparer.last?.rtpTimestamp == 3) // newest, frame 2 was replaced + } + + @Test func stopWhilePreparationIsHeldClearsTheSlotOnTheLateResultAndReleasesEverything() throws { + let f = Fixture() + var sizes: [CGSize] = [] + f.coordinator.onFrameSize = { sizes.append($0) } + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer)) + _ = f.queue.drain() + #expect(f.drawRequested()) + } + #expect(weakBuffer != nil && f.preparer.outstandingCount == 1) // held by the cache and the outstanding request + f.coordinator.stop() + // the executing preparation is still accounted + #expect(f.coordinator.inFlight && f.coordinator.preparationToken != nil) + #expect(!f.coordinator.isHoldingCachedFrame && weakBuffer != nil) // only the worker's request holds it now + autoreleasepool { f.preparer.finish(f.prepared(submission)) } // the worker returns once; its request is released + #expect(f.queue.drain() == 1) + #expect(!f.coordinator.inFlight && f.coordinator.preparationToken == nil) // slot gone + #expect(!submission.committed && !submission.holdsHandlers && sizes.isEmpty && f.draws.value == 1) + #expect(weakBuffer == nil) // the dropped result retained nothing + #expect(f.counter("renderedFrames") == 0 && f.counter("renderDrops") == 0) + #expect(f.queue.pending == 0) + } + + @Test func committedResultKeepsResourcesThroughStopUntilGPUCompletionWhilePresentationHoldsScalarsOnly() throws { + let f = Fixture() + var notified: [UInt32] = [] + f.coordinator.onPresented = { notified.append($0) } + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer, rtp: 5)) + _ = f.queue.drain() + #expect(f.drawRequested()) + f.preparer.finish(f.prepared(submission)) + _ = f.queue.drain() + } + #expect(submission.committed && f.coordinator.inFlight) + f.coordinator.stop() + #expect(f.coordinator.inFlight && weakBuffer != nil) // GPU still owns the buffer and textures + submission.complete() + submission.forgetInstalledCompletion() + _ = f.queue.drain() + #expect(!f.coordinator.inFlight && f.draws.value == 1) // slot cleared, no restart after stop + // released; the pending presentation handler holds scalars only + #expect(weakBuffer == nil && submission.holdsPresentationHandler) + submission.present(at: 3) + _ = f.queue.drain() + #expect(notified.isEmpty && f.counter("presentationCallbacks") == 1) + } + + @Test func preparationFailureReleasesTheSlotAndItsRequestAndDrivesTheWaitingFrame() throws { + let f = Fixture() + weak var weakFirst: CVPixelBuffer? + try autoreleasepool { + let buffer = try makeBuffer() + weakFirst = buffer + f.mailbox.put(frame(buffer, rtp: 1)) + _ = f.queue.drain() + #expect(f.drawRequested()) + f.mailbox.put(frame(try makeBuffer(), rtp: 2)) + _ = f.queue.drain() // arrival while held: no preparation + #expect(f.preparer.requestCount == 1) + f.preparer.finish { _ in nil } // e.g. no drawable (timeout or invalid layer properties) + } + #expect(f.queue.drain() == 1) + #expect(!f.coordinator.inFlight && f.counter("renderDrops") == 1 && f.draws.value == 3) + #expect(f.drawRequested() && f.preparer.last?.rtpTimestamp == 2) // the waiting newest frame proceeds + #expect(weakFirst == nil) // frame 1: request released on failure, cache replaced by frame 2 + } + + @Test func idleCachedFrameRedrawsOnResizeOrFitWithoutANewVideoFrame() throws { + let f = Fixture() + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + _ = f.queue.drain() + #expect(f.drawRequested()) + let submission = ControlledSubmission() + f.preparer.finish(f.prepared(submission)) + _ = f.queue.drain() + submission.complete() + _ = f.queue.drain() + #expect(!f.coordinator.inFlight && !f.mailbox.isHolding) // idle, cache held, no video frame coming + let drawsBefore = f.draws.value + f.coordinator.setNeedsRedraw() // resize / backing scale / fit change + f.coordinator.setNeedsRedraw() // coalesced: one scheduled hop for the burst + #expect(f.draws.value == drawsBefore && f.queue.pending == 1) // never a synchronous reentrant draw + #expect(f.queue.drain() == 1 && f.draws.value == drawsBefore + 1) + #expect(f.drawRequested(size: CGSize(width: 1280, height: 720))) + let redraw = try #require(f.preparer.last) + #expect(!redraw.isNewFrame && redraw.rtpTimestamp == 1) + #expect(redraw.geometry.drawableSize == CGSize(width: 1280, height: 720)) + // A redraw requested while the slot is held waits for completion (no extra draw). + f.coordinator.setNeedsRedraw() + #expect(f.queue.drain() == 1 && f.draws.value == drawsBefore + 1) + let second = ControlledSubmission() + f.preparer.finish(f.prepared(second)) + _ = f.queue.drain() + second.complete() + _ = f.queue.drain() + // completion drove the pending redraw + #expect(f.draws.value == drawsBefore + 2 && f.coordinator.select()?.isNewFrame == false) + } + + @Test func sizeCallbackStoppingTheRendererDropsThePreparedResultAndClearsTheSlot() throws { + let f = Fixture() + let coordinator = f.coordinator + coordinator.onFrameSize = { _ in coordinator.stop() } + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + f.mailbox.put(frame(buffer)) + _ = f.queue.drain() + #expect(f.drawRequested()) + f.preparer.finish(f.prepared(submission)) + _ = f.queue.drain() + } + #expect(coordinator.stopped && !submission.committed && !submission.holdsHandlers) + #expect(!coordinator.inFlight && coordinator.preparationToken == nil) // prepared-but-unsubmitted slot released + #expect(weakBuffer == nil) + } + + @Test func staleResultsAndReplayedOldCompletionsNeverScheduleOrReleaseAnotherOccupancy() throws { + let f = Fixture() + f.mailbox.put(frame(try makeBuffer(), rtp: 1)) + _ = f.queue.drain() + #expect(f.drawRequested()) + let firstSubmission = ControlledSubmission() + f.preparer.finish(f.prepared(firstSubmission)) + _ = f.queue.drain() + firstSubmission.complete() + _ = f.queue.drain() + #expect(!f.coordinator.inFlight) + // A second occupancy is reserved; then the FIRST submission's completion fires again (replayed). + f.mailbox.put(frame(try makeBuffer(), rtp: 2)) + _ = f.queue.drain() + #expect(f.drawRequested() && f.coordinator.inFlight) + let drawsBefore = f.draws.value + #expect(firstSubmission.replayCompletion()) + #expect(f.queue.drain() == 1) + #expect(f.coordinator.inFlight && f.coordinator.preparationToken != nil && f.draws.value == drawsBefore) + // A stale result for the first token is ignored while the second preparation is held. + let stale = ControlledSubmission() + let staleFrame = frame(try makeBuffer(), rtp: 1) + f.coordinator.finishPreparation( + token: 1, .init(submission: stale, retained: Retained(staleFrame), videoSize: .zero)) + #expect(!stale.committed && f.coordinator.inFlight && f.coordinator.preparationToken != nil) + // The second result commits; a duplicate result for the same token is ignored. + let secondSubmission = ControlledSubmission() + f.preparer.finish(f.prepared(secondSubmission)) + _ = f.queue.drain() + #expect(secondSubmission.committed) + let duplicate = ControlledSubmission() + f.coordinator.finishPreparation( + token: 2, .init(submission: duplicate, retained: Retained(staleFrame), videoSize: .zero)) + #expect(!duplicate.committed && f.coordinator.inFlight) + secondSubmission.complete() + _ = f.queue.drain() + #expect(!f.coordinator.inFlight) + } +} + +/// Audit boundaries through the real coordinator seams (main-actor commit and +/// off-main preparation): identity carried as scalars from the selected frame +/// to submission, GPU completion callback and presented result; nil audit = no +/// recording and no clock reads. +@MainActor +struct ScreenSharingRenderCoordinatorAuditTests { + private final class Clock: @unchecked Sendable { + private let lock = NSLock() + private(set) var reads = 0 + var read: @Sendable () -> Int64 { + { [self] in + self.lock.withLock { + self.reads += 1; return 1_500_000_000 + } + } + } + } + private final class ControlledSubmission: ScreenSharingRenderSubmission, @unchecked Sendable { + private let lock = NSLock() + private var completed: (@Sendable (Bool) -> Void)? + private var presented: (@Sendable (Double) -> Void)? + func onCompleted(_ handler: @escaping @Sendable (Bool) -> Void) { lock.withLock { completed = handler } } + func onPresented(_ handler: @escaping @Sendable (Double) -> Void) { lock.withLock { presented = handler } } + func commit() {} + func complete(_ ok: Bool) { + lock.withLock { + let h = completed; completed = nil; return h + }?(ok) + } + func present(at t: Double) { + lock.withLock { + let h = presented; presented = nil; return h + }?(t) + } + } + private final class Retained: @unchecked Sendable { + let frame: ScreenSharingVideoFrame + init(_ frame: ScreenSharingVideoFrame) { self.frame = frame } + } + private final class ControlledPreparer: ScreenSharingRenderPreparer, @unchecked Sendable { + private let lock = NSLock() + private var pending: [(ScreenSharingPreparationRequest, @Sendable (ScreenSharingPreparedSubmission?) -> Void)] = [] + func prepare( + _ request: ScreenSharingPreparationRequest, + completion: @escaping @Sendable (ScreenSharingPreparedSubmission?) -> Void + ) { + lock.withLock { pending.append((request, completion)) } + } + var lastIdentity: ScreenSharingFrameDeliveryAudit.Identity?? { + lock.withLock { pending.last.map { $0.0.auditIdentity } } + } + func finish(_ make: (ScreenSharingVideoFrame) -> ScreenSharingPreparedSubmission?) { + let entry = lock.withLock { pending.isEmpty ? nil : pending.removeFirst() } + guard let entry else { return } + entry.1(make(entry.0.frame)) + } + } + + private func makeBuffer() throws -> CVPixelBuffer { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + return try #require(pixel) + } + + @Test func mainActorPathRecordsSelectionSubmissionCompletionAndPresentedWithTheFrameIdentity() throws { + let clock = Clock() + let audit = ScreenSharingFrameDeliveryAudit( + window: try .init(beginSeconds: 0, durationSeconds: 10), clock: clock.read) + audit.start(originNs: 0) + let mailbox = ScreenSharingFrameMailbox() + let coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: ScreenSharingMetrics(), renderOnArrival: false, + hop: { work in MainActor.assumeIsolated { work() } }) + coordinator.bind {} + coordinator.audit = audit + let identity = ScreenSharingFrameDeliveryAudit.Identity(sequence: 42, generation: 1) + weak var weakBuffer: CVPixelBuffer? + let submission = ControlledSubmission() + try autoreleasepool { + let buffer = try makeBuffer() + weakBuffer = buffer + mailbox.put( + .init( + pixelBuffer: buffer, timestampNs: 1, rtpTimestamp: 900, receivedAtSeconds: 1.0, + deliveryAuditIdentity: identity)) + let selected = try #require(coordinator.select()) + #expect( + coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1.2)) + coordinator.setNeedsRedraw() + } + submission.present(at: 1.25) + submission.complete(true) + let s = audit.snapshot() + #expect(s.stages == [4, 9, 11, 10] && s.sequences == [42, 42, 42, 42] && s.rtpTimestamps == [900, 900, 900, 900]) + #expect(s.valueNs == [0, 0, 1_250_000_000, 1] && s.duplicateEvents == 0 && s.missingIdentityEvents == 0) + // a cached redraw is not a delivery: selecting the cache records nothing + #expect(coordinator.select()?.isNewFrame == false && audit.snapshot().recorded == 4) + #expect(weakBuffer != nil) // only the coordinator's redraw cache still holds the buffer … + coordinator.stop() + #expect(weakBuffer == nil && audit.snapshot().recorded == 4) // … the audit's four records retained nothing + } + + @Test func preparationPathCarriesTheIdentityToPreparedReceiptSubmissionAndCompletion() throws { + let clock = Clock() + let audit = ScreenSharingFrameDeliveryAudit( + window: try .init(beginSeconds: 0, durationSeconds: 10), clock: clock.read) + audit.start(originNs: 0) + let mailbox = ScreenSharingFrameMailbox() + let coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: ScreenSharingMetrics(), renderOnArrival: true, redrawsOnDemand: true, + hop: { work in MainActor.assumeIsolated { work() } }) + coordinator.bind {} + coordinator.audit = audit + let preparer = ControlledPreparer() + let identity = ScreenSharingFrameDeliveryAudit.Identity(sequence: 7, generation: 2) + mailbox.put( + .init( + pixelBuffer: try makeBuffer(), timestampNs: 1, rtpTimestamp: 70, receivedAtSeconds: 1.0, + deliveryAuditIdentity: identity)) + #expect( + coordinator.prepare( + with: preparer, geometry: .init(fitToWindow: true, clearColor: .zero, drawableSize: CGSize(width: 8, height: 8)) + )) + #expect(preparer.lastIdentity == .some(identity)) // the worker request carries the identity as a scalar + let submission = ControlledSubmission() + preparer.finish { frame in + .init(submission: submission, retained: Retained(frame), videoSize: CGSize(width: 16, height: 16)) + } + submission.complete(true) + submission.present(at: 0) // unpresented result must be declared, not skipped + let s = audit.snapshot() + #expect(s.stages == [4, 8, 9, 10, 11] && s.sequences.allSatisfy { $0 == 7 } && s.valueNs == [0, 1, 0, 1, 0]) + #expect(s.coverage["presentedResultZero"] == 1) + // a failed preparation is still receipted (value 0) and nothing else is recorded for it + mailbox.put( + .init( + pixelBuffer: try makeBuffer(), timestampNs: 2, rtpTimestamp: 71, receivedAtSeconds: 1.1, + deliveryAuditIdentity: .init(sequence: 8, generation: 2))) + #expect( + coordinator.prepare( + with: preparer, geometry: .init(fitToWindow: true, clearColor: .zero, drawableSize: CGSize(width: 8, height: 8)) + )) + preparer.finish { _ in nil } + let t = audit.snapshot() + #expect(t.stages.suffix(2) == [4, 8] && t.valueNs.last == 0 && t.sequences.suffix(2) == [8, 8]) + } + + @Test func nilAuditRecordsNothingAndReadsNoClock() throws { + let clock = Clock() + let audit = ScreenSharingFrameDeliveryAudit( + window: try .init(beginSeconds: 0, durationSeconds: 10), clock: clock.read) + let mailbox = ScreenSharingFrameMailbox() + let coordinator = ScreenSharingRenderCoordinator( + mailbox: mailbox, metrics: ScreenSharingMetrics(), renderOnArrival: false, + hop: { work in MainActor.assumeIsolated { work() } }) + coordinator.bind {} + #expect(coordinator.audit == nil) // default: disabled + mailbox.put(.init(pixelBuffer: try makeBuffer(), timestampNs: 1, rtpTimestamp: 5, receivedAtSeconds: 1.0)) + let selected = try #require(coordinator.select()) + let submission = ControlledSubmission() + #expect( + coordinator.commit( + submission, retaining: Retained(selected.frame), frame: selected.frame, isNewFrame: true, submittedAt: 1.2)) + submission.present(at: 1.3) + submission.complete(true) + #expect(clock.reads == 0 && audit.snapshot().recorded == 0 && selected.frame.deliveryAuditIdentity == nil) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRtcEventLogDiagnosticTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRtcEventLogDiagnosticTests.swift new file mode 100644 index 000000000..9faa5250f --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingRtcEventLogDiagnosticTests.swift @@ -0,0 +1,351 @@ +import Foundation +import Testing + +@testable import CodevisorScreenSharing + +/// Lifecycle of the receiver-only RTC event-log diagnostic with injected +/// clock/start/stop boundaries: window validation, once-only start and stop, +/// start false, stop boundary not invoked, cancellation and repeated cleanup, +/// late and missed ticks, zero-begin at the media-start tick, the sleep seam, +/// exclusive output reservation, preflight refusals, immutable record. The +/// fake start/stop advance the injected clock synchronously: these are +/// simulated-call-duration tests, not a concurrent held-call guarantee. +struct ScreenSharingRtcEventLogDiagnosticTests { + private final class Fake { + var now: Int64 = 1_000 + var startCalls: [(String, Int64)] = [] + var stopCalls = 0 + var startResult = true + var stopResult = true + var duringStart: Int64? + var duringStop: Int64? + var writeOnStart: Data? + var boundaries: ScreenSharingRtcEventLogDiagnostic.Boundaries { + .init( + clock: { [self] in + now += 1 // every read advances, so before < after is a real ordering, not equality + return now + }, + start: { [self] path, cap in + startCalls.append((path, cap)) + now += 100 // simulated call duration; the bracket must enclose it + duringStart = now + if let data = writeOnStart { try? data.write(to: URL(fileURLWithPath: path)) } // "wb" onto the reserved file + return startResult + }, + stop: { [self] in + stopCalls += 1 + now += 100 + duringStop = now + return stopResult + }) + } + } + + private struct Fixture { + let log: ScreenSharingRtcEventLogDiagnostic + let fake: Fake + let directory: URL + var path: String { log.path } + func remove() { try? FileManager.default.removeItem(at: directory) } + } + + private func make(begin: Double = 45, duration: Double = 70) throws -> Fixture { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("rtc-event-log-" + UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let path = directory.appendingPathComponent("x.json.rtc-event-log.binarypb").path + try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(path) // option-time preflight passes on a fresh stem + let fake = Fake() + let log = ScreenSharingRtcEventLogDiagnostic( + window: try .init(beginSeconds: begin, durationSeconds: duration), path: path, boundaries: fake.boundaries) + return Fixture(log: log, fake: fake, directory: directory) + } + + @Test func windowShapeIsBounded() throws { + #expect(throws: ScreenSharingError.self) { + try ScreenSharingRtcEventLogDiagnostic.Window(beginSeconds: -1, durationSeconds: 10) + } + #expect(throws: ScreenSharingError.self) { + try ScreenSharingRtcEventLogDiagnostic.Window(beginSeconds: 0, durationSeconds: 0.5) + } + #expect(throws: ScreenSharingError.self) { + try ScreenSharingRtcEventLogDiagnostic.Window(beginSeconds: 0, durationSeconds: 121) + } + #expect(throws: ScreenSharingError.self) { + try ScreenSharingRtcEventLogDiagnostic.Window(beginSeconds: .nan, durationSeconds: 10) + } + let window = try ScreenSharingRtcEventLogDiagnostic.Window(beginSeconds: 45, durationSeconds: 70) + #expect(window.endSeconds == 115 && ScreenSharingRtcEventLogDiagnostic.fixedMaxSizeBytes == 8 * 1024 * 1024) + } + + @Test func normalLifecycleStartsAndStopsExactlyOnce() throws { + let f = try make() + defer { f.remove() } + let (log, fake) = (f.log, f.fake) + fake.writeOnStart = Data([0x82, 0x01]) + #expect(log.nextBoundarySeconds() == 45 && log.record.state == .waiting && !log.record.final) + log.tick(measuredSeconds: 0) + log.tick(measuredSeconds: 44.999) + // nothing reserved before begin + #expect(fake.startCalls.isEmpty && !FileManager.default.fileExists(atPath: f.path)) + log.tick(measuredSeconds: 45.003) + #expect(fake.startCalls.count == 1 && fake.startCalls[0].0 == f.path && fake.startCalls[0].1 == 8 * 1024 * 1024) + var r = log.record + #expect( + r.state == .started && r.startReturned == true && r.startApiInvoked && r.startAttemptedAtMeasuredSeconds == 45.003 + ) + #expect(r.outputReservation?.hasPrefix("reserved") == true) + // the API overwrote only the reserved file + #expect(try Data(contentsOf: URL(fileURLWithPath: f.path)) == Data([0x82, 0x01])) + let startBefore = try #require(r.startCallBeforeNs) + let duringStart = try #require(fake.duringStart) + let startAfter = try #require(r.startCallAfterNs) + #expect(startBefore < duringStart && duringStart < startAfter) + #expect(log.nextBoundarySeconds() == 115) + log.tick(measuredSeconds: 46) + log.tick(measuredSeconds: 114.9) + #expect(fake.startCalls.count == 1 && fake.stopCalls == 0) + log.tick(measuredSeconds: 115.001) + r = log.record + #expect( + fake.stopCalls == 1 && r.state == .stopped && r.final && r.stopAttempted && r.stopApiInvoked + && r.stoppedSuccessfulStart) + #expect(r.stopAttemptedAtMeasuredSeconds == 115.001 && r.closeReason == "requested window end") + let stopBefore = try #require(r.stopCallBeforeNs) + let duringStop = try #require(fake.duringStop) + let stopAfter = try #require(r.stopCallAfterNs) + #expect(stopBefore < duringStop && duringStop < stopAfter) + #expect(log.nextBoundarySeconds() == nil) + // Immutable once final: later ticks, finish and early close change nothing and make no call. + log.tick(measuredSeconds: 200) + log.finish(measuredSeconds: 200) + log.closeEarly(measuredSeconds: nil, reason: "x") + #expect(log.record == r && fake.startCalls.count == 1 && fake.stopCalls == 1) + #expect( + r.requestedBeginSeconds == 45 && r.requestedEndSeconds == 115 && r.kind == "codevisor.rtcEventLogDiagnostic") + #expect(r.notes.contains { $0.contains("[t, t+1) ms") } && r.notes.contains { $0.contains("not NIC arrival") }) + #expect(r.notes.contains { $0.contains("a write failure") }) + } + + @Test func startFalseIsNeverStopped() throws { + let f = try make() + defer { f.remove() } + f.fake.startResult = false + f.log.tick(measuredSeconds: 45) + let r = f.log.record + #expect( + r.state == .startFailed && r.startReturned == false && r.startApiInvoked && r.final && !r.stopAttempted + && !r.stoppedSuccessfulStart) + #expect(r.closeReason == "start returned false at 45.0 s; no stop call" && f.log.nextBoundarySeconds() == nil) + f.log.tick(measuredSeconds: 115) + f.log.finish(measuredSeconds: 116) + f.log.closeEarly(measuredSeconds: 117, reason: "failure") + #expect(f.fake.startCalls.count == 1 && f.fake.stopCalls == 0 && f.log.record == r) + // The owned empty reservation stays; it holds no evidence. + #expect(try Data(contentsOf: URL(fileURLWithPath: f.path)).isEmpty) + } + + @Test func stopNotInvokedByBoundaryIsNeverCredited() throws { + let f = try make() + defer { f.remove() } + f.fake.stopResult = false // the peer was already closed: the native stop API did not run + f.log.tick(measuredSeconds: 45) + f.log.tick(measuredSeconds: 115) + let r = f.log.record + #expect( + f.fake.stopCalls == 1 && r.state == .stopSkipped && r.final && r.stopAttempted && !r.stopApiInvoked + && !r.stoppedSuccessfulStart) + #expect(r.closeReason?.contains("native API was not invoked") == true && r.stopCallBeforeNs != nil) + f.log.closeEarly(measuredSeconds: 116, reason: "again") + f.log.finish(measuredSeconds: 117) + #expect(f.fake.stopCalls == 1 && f.log.record == r) // once only, even when not credited + } + + @Test func earlyCloseStopsOnceAndRepeatedCleanupIsNoOp() throws { + let f = try make() + defer { f.remove() } + f.log.tick(measuredSeconds: 45) + f.log.closeEarly(measuredSeconds: 60.5, reason: "runner stop") + let r = f.log.record + #expect( + f.fake.stopCalls == 1 && r.state == .stopped && r.stoppedSuccessfulStart + && r.stopAttemptedAtMeasuredSeconds == 60.5) + #expect(r.closeReason == "early close: runner stop") + f.log.closeEarly(measuredSeconds: 61, reason: "again") + f.log.finish(measuredSeconds: 62) + f.log.tick(measuredSeconds: 200) + #expect(f.fake.stopCalls == 1 && f.log.record == r) + } + + @Test func closeBeforeStartMakesNoCallAndNoReservation() throws { + let f = try make() + defer { f.remove() } + f.log.tick(measuredSeconds: 10) + f.log.closeEarly(measuredSeconds: 12, reason: "cancelled") + let r = f.log.record + #expect( + r.state == .closedWithoutStart && r.final && r.startReturned == nil && !r.startApiInvoked && !r.stopAttempted) + #expect(f.fake.startCalls.isEmpty && f.fake.stopCalls == 0 && !FileManager.default.fileExists(atPath: f.path)) + #expect(r.closeReason == "early close: cancelled; no start call was made") + f.log.tick(measuredSeconds: 45) // the window begin arriving later never starts a closed diagnostic + #expect(f.fake.startCalls.isEmpty && f.log.record == r) + let g = try make() + defer { g.remove() } + g.log.finish(measuredSeconds: 5) + #expect( + g.log.record.state == .closedWithoutStart + && g.log.record.closeReason == "normal completion; no start call was made" && g.fake.startCalls.isEmpty) + } + + @Test func lateStartAndMissedStopStayTruthful() throws { + let f = try make() + defer { f.remove() } + f.log.tick(measuredSeconds: 47.5) // a late tick: requested 45, actual 47.5, both kept + #expect(f.log.record.startAttemptedAtMeasuredSeconds == 47.5 && f.log.record.requestedBeginSeconds == 45) + f.log.tick(measuredSeconds: 100) + f.log.finish(measuredSeconds: 130) // no tick ever reached the end; the normal finish stops it and says so + let r = f.log.record + #expect( + f.fake.stopCalls == 1 && r.stopAttemptedAtMeasuredSeconds == 130 && r.closeReason == "normal completion" + && r.requestedEndSeconds == 115) + // A first opportunity already past the requested end never starts (and reserves nothing). + let late = try make(begin: 1, duration: 1) + defer { late.remove() } + late.log.tick(measuredSeconds: 5) + #expect(late.log.record.state == .closedWithoutStart && late.fake.startCalls.isEmpty && late.log.record.final) + #expect( + late.log.record.closeReason?.hasPrefix( + "first start opportunity at 5.0 s was already past the requested end 2.0 s") == true) + #expect(!FileManager.default.fileExists(atPath: late.path)) + } + + @Test func zeroBeginStartsAtTheMediaStartTick() throws { + let f = try make(begin: 0, duration: 1) + defer { f.remove() } + // begin 0 is not in the future + #expect(ScreenSharingRtcEventLogDiagnostic.sleepSeconds(measured: 0, remaining: 200, nextBoundary: 0) == 1) + f.log.tick(measuredSeconds: 0) // the runner's media-start tick, before its first sleep + #expect( + f.log.record.state == .started && f.log.record.startAttemptedAtMeasuredSeconds == 0 + && f.log.nextBoundarySeconds() == 1) + #expect(ScreenSharingRtcEventLogDiagnostic.sleepSeconds(measured: 0, remaining: 200, nextBoundary: 1) == 1) + #expect( + abs(ScreenSharingRtcEventLogDiagnostic.sleepSeconds(measured: 0.4, remaining: 200, nextBoundary: 1) - 0.6) < 1e-9) + f.log.tick(measuredSeconds: 1.002) + #expect(f.log.record.state == .stopped && f.fake.startCalls.count == 1 && f.fake.stopCalls == 1) + } + + @Test func sleepSeamNeverExceedsTickRemainingOrNextBoundary() { + let sleep = ScreenSharingRtcEventLogDiagnostic.sleepSeconds + #expect(sleep(10, 200, nil, 1) == 1) + #expect(sleep(199.7, 0.3, nil, 1) == 0.3) + #expect(abs(sleep(44.2, 200, 45, 1) - 0.8) < 1e-9) // floating-point subtraction: compare with tolerance + #expect(abs(sleep(114.5, 0.2, 115, 1) - 0.2) < 1e-9) + #expect(sleep(50, 200, 45, 1) == 1) // a boundary already passed does not shorten the tick + #expect(sleep(0, 0, nil, 1) == 0.001) // never a non-positive sleep + } + + @Test func entryAppearingAfterPreflightRefusesTheStartWithoutInvokingTheApi() throws { + let f = try make() + defer { f.remove() } + // appears between option preflight and start + try Data("prior evidence".utf8).write(to: URL(fileURLWithPath: f.path)) + f.log.tick(measuredSeconds: 45) + let r = f.log.record + #expect( + f.fake.startCalls.isEmpty && r.state == .startRefused && r.final && !r.startApiInvoked && r.startReturned == nil + && !r.stopAttempted) + #expect( + r.outputReservation?.hasPrefix("refused:") == true + && r.closeReason == "output reservation failed at 45.0 s; start API not invoked") + #expect(try Data(contentsOf: URL(fileURLWithPath: f.path)) == Data("prior evidence".utf8)) + f.log.finish(measuredSeconds: 200) + #expect(f.fake.stopCalls == 0 && f.log.record == r) + // A dangling symlink appearing after preflight refuses too, and stays. + let g = try make() + defer { g.remove() } + try FileManager.default.createSymbolicLink( + atPath: g.path, withDestinationPath: g.directory.appendingPathComponent("missing").path) + g.log.tick(measuredSeconds: 45) + #expect(g.fake.startCalls.isEmpty && g.log.record.state == .startRefused) + #expect((try? FileManager.default.destinationOfSymbolicLink(atPath: g.path))?.hasSuffix("missing") == true) + // A directory at the path refuses too. + let h = try make() + defer { h.remove() } + try FileManager.default.createDirectory(atPath: h.path, withIntermediateDirectories: false) + h.log.tick(measuredSeconds: 45) + #expect(h.fake.startCalls.isEmpty && h.log.record.state == .startRefused) + } + + @Test func preflightRefusesExistingEntriesAndUnknownErrors() throws { + let dir = FileManager.default.temporaryDirectory.appendingPathComponent("rtc-event-log-" + UUID().uuidString) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(dir.appendingPathComponent("fresh.binarypb").path) + let existing = dir.appendingPathComponent("existing.binarypb") + try Data([1, 2, 3]).write(to: existing) + #expect(throws: ScreenSharingError.self) { try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(existing.path) } + #expect(try Data(contentsOf: existing) == Data([1, 2, 3])) + let dangling = dir.appendingPathComponent("dangling.binarypb") + try FileManager.default.createSymbolicLink( + at: dangling, withDestinationURL: dir.appendingPathComponent("missing-target")) + #expect(throws: ScreenSharingError.self) { try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(dangling.path) } + #expect(throws: ScreenSharingError.self) { try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(dir.path) } + // An lstat error other than absence (ENOTDIR: a path component is a regular file) is refused, not treated as free. + #expect(throws: ScreenSharingError.self) { + try ScreenSharingRtcEventLogDiagnostic.checkOutputPath(existing.appendingPathComponent("child").path) + } + } + + @Test func notInitializedRecordKeepsTheRequest() throws { + let r = ScreenSharingRtcEventLogDiagnostic.Record.notInitialized( + beginSeconds: 45, durationSeconds: 70, path: "/tmp/x.json.rtc-event-log.binarypb", maxSizeBytes: 8 * 1024 * 1024, + reason: "probe failed before the receiver and the log were constructed") + #expect(r.state == .notInitialized && r.final && !r.startApiInvoked && !r.stopAttempted && r.startReturned == nil) + #expect( + r.requestedEndSeconds == 115 && r.path.hasSuffix(".rtc-event-log.binarypb") + && r.closeReason?.hasSuffix("; no API call") == true) + let object = try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(r)) as? [String: Any]) + #expect(object["state"] as? String == "notInitialized" && object["requestedBeginSeconds"] as? Double == 45) + } + + @Test func roleIsCarriedInRecordNotesAndNotInitialized() throws { + let f = try make() + defer { f.remove() } + #expect(f.log.role == .receiver && f.log.record.role == "receiver") + #expect(f.log.record.notes.contains { $0.hasPrefix("receiver log:") }) + let sender = ScreenSharingRtcEventLogDiagnostic( + window: try .init(beginSeconds: 45, durationSeconds: 70), + path: f.directory.appendingPathComponent("s.json.sender-rtc-event-log.binarypb").path, role: .sender, + boundaries: f.fake.boundaries) + sender.tick(measuredSeconds: 45) + sender.tick(measuredSeconds: 115) + let r = sender.record + #expect(r.role == "sender" && r.state == .stopped && r.stopApiInvoked) + // the boundary is conditional: a successful SendRtp either sends directly or queues a network-thread task + let senderNote = try #require(r.notes.first { $0.hasPrefix("sender log:") }) + #expect(senderNote.contains("TRANSPORT HAND-OFF") && senderNote.contains("never socket/NIC egress")) + #expect(senderNote.contains("either sends directly") && senderNote.contains("queues a task there")) + #expect(senderNote.contains("downstream unknowns")) + #expect(!r.notes.contains { $0.hasPrefix("receiver log:") } && r.notes.contains { $0.hasPrefix("sender log:") }) + let n = ScreenSharingRtcEventLogDiagnostic.Record.notInitialized( + beginSeconds: 1, durationSeconds: 2, path: "/x", maxSizeBytes: 8 * 1024 * 1024, reason: "failed early", + role: .sender) + #expect(n.role == "sender" && n.state == .notInitialized && n.notes.contains { $0.hasPrefix("sender log:") }) + let object = try #require(JSONSerialization.jsonObject(with: JSONEncoder().encode(r)) as? [String: Any]) + #expect(object["role"] as? String == "sender") + } + + @Test func recordEncodesAsJSON() throws { + let f = try make() + defer { f.remove() } + f.log.tick(measuredSeconds: 45) + f.log.tick(measuredSeconds: 115) + let data = try JSONEncoder().encode(f.log.record) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect( + object["state"] as? String == "stopped" && object["stopApiInvoked"] as? Bool == true + && object["maxSizeBytes"] as? Int == 8_388_608) + #expect((object["notes"] as? [String])?.count == ScreenSharingRtcEventLogDiagnostic.notes(for: .receiver).count) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceIdleTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceIdleTests.swift new file mode 100644 index 000000000..8d0a84837 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceIdleTests.swift @@ -0,0 +1,479 @@ +import CodevisorTestSupport +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingSourceIdleTests { + private static let threshold: Int64 = 500_000_000 + + private func nanoseconds(_ clock: TestClock, since origin: ContinuousClock.Instant) -> Int64 { + let parts = origin.duration(to: clock.now).components + return parts.seconds * 1_000_000_000 + parts.attoseconds / 1_000_000_000 + } + + @Test func idleNoticeProtocolRoundTripsAndRejectsMalformedBytes() throws { + for value: Int64 in [0, 1, Int64.max] { + let message = ScreenSharingVideoRefreshMessage.sourceIdle(latestTimestampNs: value) + let encoded = message.encoded() + #expect(encoded.count == 9 && encoded[0] == 2) + #expect(try ScreenSharingVideoRefreshMessage.decode(encoded) == message) + } + #expect(try ScreenSharingVideoRefreshMessage.decode(Data([1])) == .keyframe) + let negative = Data([2, 0x80, 0, 0, 0, 0, 0, 0, 0]) + let short = Data([2, 0, 0, 0, 0, 0, 0, 0]) + let long = Data([2, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + for bytes in [Data([2]), short, long, negative, Data([3, 1])] { + #expect(throws: (any Error).self) { try ScreenSharingVideoRefreshMessage.decode(bytes) } + } + } + + @Test func monitorArmsOncePerActivityPeriodAndAnnouncesTheNewestEncodedContent() { + let monitor = ScreenSharingSourceIdleMonitor(thresholdNs: Self.threshold) + #expect(monitor.evaluate(nowNs: 0) == .inactive) + #expect(monitor.recordSubmission(timestampNs: 10, nowNs: 0)) + #expect(!monitor.recordSubmission(timestampNs: 20, nowNs: 100)) + #expect(monitor.evaluate(nowNs: 100) == .wait(untilNs: 100 + Self.threshold)) + monitor.recordEncoded(timestampNs: 10) + monitor.recordEncoded(timestampNs: 20) + monitor.recordEncoded(timestampNs: 15) + #expect(monitor.latestEncodedNs == 20) + #expect(monitor.evaluate(nowNs: 99 + Self.threshold) == .wait(untilNs: 100 + Self.threshold)) + #expect(monitor.evaluate(nowNs: 100 + Self.threshold) == .idle(latestEncodedNs: 20)) + #expect(monitor.evaluate(nowNs: 100 + Self.threshold) == .inactive) + #expect(monitor.recordSubmission(timestampNs: 30, nowNs: 200 + Self.threshold)) + monitor.recordEncoded(timestampNs: 30) + #expect(monitor.evaluate(nowNs: 200 + 2 * Self.threshold) == .idle(latestEncodedNs: 30)) + } + + @Test func monitorReoffersUnencodedContentQuicklyThenAtASlowBoundedRateUntilItIsEncoded() { + let slow: Int64 = 2_000_000_000 + let monitor = ScreenSharingSourceIdleMonitor(thresholdNs: Self.threshold, slowIntervalNs: slow) + #expect(monitor.recordSubmission(timestampNs: 30, nowNs: 0)) + monitor.recordEncoded(timestampNs: 20) + #expect(monitor.evaluate(nowNs: Self.threshold) == .resubmit) + #expect(monitor.evaluate(nowNs: Self.threshold) == .wait(untilNs: 2 * Self.threshold)) + // A refresh re-offers the same buffer, so the encoded identity is the capture's. + monitor.recordEncoded(timestampNs: 30) + #expect(monitor.evaluate(nowNs: 2 * Self.threshold) == .idle(latestEncodedNs: 30)) + // Older encoded content is never announced as current; re-offers slow down. + #expect(monitor.recordSubmission(timestampNs: 40, nowNs: 3 * Self.threshold)) + var now = 3 * Self.threshold + for attempt in 1...ScreenSharingSourceIdleMonitor.maximumQuickResubmissions { + now += Self.threshold + #expect(monitor.evaluate(nowNs: now) == .resubmit) + #expect(monitor.resubmissionCount == attempt) + } + #expect(monitor.evaluate(nowNs: now + Self.threshold) == .wait(untilNs: now + slow)) + #expect(monitor.evaluate(nowNs: now + slow) == .resubmit) + #expect(monitor.evaluate(nowNs: now + slow) == .wait(untilNs: now + 2 * slow)) + #expect(monitor.latestEncodedNs == 30) + monitor.recordEncoded(timestampNs: 40) + #expect(monitor.evaluate(nowNs: now + 2 * slow) == .idle(latestEncodedNs: 40)) + #expect(monitor.evaluate(nowNs: now + 2 * slow) == .inactive) + // The next capture resets the attempt budget to the quick threshold. + #expect(monitor.recordSubmission(timestampNs: 50, nowNs: now + 2 * slow)) + #expect(monitor.resubmissionCount == 0) + #expect(monitor.evaluate(nowNs: now + 2 * slow + Self.threshold) == .resubmit) + monitor.recordEncoded(timestampNs: 50) + #expect(monitor.evaluate(nowNs: now + 2 * slow + 2 * Self.threshold) == .idle(latestEncodedNs: 50)) + let nothingEncoded = ScreenSharingSourceIdleMonitor(thresholdNs: Self.threshold) + #expect(nothingEncoded.recordSubmission(timestampNs: 1, nowNs: 0)) + #expect(nothingEncoded.evaluate(nowNs: Self.threshold) == .resubmit) + nothingEncoded.stop() + #expect(!nothingEncoded.recordSubmission(timestampNs: 2, nowNs: 0)) + #expect(nothingEncoded.evaluate(nowNs: 3 * Self.threshold) == .inactive) + } + + @Test @MainActor func notifierWakesOncePerThresholdSendsOneNoticeAndRetainsItUntilTheChannelOpens() async { + let clock = TestClock() + let origin = clock.now + let monitor = ScreenSharingSourceIdleMonitor(thresholdNs: Self.threshold, slowIntervalNs: 2_000_000_000) + let attempted = TestSignal() + let resubmitted = TestSignal() + var available = false + var notices: [Int64] = [] + let notifier = ScreenSharingSourceIdleNotifier( + monitor: monitor, + resubmit: { resubmitted.signal() }, + notify: { + defer { attempted.signal() } + guard available else { return false } + notices.append($0) + return true + }, + nowNs: { nanoseconds(clock, since: origin) }, sleep: { try await clock.sleep(for: $0) }) + #expect(monitor.recordSubmission(timestampNs: 1, nowNs: 0)) + monitor.recordEncoded(timestampNs: 1) + notifier.activate() + notifier.activate() + await clock.waitForSleep(.milliseconds(500)) + #expect(notices.isEmpty && clock.pendingCount == 1) + clock.advance(by: .milliseconds(250)) + #expect(!monitor.recordSubmission(timestampNs: 2, nowNs: nanoseconds(clock, since: origin))) + monitor.recordEncoded(timestampNs: 2) + clock.advance(by: .milliseconds(250)) + // Activity during the wait defers the announcement by one more threshold. + await clock.waitForSleep(.milliseconds(250)) + #expect(notices.isEmpty) + clock.advance(by: .milliseconds(250)) + await attempted.wait(for: 1) + // The channel was unavailable: the notice is retained, not lost. + #expect(notices.isEmpty && notifier.hasPendingNotice && clock.pendingCount == 0) + notifier.flush() + #expect(notifier.hasPendingNotice && attempted.value == 2) + available = true + notifier.flush() + #expect(notices == [2] && !notifier.hasPendingNotice && resubmitted.value == 0) + // A dropped last frame is re-offered once; the announced content is the capture's. + #expect(monitor.recordSubmission(timestampNs: 5, nowNs: nanoseconds(clock, since: origin))) + notifier.activate() + await clock.waitForSleep(.milliseconds(500), count: 2) + clock.advance(by: .milliseconds(500)) + await resubmitted.wait(for: 1) + await clock.waitForSleep(.milliseconds(500), count: 3) + #expect(notices == [2]) + monitor.recordEncoded(timestampNs: 5) + clock.advance(by: .milliseconds(500)) + // Attempts so far: unavailable, flush while unavailable, flush, this notice. + await attempted.wait(for: 4) + #expect(notices == [2, 5] && clock.pendingCount == 0) + // Persistent failure keeps re-offering at the slow rate, never announcing stale content. + #expect(monitor.recordSubmission(timestampNs: 8, nowNs: nanoseconds(clock, since: origin))) + notifier.activate() + for attempt in 1...ScreenSharingSourceIdleMonitor.maximumQuickResubmissions { + await clock.waitForSleep(.milliseconds(500), count: 3 + attempt) + clock.advance(by: .milliseconds(500)) + await resubmitted.wait(for: 1 + attempt) + } + await clock.waitForSleep(.seconds(2)) + clock.advance(by: .seconds(2)) + await resubmitted.wait(for: 2 + ScreenSharingSourceIdleMonitor.maximumQuickResubmissions) + await clock.waitForSleep(.seconds(2), count: 2) + #expect(notices == [2, 5] && !notifier.hasPendingNotice) + monitor.recordEncoded(timestampNs: 8) + clock.advance(by: .seconds(2)) + await attempted.wait(for: 5) + #expect(notices == [2, 5, 8] && clock.pendingCount == 0) + #expect(monitor.recordSubmission(timestampNs: 9, nowNs: nanoseconds(clock, since: origin))) + notifier.activate() + await clock.waitForSleep(.milliseconds(500), count: 4 + ScreenSharingSourceIdleMonitor.maximumQuickResubmissions) + let pending = notifier.close() + await pending?.value + #expect(clock.pendingCount == 0) + notifier.activate() + notifier.flush() + clock.advance(by: .seconds(5)) + #expect(notices == [2, 5, 8] && attempted.value == 5) + } + + @Test func auditKeepsItsTargetThroughRecoveryUntilTheAnnouncedContentIsDecoded() { + let audit = ScreenSharingDeliveryAudit() + #expect(!audit.verify() && audit.recoveryFinished() == nil) + audit.decoded(sourceTimestampNs: 10) + audit.decoded(sourceTimestampNs: 9) + #expect(audit.latestDecodedTimestampNs == 10) + #expect(audit.noticed(latestTimestampNs: 10) && audit.noticed(latestTimestampNs: 9)) + #expect(audit.pendingTargetNs == nil) + // Target 20 announced; recovery completes with a delayed older keyframe. + #expect(!audit.noticed(latestTimestampNs: 20)) + #expect(audit.verify()) + #expect(!audit.verify()) + audit.decoded(sourceTimestampNs: 12) + #expect(audit.recoveryFinished() == true) + #expect(audit.pendingTargetNs == 20) + audit.decoded(sourceTimestampNs: 21) + #expect(audit.recoveryFinished() == false) + #expect(audit.pendingTargetNs == nil && audit.recoveryFinished() == nil) + // Content arriving during the grace period needs no recovery. + #expect(!audit.noticed(latestTimestampNs: 30)) + audit.decoded(sourceTimestampNs: 30) + #expect(!audit.verify() && audit.recoveryFinished() == nil) + // A newer notice replaces the target; a retry checks it is still missing. + #expect(!audit.noticed(latestTimestampNs: 40)) + #expect(!audit.noticed(latestTimestampNs: 41) && audit.pendingTargetNs == 41) + #expect(audit.verify()) + #expect(audit.targetMissing()) + audit.decoded(sourceTimestampNs: 41) + #expect(!audit.targetMissing()) + #expect(audit.pendingTargetNs == nil && audit.recoveryFinished() == nil && !audit.targetMissing()) + } + + @Test func retryDelaysGrowFromTheRefreshIntervalToACap() { + let delays = (1...8).map { ScreenSharingDeliveryVerifier.retryDelay($0) } + #expect(delays == [100, 200, 400, 800, 1600, 3200, 5000, 5000].map { Duration.milliseconds($0) }) + #expect(ScreenSharingDeliveryVerifier.retryDelay(0) == .milliseconds(100)) + #expect(ScreenSharingDeliveryVerifier.retryDelay(60) == .seconds(5)) + } + + @Test func auditRecordsWhenThePendingTargetIsFirstDecodedOnTheViewerClock() { + let audit = ScreenSharingDeliveryAudit() + audit.decoded(sourceTimestampNs: 10, nowNs: 100) + #expect(audit.targetMetAtTimestampNs == nil) + #expect(!audit.noticed(latestTimestampNs: 20)) + #expect(audit.verify()) + audit.decoded(sourceTimestampNs: 15, nowNs: 200) + #expect(audit.targetMetAtTimestampNs == nil) + audit.decoded(sourceTimestampNs: 20, nowNs: 300) + audit.decoded(sourceTimestampNs: 21, nowNs: 400) + #expect(audit.targetMetAtTimestampNs == 300) + #expect(audit.recoveryFinished() == false) + // A new notice starts a fresh measurement; content already present keeps + // its first-decode time from the bounded ring. + #expect(audit.noticed(latestTimestampNs: 21)) + #expect(audit.targetMetAtTimestampNs == 400) + #expect(!audit.noticed(latestTimestampNs: 30)) + audit.decoded(sourceTimestampNs: 30, nowNs: 500) + #expect(audit.targetMetAtTimestampNs == 500 && !audit.verify()) + // Content decoded before its notice keeps its first-decode time from the bounded ring. + audit.decoded(sourceTimestampNs: 40, nowNs: 600) + audit.decoded(sourceTimestampNs: 41, nowNs: 700) + #expect(audit.noticed(latestTimestampNs: 40) && audit.targetMetAtTimestampNs == 600) + #expect(audit.noticed(latestTimestampNs: 41) && audit.targetMetAtTimestampNs == 700) + let ring = ScreenSharingDeliveryAudit() + for index in 0..<(ScreenSharingDeliveryAudit.recentDecodeCapacity + 1) { + ring.decoded(sourceTimestampNs: Int64(index + 1), nowNs: Int64(index + 1) * 10) + } + #expect(ring.noticed(latestTimestampNs: 1) && ring.targetMetAtTimestampNs == 20) + #expect(ring.noticed(latestTimestampNs: 2) && ring.targetMetAtTimestampNs == 20) + } + + @Test @MainActor func verifierRefreshesAfterGraceAndKeepsRetryingWithBackoffUntilTheContentArrives() async { + let clock = TestClock() + let audit = ScreenSharingDeliveryAudit() + let settled = TestSignal() + let refreshed = TestSignal() + var outcomes: [ScreenSharingDeliveryVerifier.Outcome] = [] + let verifier = ScreenSharingDeliveryVerifier( + audit: audit, grace: .milliseconds(500), + refresh: { refreshed.signal() }, + report: { + outcomes.append($0) + settled.signal() + }, + sleep: { try await clock.sleep(for: $0) }) + verifier.noticed(latestTimestampNs: 5) + await clock.waitForSleep(.milliseconds(500)) + audit.decoded(sourceTimestampNs: 5) + clock.advance(by: .milliseconds(500)) + await settled.wait(for: 1) + #expect(outcomes == [.verifiedAfterGrace] && refreshed.value == 0) + verifier.noticed(latestTimestampNs: 5) + #expect(outcomes == [.verifiedAfterGrace, .verified]) + verifier.noticed(latestTimestampNs: 6) + await clock.waitForSleep(.milliseconds(500), count: 2) + // A newer notice replaces the pending one and restarts its grace period. + verifier.noticed(latestTimestampNs: 20) + await clock.waitForSleep(.milliseconds(500), count: 3) + #expect(clock.pendingCount == 1) + clock.advance(by: .milliseconds(500)) + await settled.wait(for: 3) + #expect(outcomes.last == .refresh && refreshed.value == 1) + // Recovery completes with a keyframe marked 10: the target 20 stays pending + // and a retry follows the first backoff delay. + audit.decoded(sourceTimestampNs: 10) + verifier.recoveryChanged(.init(revision: 1, needed: true)) + verifier.recoveryChanged(.init(revision: 2, needed: false)) + #expect(outcomes.last == .retry && audit.pendingTargetNs == 20) + verifier.recoveryChanged(.init(revision: 2, needed: false)) + await clock.waitForSleep(.milliseconds(100)) + #expect(refreshed.value == 1) + clock.advance(by: .milliseconds(100)) + await refreshed.wait(for: 2) + // An executed retry is reported distinctly from its scheduling. + #expect(outcomes.suffix(2) == [.retry, .retryExecuted]) + // A second stale completion waits twice as long; content arriving during + // the delay cancels the retry (scheduled, never executed). + verifier.recoveryChanged(.init(revision: 3, needed: false)) + await clock.waitForSleep(.milliseconds(200)) + audit.decoded(sourceTimestampNs: 20) + clock.advance(by: .milliseconds(200)) + await settled.wait(for: 7) + #expect(outcomes.last == .recovered && refreshed.value == 2 && audit.pendingTargetNs == nil) + #expect(outcomes.filter { $0 == .retryExecuted }.count == 1) + verifier.recoveryChanged(.init(revision: 4, needed: false)) + #expect(outcomes.count == 7) + // Retries never stop while the content is missing; the delay caps at 5 s. + verifier.noticed(latestTimestampNs: 30) + await clock.waitForSleep(.milliseconds(500), count: 4) + clock.advance(by: .milliseconds(500)) + await refreshed.wait(for: 3) + var revision: UInt64 = 5 + for retry in 1...9 { + verifier.recoveryChanged(.init(revision: revision, needed: false)) + revision += 1 + let delay = ScreenSharingDeliveryVerifier.retryDelay(retry) + await clock.waitForSleep(delay, count: retry >= 7 ? retry - 6 : 1) + clock.advance(by: delay) + await refreshed.wait(for: 3 + retry) + } + #expect(outcomes.filter { $0 == .retry }.count == 11 && audit.pendingTargetNs == 30) + #expect(outcomes.filter { $0 == .retryExecuted }.count == 10) + // Closing cancels a pending retry; nothing runs afterwards. + verifier.recoveryChanged(.init(revision: revision, needed: false)) + await clock.waitForSleep(.seconds(5), count: 4) + let pending = verifier.close() + await pending?.value + #expect(clock.pendingCount == 0) + verifier.noticed(latestTimestampNs: 41) + verifier.recoveryChanged(.init(revision: revision + 1, needed: false)) + clock.advance(by: .seconds(30)) + #expect(refreshed.value == 12) + } + + @Test func monitorSlowPhaseNeverReoffersFasterThanTheThreshold() { + let monitor = ScreenSharingSourceIdleMonitor(thresholdNs: 3_000_000_000, slowIntervalNs: 2_000_000_000) + #expect(monitor.slowResubmissionIntervalNs == 3_000_000_000) + #expect(monitor.recordSubmission(timestampNs: 1, nowNs: 0)) + var now: Int64 = 0 + for _ in 0.. String { + evaluated += 1 + return "event" + } + metrics.trace("boundary", entry()) + #expect(evaluated == 0 && metrics.snapshot().traces == nil) + metrics.enableTracing() + for _ in 0..<70 { metrics.trace("boundary", entry()) } + #expect(evaluated == 70) + #expect(metrics.snapshot().traces?["boundary"]?.count == 64) + } + + @Test func shortfallRequestDoesNotInvalidateAPendingRecovery() { + let signal = ScreenSharingRefreshSignal() + let metrics = ScreenSharingMetrics() + signal.onChange { event in metrics.increment(event.needed ? "requested" : "recovered") } + signal.request() + let generation = signal.keyframeGeneration + #expect(!signal.requestUnlessPending()) + #expect(signal.keyframeGeneration == generation) + signal.decodedKeyframe(generation: generation) + #expect(metrics.snapshot().counters["recovered"] == 1) + #expect(signal.requestUnlessPending()) + #expect(signal.keyframeGeneration == generation + 1) + #expect(metrics.snapshot().counters["requested"] == 2) + signal.close() + #expect(!signal.requestUnlessPending()) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceMarkerTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceMarkerTests.swift new file mode 100644 index 000000000..053f258ff --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingSourceMarkerTests.swift @@ -0,0 +1,102 @@ +import CoreVideo +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingSourceMarkerTests { + private let sps = Data([0x67, 0x42, 0xe0, 0x34]) + private let pps = Data([0x68, 0xce, 0x3c, 0x80]) + private let slice = Data([0x65, 0x88, 0x84, 0x00, 0x00, 0x03, 0x01]) + + @Test func markerRoundTripsForEachCodecWithoutStartCodeEmulation() { + for codec in ScreenSharingVideoCodec.allCases { + for value: Int64 in [0, 1, 127, 128, 1_726_100_000_123_456_789, Int64.max] { + let unit = ScreenSharingSourceMarker.nalUnit(timestampNs: value, codec: codec) + #expect(!unit.contains(0)) + #expect(unit.count == (codec == .h264 ? 30 : 31)) + let type = codec == .h264 ? Int(unit[0] & 0x1f) : Int((unit[0] >> 1) & 0x3f) + #expect(type == (codec == .h264 ? 6 : 39)) + // Neither decoder filter (slices, parameter sets) may select the marker. + #expect(!(codec == .h264 ? 1...5 : 0...31).contains(type) && ![7, 8, 32, 33, 34].contains(type)) + #expect(codec == .h264 || (unit[0] & 0x80 == 0 && unit[1] & 7 != 0)) + #expect(ScreenSharingSourceMarker.isMarker(unit, codec: codec)) + #expect(ScreenSharingSourceMarker.timestampNs(in: [sps, pps, unit, slice], codec: codec) == value) + } + } + } + + @Test func markerSurvivesAnnexBFramingAndIsStrippedFromDecoderInput() throws { + let marker = ScreenSharingSourceMarker.nalUnit(timestampNs: 42, codec: .h264) + var frame = Data() + for unit in [sps, pps, marker, slice] { + frame.append(contentsOf: [0, 0, 0, 1]) + frame.append(unit) + } + let units = try NALUnitBitstream.nalUnits(frame) + #expect(units.count == 4) + #expect(ScreenSharingSourceMarker.timestampNs(in: units, codec: .h264) == 42) + let converted = try NALUnitBitstream.annexB(NALUnitBitstream.lengthPrefixed(units)) + #expect(ScreenSharingSourceMarker.timestampNs(in: try NALUnitBitstream.nalUnits(converted), codec: .h264) == 42) + let forVideoToolbox = units.filter { !ScreenSharingSourceMarker.isMarker($0, codec: .h264) } + #expect(forVideoToolbox == [sps, pps, slice]) + } + + @Test func absentForeignOrMalformedMarkersAreIgnoredButOwnMalformedMarkersAreStillStripped() { + let marker = ScreenSharingSourceMarker.nalUnit(timestampNs: 99, codec: .h264) + var foreignIdentifier = marker + foreignIdentifier[5] ^= 0x01 + var corruptValue = marker + corruptValue[marker.count - 2] &= 0x7f + var nonCanonicalLeadingBits = marker + nonCanonicalLeadingBits[marker.count - 11] = 0x81 + var wrongTrailer = marker + wrongTrailer[marker.count - 1] = 0x00 + let encoderVersionSei = Data([0x06, 0x05] + [0x10] + Array(repeating: UInt8(0xaa), count: 16) + [0x80]) + let hevcMarker = ScreenSharingSourceMarker.nalUnit(timestampNs: 99, codec: .hevc) + for units: [Data] in [ + [], [sps, pps, slice], [encoderVersionSei], [foreignIdentifier], [corruptValue], [nonCanonicalLeadingBits], + [wrongTrailer], [marker.dropLast()], [marker + Data([0x80])], [hevcMarker], + ] { + #expect(ScreenSharingSourceMarker.timestampNs(in: units, codec: .h264) == nil) + } + #expect(ScreenSharingSourceMarker.timestampNs(in: [marker], codec: .hevc) == nil) + #expect(ScreenSharingSourceMarker.timestampNs(in: [corruptValue, marker], codec: .h264) == 99) + for own in [corruptValue, nonCanonicalLeadingBits, wrongTrailer, marker.dropLast(), marker + Data([0x80])] { + #expect(ScreenSharingSourceMarker.isMarker(own, codec: .h264)) + } + for foreign in [encoderVersionSei, foreignIdentifier, hevcMarker, sps, slice, Data()] { + #expect(!ScreenSharingSourceMarker.isMarker(foreign, codec: .h264)) + } + } + + @Test func identityTravelsWithTheBufferIndependentlyOfTranslatedTimestamps() throws { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + let buffer = try #require(pixel) + #expect(ScreenSharingFrameIdentity.sourceTimestampNs(of: buffer) == nil) + // Non-microsecond capture time; WebRTC would truncate and translate it. + ScreenSharingFrameIdentity.attach(sourceTimestampNs: 1_726_100_000_123_456_789, to: buffer) + let translated = ScreenSharingVideoFrame(pixelBuffer: buffer, timestampNs: 1_726_100_000_130_000_000) + #expect(ScreenSharingFrameIdentity.sourceTimestampNs(of: translated.pixelBuffer) == 1_726_100_000_123_456_789) + // A recycled buffer receives the new capture's identity. + ScreenSharingFrameIdentity.attach(sourceTimestampNs: 7, to: buffer) + #expect(ScreenSharingFrameIdentity.sourceTimestampNs(of: buffer) == 7) + } + + @Test func transportEncoderRefusesInputWithoutIdentityThroughTheEncoderFailureLifecycle() throws { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + let buffer = try #require(pixel) + let metrics = ScreenSharingMetrics() + #expect(ScreenSharingFrameIdentity.required(of: buffer, metrics: metrics) == nil) + let failed = metrics.snapshot() + #expect(failed.counters["encoderInputWithoutIdentity"] == 1) + #expect(failed.labels["encoderError"] == "Encoder input carries no content identity.") + let healthy = ScreenSharingMetrics() + ScreenSharingFrameIdentity.attach(sourceTimestampNs: 42, to: buffer) + #expect(ScreenSharingFrameIdentity.required(of: buffer, metrics: healthy) == 42) + #expect( + healthy.snapshot().counters["encoderInputWithoutIdentity"] == nil + && healthy.snapshot().labels["encoderError"] == nil) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingVideoRefreshTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingVideoRefreshTests.swift new file mode 100644 index 000000000..0d282ff11 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingVideoRefreshTests.swift @@ -0,0 +1,177 @@ +import CodevisorTestSupport +import CoreVideo +import Foundation +import Testing +@testable import CodevisorScreenSharing + +struct ScreenSharingVideoRefreshTests { + @Test func refreshProtocolRejectsUnknownAndTrailingBytes() throws { + #expect(try ScreenSharingVideoRefreshMessage.decode(Data([1])).encoded() == Data([1])) + for bytes in [Data(), Data([0]), Data([2]), Data([1, 1])] { + #expect(throws: (any Error).self) { try ScreenSharingVideoRefreshMessage.decode(bytes) } + } + } + + @Test func failedDecodeCoalescesNotificationsAndRejectsOldKeyframeCompletion() { + let signal = ScreenSharingRefreshSignal() + let metrics = ScreenSharingMetrics() + signal.onChange { event in + // Reentry verifies notifications do not hold the signal's lock. + _ = signal.keyframeGeneration + metrics.increment(event.needed ? "requested" : "recovered") + } + signal.request() + let old = signal.keyframeGeneration + signal.request(resetDecoder: true) + #expect(metrics.snapshot().counters["requested"] == 1) + signal.decodedKeyframe(generation: old) + #expect(metrics.snapshot().counters["recovered"] == nil) + // A callback cannot acknowledge recovery before the invalid session has + // been handed back to the decoder queue for reset. + signal.decodedKeyframe(generation: signal.keyframeGeneration) + #expect(metrics.snapshot().counters["recovered"] == nil) + #expect(signal.consumeDecoderReset()) + #expect(!signal.consumeDecoderReset()) + signal.decodedKeyframe(generation: signal.keyframeGeneration) + #expect(metrics.snapshot().counters["recovered"] == 1) + signal.close() + signal.request() + #expect(metrics.snapshot().counters["requested"] == 1) + } + + @Test func subscriptionReceivesPendingRequestAndCannotReopenClosedSignal() { + let signal = ScreenSharingRefreshSignal() + let metrics = ScreenSharingMetrics() + signal.request() + signal.onChange { _ in metrics.increment("events") } + #expect(metrics.snapshot().counters["events"] == 1) + signal.close() + signal.onChange { _ in metrics.increment("events") } + signal.request() + #expect(metrics.snapshot().counters["events"] == 1) + } + + @Test func externalEncoderRequestsCoalesceUntilConsumed() { + let request = ScreenSharingEncoderRefreshRequest() + #expect(!request.consume()) + request.request() + request.request() + #expect(request.consume()) + #expect(!request.consume()) + } + + @Test func refreshRateIsBoundedAcrossBurstsAndClockRegression() { + var limit = ScreenSharingRefreshRateLimit() + for (time, expected) in [ + (-1, false), (0, true), (0, false), (99_999_999, false), + (100_000_000, true), (99_000_000, false), (199_999_999, false), (200_000_000, true), + ] { + let allowed = limit.allow(nowNs: Int64(time)) + #expect(allowed == expected) + } + } + + @Test @MainActor func recoveryRetriesOnlyUntilCurrentKeyframeAndCancelsOnClose() async { + let clock = TestClock() + let origin = clock.now + var available = false + var sends = 0 + let requester = ScreenSharingRefreshRequester( + available: { available }, + send: { + sends += 1; return true + }, + nowNs: { + let parts = origin.duration(to: clock.now).components + return parts.seconds * 1_000_000_000 + parts.attoseconds / 1_000_000_000 + }, sleep: { try await clock.sleep(for: $0) }) + requester.update(.init(revision: 1, needed: true)) + await clock.waitForSleep(.milliseconds(100)) + #expect(sends == 0) + available = true + requester.wake() + requester.wake() + #expect(sends == 1) + clock.advance(by: .milliseconds(99)) + requester.wake() + #expect(sends == 1) + clock.advance(by: .milliseconds(1)) + await clock.waitForSleep(.milliseconds(100), count: 2) + #expect(sends == 2) + requester.update(.init(revision: 2, needed: false)) + #expect(clock.pendingCount == 0) + requester.update(.init(revision: 1, needed: true)) + clock.advance(by: .seconds(10)) + requester.wake() + #expect(sends == 2) + requester.update(.init(revision: 3, needed: true)) + await clock.waitForSleep(.milliseconds(100), count: 3) + #expect(sends == 3) + let pending = requester.close() + await pending?.value + #expect(clock.pendingCount == 0) + requester.update(.init(revision: 4, needed: true)) + requester.wake() + #expect(sends == 3) + } + + @Test func cachedRefreshUsesLatestBufferAndStrictlyIncreasingTimestamps() throws { + var pixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) + let buffer = try #require(pixel) + var store = ScreenSharingRefreshFrameStore() + let empty = store.refresh(nowNs: 1) + #expect(empty == nil) + let captured = store.capture(.init(pixelBuffer: buffer, timestampNs: 10)) + #expect(captured?.timestampNs == 10 && captured?.sourceTimestampNs == 10) + let refreshed = store.refresh(nowNs: 5) + #expect(refreshed?.timestampNs == 11 && refreshed?.sourceTimestampNs == 10) + #expect(refreshed?.pixelBuffer === buffer) + let duplicate = store.capture(.init(pixelBuffer: buffer, timestampNs: 10)) + #expect(duplicate == nil) + var nextPixel: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 32, 16, kCVPixelFormatType_32BGRA, nil, &nextPixel) == kCVReturnSuccess) + let next = try #require(nextPixel) + let replacement = store.capture(.init(pixelBuffer: next, timestampNs: 20)) + #expect(replacement?.timestampNs == 20 && replacement?.sourceTimestampNs == 20) + let latest = store.refresh(nowNs: 30) + #expect(latest?.timestampNs == 30 && latest?.sourceTimestampNs == 20) + #expect(latest?.pixelBuffer === next) + store.clear() + let cleared = store.refresh(nowNs: 40) + #expect(cleared == nil) + let outOfOrder = store.capture(.init(pixelBuffer: buffer, timestampNs: 19)) + #expect(outOfOrder == nil) + let maximum = store.capture(.init(pixelBuffer: buffer, timestampNs: Int64.max)) + #expect(maximum?.timestampNs == Int64.max) + let overflow = store.refresh(nowNs: Int64.max) + #expect(overflow == nil) + let unsubmittable = store.capture(.init(pixelBuffer: buffer, timestampNs: Int64.max)) + #expect(unsubmittable == nil) + } + + @Test func delayedNewerCaptureReplacesARefreshedOlderFrameWithMonotonicSubmission() throws { + var first: CVPixelBuffer? + var second: CVPixelBuffer? + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &first) == kCVReturnSuccess) + #expect(CVPixelBufferCreate(nil, 16, 16, kCVPixelFormatType_32BGRA, nil, &second) == kCVReturnSuccess) + let a = try #require(first) + let b = try #require(second) + var store = ScreenSharingRefreshFrameStore() + #expect(store.capture(.init(pixelBuffer: a, timestampNs: 10))?.timestampNs == 10) + #expect(store.refresh(nowNs: 30)?.timestampNs == 30) + // Capture B (PTS 20) was produced before the refresh but delivered late. + let late = store.capture(.init(pixelBuffer: b, timestampNs: 20)) + #expect(late?.pixelBuffer === b && late?.timestampNs == 31 && late?.sourceTimestampNs == 20) + let refreshedB = store.refresh(nowNs: 25) + #expect(refreshedB?.pixelBuffer === b && refreshedB?.timestampNs == 32 && refreshedB?.sourceTimestampNs == 20) + // Actual duplicates and out-of-order captures remain rejected. + #expect(store.capture(.init(pixelBuffer: b, timestampNs: 20)) == nil) + #expect(store.capture(.init(pixelBuffer: a, timestampNs: 15)) == nil) + #expect(store.capture(.init(pixelBuffer: a, timestampNs: -1)) == nil) + let c = store.capture(.init(pixelBuffer: a, timestampNs: 21)) + #expect(c?.timestampNs == 33 && c?.sourceTimestampNs == 21) + let ordinary = store.capture(.init(pixelBuffer: b, timestampNs: 100)) + #expect(ordinary?.timestampNs == 100 && ordinary?.sourceTimestampNs == 100) + } +} diff --git a/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingWorkloadSequenceTests.swift b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingWorkloadSequenceTests.swift new file mode 100644 index 000000000..71d60f394 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Tests/CodevisorScreenSharingTests/ScreenSharingWorkloadSequenceTests.swift @@ -0,0 +1,44 @@ +import Testing + +@testable import CodevisorScreenSharing + +@Suite struct ScreenSharingWorkloadSequenceTests { + @Test func drawsFollowTimeUntilFrozenAtTheLastDrawnCodeThenNeverAdvance() { + var sequence = ScreenSharingWorkloadSequence(framesPerSecond: 60, startedAtSeconds: 100) + #expect(sequence.drawn(atSeconds: 100) == 0) + #expect(sequence.drawn(atSeconds: 100.5) == 30) + #expect(sequence.drawn(atSeconds: 101) == 60) + #expect(!sequence.isFrozen && sequence.lastDrawnCode == 60) + sequence.freezeAtLastDrawn(atSeconds: 101.25) + #expect(sequence.isFrozen && sequence.frozenCode == 60 && sequence.frozenAtSeconds == 101.25) + // Incidental redraws after the pause (window server requests, later + // times, even earlier times) render and record the frozen code. + #expect(sequence.drawn(atSeconds: 105) == 60) + #expect(sequence.drawn(atSeconds: 130) == 60) + #expect(sequence.drawn(atSeconds: 100) == 60) + #expect(sequence.lastDrawnCode == 60) + // Freezing again is idempotent: the first boundary wins. + sequence.freezeAtLastDrawn(atSeconds: 140) + #expect(sequence.frozenCode == 60 && sequence.frozenAtSeconds == 101.25) + } + + @Test func pauseBetweenFramesFreezesWhatWasDrawnNotWhatTimeWouldSay() { + var sequence = ScreenSharingWorkloadSequence(framesPerSecond: 60, startedAtSeconds: 100) + #expect(sequence.drawn(atSeconds: 100.5) == 30) + // The pause lands 29 frame periods later without any draw in between: a + // time-derived code would be 59; the frozen code must be the drawn 30. + sequence.freezeAtLastDrawn(atSeconds: 100.99) + #expect(sequence.frozenCode == 30) + #expect(sequence.drawn(atSeconds: 100.99) == 30) + #expect(sequence.drawn(atSeconds: 120) == 30) + } + + @Test func timesBeforeStartClampToZeroAndFreezeBeforeAnyDrawHoldsZero() { + var sequence = ScreenSharingWorkloadSequence(framesPerSecond: 30, startedAtSeconds: 50) + #expect(sequence.drawn(atSeconds: 49) == 0) + var undrawn = ScreenSharingWorkloadSequence(framesPerSecond: 30, startedAtSeconds: 50) + undrawn.freezeAtLastDrawn(atSeconds: 49.5) + #expect(undrawn.frozenCode == 0 && undrawn.lastDrawnCode == nil) + #expect(undrawn.drawn(atSeconds: 60) == 0) + } +} diff --git a/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentFailure.swift b/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentFailure.swift index 7f4079a48..965011001 100644 --- a/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentFailure.swift +++ b/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentFailure.swift @@ -59,3 +59,11 @@ struct MarkdownDocumentFailure { } } } + +/// Raised when a document pane has no chat session to fetch through. +public struct MarkdownDocumentUnavailable: LocalizedError { + public init() {} + public var errorDescription: String? { + "This document needs a chat in this workspace before it can be opened." + } +} diff --git a/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentModel.swift b/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentModel.swift index 951c28590..7fac6884c 100644 --- a/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentModel.swift +++ b/packages/swift/CodevisorUI/Sources/CodevisorUI/MarkdownDocumentModel.swift @@ -28,6 +28,14 @@ public final class MarkdownDocumentModel { self.init(path: path, fetch: { try await client.fileData(sessionId: sessionId, path: path) }) } + /// A document with no chat session to read files through — a pane group whose + /// identity comes from its workspace rather than a chat. The path still + /// renders; loading reports it as unavailable instead of fetching under a + /// substituted session id. + public convenience init(unavailablePath path: String) { + self.init(path: path, fetch: { throw MarkdownDocumentUnavailable() }) + } + init( path: String, fetch: @escaping @Sendable () async throws -> Data, diff --git a/packages/swift/Package.resolved b/packages/swift/Package.resolved index 1fa6f0345..4e1dba330 100644 --- a/packages/swift/Package.resolved +++ b/packages/swift/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "fa971cb701882a40340f7655997230e7b0c7030bd3850a608d53e07e1750e293", + "originHash" : "afc68d4af172302607d270ff6a68499115a105bf12203044665dbfb4fac63673", "pins" : [ { "identity" : "posthog-ios", @@ -18,6 +18,15 @@ "revision" : "afa7510e05b99f35c1febe47566bfd868fa53fb9", "version" : "9.23.0" } + }, + { + "identity" : "webrtc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/stasel/WebRTC.git", + "state" : { + "revision" : "1d04692697cb642bfebf6ad2dd99fe52649c3d6d", + "version" : "152.0.0" + } } ], "version" : 3 diff --git a/packages/swift/Package.swift b/packages/swift/Package.swift index 5a0914eaa..14d3bb70d 100644 --- a/packages/swift/Package.swift +++ b/packages/swift/Package.swift @@ -25,12 +25,38 @@ let package = Package( .library(name: "CodevisorCoreMac", targets: ["CodevisorCoreMac"]), .library(name: "CodevisorUI", targets: ["CodevisorUI"]), .library(name: "Autocomplete", targets: ["Autocomplete"]), + .library(name: "CodevisorScreenSharing", targets: ["CodevisorScreenSharing"]), + .executable(name: "screen-sharing-probe", targets: ["ScreenSharingProbe"]), ], dependencies: [ .package(url: "https://github.com/PostHog/posthog-ios.git", exact: "3.59.3"), .package(url: "https://github.com/getsentry/sentry-cocoa.git", exact: "9.23.0"), + .package(url: "https://github.com/stasel/WebRTC.git", exact: "152.0.0"), ], targets: [ + .target( + name: "CodevisorScreenSharing", + dependencies: [.product(name: "WebRTC", package: "WebRTC")], + path: "CodevisorScreenSharing/Sources/CodevisorScreenSharing", + resources: [.copy("Resources/WebRTC-LICENSE.txt")], + swiftSettings: [.swiftLanguageMode(.v6)] + ), + .executableTarget( + name: "ScreenSharingProbe", + dependencies: ["CodevisorScreenSharing", "CodevisorClient"], + path: "CodevisorScreenSharing/Probe", + swiftSettings: [.swiftLanguageMode(.v6)] + ), + .testTarget( + name: "CodevisorScreenSharingTests", + dependencies: ["CodevisorScreenSharing", "CodevisorTestSupport"], + path: "CodevisorScreenSharing/Tests/CodevisorScreenSharingTests", + swiftSettings: [.swiftLanguageMode(.v6)], + // SwiftPM's macOS test bundle loader needs the sibling binary framework. + linkerSettings: [ + .unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@loader_path/../../.."], .when(platforms: [.macOS])) + ] + ), .target(name: "CodevisorTestSupport", path: "TestSupport", swiftSettings: [.swiftLanguageMode(.v6)]), // MARK: CodevisorTheming (VSCode/Shiki theme parsing, normalization, // palette derivation — Foundation-only, no SwiftUI) @@ -230,7 +256,7 @@ let package = Package( // iOS apps depend on CodevisorCore only; never link this on iOS.) .target( name: "CodevisorCoreMac", - dependencies: ["CodevisorCore"], + dependencies: ["CodevisorCore", "CodevisorScreenSharing"], path: "CodevisorCoreMac/Sources/CodevisorCoreMac", swiftSettings: [.swiftLanguageMode(.v6)] ), @@ -291,7 +317,10 @@ let package = Package( "ACPKit", ], path: "CodevisorCoreMac/Tests/CodevisorCoreMacTests", - swiftSettings: [.swiftLanguageMode(.v6)] + swiftSettings: [.swiftLanguageMode(.v6)], + linkerSettings: [ + .unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@loader_path/../../.."], .when(platforms: [.macOS])) + ] ), ] ) diff --git a/packages/swift/TestSupport/TestClock.swift b/packages/swift/TestSupport/TestClock.swift index 37a7825cc..0a01b2e66 100644 --- a/packages/swift/TestSupport/TestClock.swift +++ b/packages/swift/TestSupport/TestClock.swift @@ -22,6 +22,9 @@ public final class TestClock: @unchecked Sendable { public var pendingCount: Int { lock.withLock { pending.count } } + /// How many sleeps of exactly `duration` were ever requested (pending or resolved). + public func requestCount(_ duration: Duration) -> Int { lock.withLock { requests.filter { $0 == duration }.count } } + public func sleep(for duration: Duration) async throws { try Task.checkCancellation() if duration <= .zero { return } diff --git a/scripts/build-webrtc.mjs b/scripts/build-webrtc.mjs new file mode 100644 index 000000000..f5153600c --- /dev/null +++ b/scripts/build-webrtc.mjs @@ -0,0 +1,297 @@ +#!/usr/bin/env node +// An isolated, pinned source recipe. Does not change the reference submodule, +// installed SwiftPM artifact, Xcode selection, or any published release. +import { createHash } from "node:crypto" +import { spawnSync } from "node:child_process" +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync +} from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))) +const lockBytes = readFileSync(new URL("./webrtc-build.lock.json", import.meta.url)) +const lock = JSON.parse(lockBytes) +const recipeSHA256 = createHash("sha256") + .update(readFileSync(fileURLToPath(import.meta.url))) + .digest("hex") +const stamp = createHash("sha256").update(lockBytes).update(recipeSHA256).digest("hex").slice(0, 16) +const workspace = join(repoRoot, "tmp/webrtc-source", stamp) +const source = join(workspace, "src") +const depot = join(workspace, "depot_tools") +const output = join(workspace, "artifacts") +const framework = join(output, "WebRTC.xcframework") +const environment = { + ...process.env, + PATH: `${depot}:${process.env.PATH}`, + DEPOT_TOOLS_UPDATE: "0", + DEPOT_TOOLS_METRICS: "0" +} +const variants = [ + { name: "macos-arm64", cpu: "arm64", os: "mac", target: "mac_framework_objc" }, + { name: "macos-x64", cpu: "x64", os: "mac", target: "mac_framework_objc" }, + { + name: "ios-arm64-device", + cpu: "arm64", + os: "ios", + target: "framework_objc", + platform: "device" + }, + { + name: "ios-arm64-simulator", + cpu: "arm64", + os: "ios", + target: "framework_objc", + platform: "simulator" + }, + { + name: "ios-x64-simulator", + cpu: "x64", + os: "ios", + target: "framework_objc", + platform: "simulator" + } +] + +function run(command, args, cwd = workspace, capture = false) { + const result = spawnSync(command, args, { + cwd, + env: environment, + encoding: "utf8", + stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit", + maxBuffer: 32 * 1024 * 1024 + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`${command} failed (${result.status})`) + return result.stdout?.trim() +} + +function argumentsFor(variant) { + return [ + ...lock.gnArguments, + `target_cpu="${variant.cpu}"`, + `target_os="${variant.os}"`, + ...(variant.os === "ios" + ? [ + `target_environment="${variant.platform}"`, + 'ios_deployment_target="12.0"', + "ios_enable_code_signing=false" + ] + : []) + ].join(" ") +} + +function checkout(directory, url, revision) { + if (!existsSync(directory)) { + mkdirSync(directory) + run("git", ["init", "--quiet"], directory) + run("git", ["remote", "add", "origin", url], directory) + } + // Never discard a source edit when resuming a failed build. + if (existsSync(join(directory, ".git/HEAD"))) { + const status = run("git", ["status", "--porcelain", "--untracked-files=no"], directory, true) + if (status) throw new Error(`Source checkout has edits: ${directory}`) + } + if (run("git", ["remote", "get-url", "origin"], directory, true) !== url) { + throw new Error(`Unexpected source remote in ${directory}`) + } + run("git", ["fetch", "--depth=1", "origin", revision], directory) + run("git", ["checkout", "--detach", revision], directory) + const actual = run("git", ["rev-parse", "HEAD"], directory, true) + if (actual !== revision) throw new Error(`Unexpected source revision in ${directory}`) +} + +function notices(platform, target, builds) { + const directory = join(output, `notices-${platform}`) + mkdirSync(directory, { recursive: true }) + run( + "vpython3", + [ + join(source, "tools_webrtc/libs/generate_licenses.py"), + "--target", + `//sdk:${target}`, + directory, + ...builds.map((name) => join(source, "out", name)) + ], + source + ) + const text = readFileSync(join(directory, "LICENSE.md"), "utf8") + if (!text.includes("# webrtc") || text.length < 1000) + throw new Error("Generated notices are incomplete") + return text +} + +function stage(identifier, builds, licenseText) { + const directory = join(output, "staging", identifier) + mkdirSync(directory, { recursive: true }) + const staged = join(directory, "WebRTC.framework") + const dsym = join(directory, "WebRTC.dSYM") + const paths = builds.map((name) => join(source, "out", name)) + cpSync(join(paths[0], "WebRTC.framework"), staged, { recursive: true, verbatimSymlinks: true }) + cpSync(join(paths[0], "WebRTC.dSYM"), dsym, { recursive: true, verbatimSymlinks: true }) + const mac = identifier.startsWith("macos") + const resources = mac ? join(staged, "Versions/A/Resources") : staged + if (mac) { + // M152 leaves public macOS headers in gen/ and nests the privacy manifest. + const headers = join(paths[0], "gen/sdk/WebRTC.framework/Headers") + for (const file of readdirSync(headers).filter((file) => file.endsWith(".h"))) { + cpSync(join(headers, file), join(staged, "Versions/A/Headers", file)) + } + const nested = join(staged, "Versions/A/Versions") + const manifest = join(nested, "A/Resources/PrivacyInfo.xcprivacy") + if (existsSync(manifest)) { + cpSync(manifest, join(resources, "PrivacyInfo.xcprivacy")) + rmSync(nested, { recursive: true }) + } + } + if (!existsSync(join(resources, "PrivacyInfo.xcprivacy"))) + throw new Error(`Missing privacy manifest: ${identifier}`) + writeFileSync(join(resources, "WebRTC-ThirdPartyNotices.md"), licenseText) + if (paths.length > 1) { + run("lipo", [ + "-create", + ...paths.map((path) => join(path, "WebRTC.framework/WebRTC")), + "-output", + mac ? join(staged, "Versions/A/WebRTC") : join(staged, "WebRTC") + ]) + run("lipo", [ + "-create", + ...paths.map((path) => join(path, "WebRTC.dSYM/Contents/Resources/DWARF/WebRTC")), + "-output", + join(dsym, "Contents/Resources/DWARF/WebRTC") + ]) + for (const path of paths.slice(1)) { + const relocations = join(path, "WebRTC.dSYM/Contents/Resources/Relocations") + if (existsSync(relocations)) + cpSync(relocations, join(dsym, "Contents/Resources/Relocations"), { recursive: true }) + } + } + run("codesign", ["--force", "--sign", "-", staged]) + run("codesign", ["--verify", "--strict", staged]) + return ["-framework", staged, "-debug-symbols", dsym] +} + +function build() { + if (process.platform !== "darwin") throw new Error("The WebRTC artifact requires macOS") + // Validate tools before fetching gigabytes of sources. Respect DEVELOPER_DIR; + // this script never changes the user's global xcode-select setting. + const xcode = run("xcodebuild", ["-version"], repoRoot, true) + const python = run("python3", ["--version"], repoRoot, true) + if (!xcode.split("\n").includes(`Xcode ${lock.xcodeVersion}`)) { + throw new Error( + `Select Xcode ${lock.xcodeVersion} with DEVELOPER_DIR; found ${xcode.split("\n")[0]}` + ) + } + if (python !== `Python ${lock.pythonVersion}`) + throw new Error(`Python ${lock.pythonVersion} is required; found ${python}`) + mkdirSync(workspace, { recursive: true }) + const buildLock = join(workspace, ".building") + mkdirSync(buildLock) // Refuse concurrent builds of this exact artifact. + try { + if (existsSync(output)) + throw new Error( + `Preserve or remove the prior artifact directory before rebuilding: ${output}` + ) + checkout( + depot, + "https://chromium.googlesource.com/chromium/tools/depot_tools.git", + lock.depotToolsRevision + ) + checkout(source, "https://webrtc.googlesource.com/src.git", lock.sourceRevision) + writeFileSync( + join(workspace, ".gclient"), + `solutions = [{"name": "src", "url": "https://webrtc.googlesource.com/src.git", "managed": False, "deps_file": "DEPS", "custom_deps": {}}]\ntarget_os = ["ios", "mac"]\n` + ) + run("gclient", [ + "sync", + "--no-history", + "--nohooks", + "--revision", + `src@${lock.sourceRevision}` + ]) + run("gclient", ["runhooks"]) + mkdirSync(output) + writeFileSync( + join(output, "revisions.txt"), + run("gclient", ["revinfo", "--actual"], workspace, true) + "\n" + ) + for (const variant of variants) { + const directory = join(source, "out", variant.name) + run("gn", ["gen", directory, `--args=${argumentsFor(variant)}`], source) + writeFileSync( + join(output, `${variant.name}-gn-args.txt`), + run("gn", ["args", directory, "--list"], source, true) + "\n" + ) + run("ninja", ["-C", directory, variant.target], source) + } + const macNotices = notices("macos", "mac_framework_objc", ["macos-arm64", "macos-x64"]) + const iosNotices = notices("ios", "framework_objc", [ + "ios-arm64-device", + "ios-arm64-simulator", + "ios-x64-simulator" + ]) + const slices = [ + ...stage("macos-arm64_x86_64", ["macos-arm64", "macos-x64"], macNotices), + ...stage("ios-arm64", ["ios-arm64-device"], iosNotices), + ...stage( + "ios-arm64_x86_64-simulator", + ["ios-arm64-simulator", "ios-x64-simulator"], + iosNotices + ) + ] + run("xcodebuild", ["-create-xcframework", ...slices, "-output", framework]) + cpSync(join(source, "LICENSE"), join(framework, "LICENSE")) + cpSync(join(source, "PATENTS"), join(framework, "PATENTS")) + const archive = join(output, "WebRTC.xcframework.zip") + run("ditto", ["-c", "-k", "--sequesterRsrc", "--keepParent", framework, archive]) + writeFileSync( + join(output, "manifest.json"), + JSON.stringify( + { + ...lock, + recipeStamp: stamp, + recipeSHA256, + xcode, + python, + sha256: createHash("sha256").update(readFileSync(archive)).digest("hex"), + archive: "WebRTC.xcframework.zip" + }, + null, + 2 + ) + "\n" + ) + process.stdout.write( + `Artifact and source records: ${output}\nNo package pin or published artifact was changed.\n` + ) + } finally { + rmSync(buildLock, { recursive: true }) + } +} + +const args = process.argv.slice(2) +if (args.length === 1 && args[0] === "--plan") { + process.stdout.write( + JSON.stringify( + { + ...lock, + workspace, + output, + builds: variants.map((variant) => + Object.assign({}, variant, { gnArguments: argumentsFor(variant) }) + ) + }, + null, + 2 + ) + ) +} else if (args.length === 0) { + build() +} else { + throw new Error("Usage: node scripts/build-webrtc.mjs [--plan]") +} diff --git a/scripts/dev-instance.mjs b/scripts/dev-instance.mjs index fb9c01724..795c884a5 100644 --- a/scripts/dev-instance.mjs +++ b/scripts/dev-instance.mjs @@ -39,6 +39,8 @@ const ambientAllowlist = new Set([ "CODEVISOR_PLUGINS_ROOT", "CODEVISOR_GHOSTTY_ARTIFACTS_ROOT", "CODEVISOR_GHOSTTY_ARTIFACT_ORIGIN", + // Explicit, default-off native diagnostic; the app validates the value. + "CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE", "CODEVISOR_IOS_SIMULATOR", "CODEVISOR_VERSION", "HERDMAN_VERSION" diff --git a/scripts/dev-instance.test.mjs b/scripts/dev-instance.test.mjs new file mode 100644 index 000000000..8275365bb --- /dev/null +++ b/scripts/dev-instance.test.mjs @@ -0,0 +1,40 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import { sanitizeAmbientEnvironment } from "./dev-instance.mjs" +import { developmentLayout, localDevelopmentEnvironment } from "./dev-layout.mjs" + +const profileKey = "CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE" + +test("native diagnostic selection survives dev sanitization without inheriting app-hosted state", () => { + // Preserve unknown spellings too: native validation must reject them visibly + // instead of the runner silently turning the requested profile off. + for (const selection of ["paced15-worker", "", "unknown-profile"]) { + const supplied = { + [profileKey]: selection, + CODEVISOR_APP_HOSTED: "1", + CODEVISOR_APP_BUNDLE_PATH: "/Applications/Other Codevisor.app", + HERDMAN_APP_HOSTED: "1", + CODEVISOR_DEV_PORT: "45123", + PATH: "/usr/bin" + } + sanitizeAmbientEnvironment(supplied) + const launch = localDevelopmentEnvironment(developmentLayout("/repo/worktree", {}), supplied) + + assert.equal(launch[profileKey], selection) + assert.equal(launch.CODEVISOR_DEV_PORT, "45123") + assert.equal(launch.PATH, "/usr/bin") + assert.equal(Object.hasOwn(launch, "CODEVISOR_APP_HOSTED"), false) + assert.equal(Object.hasOwn(launch, "CODEVISOR_APP_BUNDLE_PATH"), false) + assert.equal(Object.hasOwn(launch, "HERDMAN_APP_HOSTED"), false) + } +}) + +test("ordinary development does not acquire a screen-sharing diagnostic profile", () => { + const supplied = { PATH: "/usr/bin", CODEVISOR_APP_HOSTED: "1" } + sanitizeAmbientEnvironment(supplied) + const launch = localDevelopmentEnvironment(developmentLayout("/repo/worktree", {}), supplied) + + assert.equal(Object.hasOwn(launch, profileKey), false) + assert.equal(Object.hasOwn(supplied, profileKey), false) +}) diff --git a/scripts/dev-macos-reuse.mjs b/scripts/dev-macos-reuse.mjs new file mode 100644 index 000000000..6590abf72 --- /dev/null +++ b/scripts/dev-macos-reuse.mjs @@ -0,0 +1,34 @@ +import { join } from "node:path" + +export function requestsMacOSBuildReuse(arguments_) { + const requested = arguments_.includes("--reuse-macos-build") + if (requested && !arguments_.includes("--no-ios")) { + throw new Error("--reuse-macos-build is supported only by dev:macos.") + } + return requested +} + +/// Reusing a granted ad-hoc app must never fall back to a rebuild: that +/// would change its signing requirement while appearing to preserve it. +export async function verifyReusableMacOSApp({ + appBundle, + bundleIdentifier, + executableName, + capture, + run +}) { + const plist = join(appBundle, "Contents", "Info.plist") + const metadata = JSON.parse( + await capture("/usr/bin/plutil", ["-convert", "json", "-o", "-", plist]) + ) + for (const [key, expected] of [ + ["CFBundleIdentifier", bundleIdentifier], + ["CFBundleExecutable", executableName] + ]) { + const actual = metadata[key] + if (actual !== expected) { + throw new Error(`Cannot reuse macOS build: ${key} is ${actual}, expected ${expected}.`) + } + } + await run("/usr/bin/codesign", ["--verify", "--deep", "--strict", appBundle]) +} diff --git a/scripts/dev-macos-reuse.test.mjs b/scripts/dev-macos-reuse.test.mjs new file mode 100644 index 000000000..5b320e9ca --- /dev/null +++ b/scripts/dev-macos-reuse.test.mjs @@ -0,0 +1,85 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { requestsMacOSBuildReuse, verifyReusableMacOSApp } from "./dev-macos-reuse.mjs" +import { parseDevelopmentRunnerArguments } from "./dev-arguments.mjs" + +function fixture(overrides = {}) { + const calls = [] + return { + calls, + options: { + appBundle: "/owned/Debug/Example.app", + bundleIdentifier: "dev.example", + executableName: "Example", + capture: async (command, args) => { + calls.push([command, ...args]) + return JSON.stringify({ CFBundleIdentifier: "dev.example", CFBundleExecutable: "Example" }) + }, + run: async (command, args) => calls.push([command, ...args]), + ...overrides + } + } +} + +test("reuse performs only metadata reading and strict signature verification", async () => { + const { options, calls } = fixture() + await verifyReusableMacOSApp(options) + assert.deepEqual(calls, [ + [ + "/usr/bin/plutil", + "-convert", + "json", + "-o", + "-", + "/owned/Debug/Example.app/Contents/Info.plist" + ], + ["/usr/bin/codesign", "--verify", "--deep", "--strict", "/owned/Debug/Example.app"] + ]) +}) + +for (const [name, overrides, message] of [ + [ + "missing", + { + capture: async () => { + throw new Error("missing plist") + } + }, + /missing plist/ + ], + [ + "foreign", + { capture: async () => JSON.stringify({ CFBundleIdentifier: "other" }) }, + /Cannot reuse/ + ], + [ + "wrong executable", + { + capture: async () => + JSON.stringify({ CFBundleIdentifier: "dev.example", CFBundleExecutable: "Other" }) + }, + /Cannot reuse/ + ], + [ + "invalid signature", + { + run: async () => { + throw new Error("invalid signature") + } + }, + /invalid signature/ + ] +]) { + test(`${name} artifact rejects reuse without a rebuild or signing command`, async () => { + const { options, calls } = fixture(overrides) + await assert.rejects(verifyReusableMacOSApp(options), message) + assert.ok(calls.every(([command]) => command === "/usr/bin/plutil")) + }) +} + +test("reuse is explicit and rejected by combined or iOS runners", () => { + assert.equal(requestsMacOSBuildReuse(["--no-ios"]), false) + assert.equal(requestsMacOSBuildReuse(["--no-ios", "--reuse-macos-build"]), true) + assert.throws(() => requestsMacOSBuildReuse(["--reuse-macos-build"]), /only by dev:macos/) + assert.throws(() => parseDevelopmentRunnerArguments(["--reuse-macos-build"]), /Unknown/) +}) diff --git a/scripts/dev.mjs b/scripts/dev.mjs index b8c12b611..8270cc3e7 100644 --- a/scripts/dev.mjs +++ b/scripts/dev.mjs @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url" import { bootstrapDevelopment } from "./dev-bootstrap.mjs" import { parseDevelopmentRunnerArguments } from "./dev-arguments.mjs" +import { requestsMacOSBuildReuse, verifyReusableMacOSApp } from "./dev-macos-reuse.mjs" import { applyCloudDevMigrations, prepareCloudSession, @@ -47,11 +48,13 @@ import { runXcodebuild } from "./xcodebuild.mjs" const repoRoot = await realpath(fileURLToPath(new URL("..", import.meta.url))) const arguments_ = process.argv.slice(2) const { wantsContainers, containerEnginePreference } = parseDevelopmentRunnerArguments(arguments_, { - allowedArguments: ["--no-ios"] + allowedArguments: ["--no-ios", "--reuse-macos-build"] }) +const reuseMacOSBuild = requestsMacOSBuildReuse(arguments_) // Both apps by default — `dev` means the whole rig. `--no-ios` backs the // dev:macos script; it is plumbing, not a user-facing option (the public -// surface is dev / dev:macos / dev:ios plus the container flags). +// surface is dev / dev:macos / dev:ios plus the container flags; dev:macos +// also permits explicit reuse of its already-built, signed application). const includesIOS = !arguments_.includes("--no-ios") // Containerized dev remotes are the default: real Linux machines make // cross-machine sync honest. --no-containers opts out; a missing engine @@ -83,6 +86,17 @@ const { wwwPort } = instance const { capture, run } = makeCommandRunner(repoRoot) +// Fail before claiming a runner or starting services. Do not silently +// rebuild an app whose ad-hoc identity was just granted OS permissions. +if (reuseMacOSBuild) { + await verifyReusableMacOSApp({ + appBundle, + bundleIdentifier: macOSBundleIdentifier, + executableName: appName, + capture, + run + }) +} const { cloudExtraVariables, cloudPersistPath, cloudPort, cloudUrl } = await resolveCloudDevInstance({ environment: process.env, @@ -153,33 +167,37 @@ const cloud = spawnCloudDev({ worktreeName }) -const generatedIconDirectory = await createDevelopmentAppIcon(repoRoot, developmentIconColor) -const developmentSigningArguments = await resolveDevelopmentSigningArguments(capture) -try { - await runXcodebuild( - repoRoot, - "macos", - [ - "-project", - "apps/macos/Codevisor.xcodeproj", - "-scheme", - "Codevisor", - "-configuration", - "Debug", - `CODEVISOR_DEV_PRODUCT_NAME=${appName}`, - `CODEVISOR_DEV_DISPLAY_NAME=${appName}`, - `CODEVISOR_DEV_BUNDLE_IDENTIFIER=${macOSBundleIdentifier}`, - `CODEVISOR_URL_SCHEME=${urlScheme}`, - "ASSETCATALOG_COMPILER_APPICON_NAME=AppIconDevGenerated", - "INFOPLIST_KEY_CFBundleIconFile=AppIconDevGenerated", - "INFOPLIST_KEY_CFBundleIconName=AppIconDevGenerated", - ...developmentSigningArguments, - "build" - ], - { environment: process.env, layout } - ) -} finally { - await rm(generatedIconDirectory, { recursive: true, force: true }) +if (!reuseMacOSBuild) { + const generatedIconDirectory = await createDevelopmentAppIcon(repoRoot, developmentIconColor) + const developmentSigningArguments = await resolveDevelopmentSigningArguments(capture) + try { + await runXcodebuild( + repoRoot, + "macos", + [ + "-project", + "apps/macos/Codevisor.xcodeproj", + "-scheme", + "Codevisor", + "-configuration", + "Debug", + `CODEVISOR_DEV_PRODUCT_NAME=${appName}`, + `CODEVISOR_DEV_DISPLAY_NAME=${appName}`, + `CODEVISOR_DEV_BUNDLE_IDENTIFIER=${macOSBundleIdentifier}`, + `CODEVISOR_URL_SCHEME=${urlScheme}`, + "ASSETCATALOG_COMPILER_APPICON_NAME=AppIconDevGenerated", + "INFOPLIST_KEY_CFBundleIconFile=AppIconDevGenerated", + "INFOPLIST_KEY_CFBundleIconName=AppIconDevGenerated", + ...developmentSigningArguments, + "build" + ], + { environment: process.env, layout } + ) + } finally { + await rm(generatedIconDirectory, { recursive: true, force: true }) + } +} else { + console.log("Reusing the existing signed macOS app; native source changes are not built.") } let iosTarget if (includesIOS) { diff --git a/scripts/screen-sharing-probe.mjs b/scripts/screen-sharing-probe.mjs new file mode 100644 index 000000000..03a622a72 --- /dev/null +++ b/scripts/screen-sharing-probe.mjs @@ -0,0 +1,112 @@ +#!/usr/bin/env node +// Build a self-contained diagnostic app, including the pinned WebRTC framework. +import { spawnSync } from "node:child_process" +import { createHash } from "node:crypto" +import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs" +import { dirname, join } from "node:path" +import { fileURLToPath } from "node:url" + +const root = dirname(dirname(fileURLToPath(import.meta.url))) +const packagePath = join(root, "packages/swift") +const args = process.argv.slice(2) +const releaseIndex = args.indexOf("--release") +const configuration = releaseIndex >= 0 ? "release" : "debug" +if (releaseIndex >= 0) args.splice(releaseIndex, 1) +const instanceIndex = args.indexOf("--instance") +let instance = "" +if (instanceIndex >= 0) { + instance = args[instanceIndex + 1] ?? "" + if (!/^[a-z0-9-]{1,32}$/.test(instance)) + throw new Error("--instance needs a short lowercase name") + args.splice(instanceIndex, 2) +} +const app = join(root, `tmp/screen-sharing/ScreenSharingProbe${instance ? `-${instance}` : ""}.app`) +// Separate installed identities prevent Launch Services and TCC from resolving +// one diagnostic instance to another worktree's or instance's executable. +// Ad-hoc rebuilds still change the code requirement and may need a new grant. +const worktreeID = createHash("sha256").update(root).digest("hex").slice(0, 12) +const bundleID = `com.codevisor.ScreenSharingProbe.w${worktreeID}.${instance || "default"}` +const displayName = `Screen Sharing Probe${instance ? ` (${instance})` : ""}` + +function run(command, args, capture = false) { + const result = spawnSync(command, args, { + cwd: root, + stdio: capture ? ["inherit", "pipe", "inherit"] : "inherit", + encoding: "utf8" + }) + if (result.error) throw result.error + if (result.status !== 0) process.exit(result.status ?? 1) + return result.stdout?.trim() +} + +if (process.platform !== "darwin") throw new Error("The Screen Sharing probe requires macOS.") +if (existsSync(app)) { + const owners = spawnSync( + "/usr/sbin/lsof", + ["-t", join(app, "Contents/MacOS/screen-sharing-probe")], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"] + } + ) + if (owners.error) throw owners.error + if (owners.status === 0 && owners.stdout.trim()) { + throw new Error("This probe is running. Stop it or use --instance NAME before rebuilding.") + } +} +run("swift", [ + "build", + "--package-path", + packagePath, + "--configuration", + configuration, + "--product", + "screen-sharing-probe" +]) +const bin = run( + "swift", + ["build", "--package-path", packagePath, "--configuration", configuration, "--show-bin-path"], + true +) +rmSync(app, { recursive: true, force: true }) +const contents = join(app, "Contents") +const executable = join(contents, "MacOS/screen-sharing-probe") +mkdirSync(join(contents, "MacOS"), { recursive: true }) +mkdirSync(join(contents, "Frameworks"), { recursive: true }) +mkdirSync(join(contents, "Resources"), { recursive: true }) +cpSync(join(bin, "screen-sharing-probe"), executable) +cpSync(join(bin, "WebRTC.framework"), join(contents, "Frameworks/WebRTC.framework"), { + recursive: true, + verbatimSymlinks: true +}) +cpSync( + join(bin, "CodevisorKit_CodevisorScreenSharing.bundle"), + join(contents, "Resources/CodevisorKit_CodevisorScreenSharing.bundle"), + { + recursive: true + } +) +writeFileSync( + join(contents, "Info.plist"), + ` + + +CFBundleIdentifier${bundleID} +CFBundleName${displayName} +CFBundleDisplayName${displayName} +CFBundleExecutablescreen-sharing-probe +CFBundlePackageTypeAPPL +CFBundleVersion1 +CodevisorProbeBuildConfiguration${configuration} +LSMinimumSystemVersion26.0 +NSHighResolutionCapable +NSScreenCaptureUsageDescriptionCapture the display you select for a native Screen Sharing diagnostic. +NSLocalNetworkUsageDescriptionConnect to the other Mac in your Screen Sharing diagnostic. + +` +) +run("install_name_tool", ["-add_rpath", "@executable_path/../Frameworks", executable]) +run("codesign", ["--force", "--sign", "-", join(contents, "Frameworks/WebRTC.framework")]) +run("codesign", ["--force", "--sign", "-", app]) +process.stdout.write(`Built ${app}\n`) +if (!args.includes("--build-only")) run(executable, args) diff --git a/scripts/webrtc-build.lock.json b/scripts/webrtc-build.lock.json new file mode 100644 index 000000000..d42e37e70 --- /dev/null +++ b/scripts/webrtc-build.lock.json @@ -0,0 +1,23 @@ +{ + "schemaVersion": 1, + "sourceRevision": "6f37672d358475cd17544121a12494da454d85fb", + "depotToolsRevision": "7ef1747912ba653ac4f9a273d8a135abe978e914", + "packageVersion": "152.0.0", + "packageRevision": "1d04692697cb642bfebf6ad2dd99fe52649c3d6d", + "publishedSHA256": "115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8", + "xcodeVersion": "26.5", + "pythonVersion": "3.12.14", + "gnArguments": [ + "is_debug=false", + "rtc_libvpx_build_vp9=true", + "is_component_build=false", + "rtc_include_tests=false", + "rtc_enable_objc_symbol_export=true", + "enable_stripping=true", + "enable_dsyms=true", + "use_lld=true", + "rtc_ios_use_opengl_rendering=true", + "rtc_system_openh264=true", + "rtc_use_h265=true" + ] +} From dbdbcaea896cf21f9227416695c06b09421b3309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 16:41:25 -0700 Subject: [PATCH 02/11] Keep screen sharing controls interactive while connecting --- .../Features/Panes/ScreenSharingToolbar.swift | 57 +++++----- .../ScreenSharingViewerControl.swift | 2 + .../ScreenSharingViewerModel.swift | 18 +++- .../ScreenSharingViewerTests.swift | 101 +++++++++++++++++- 4 files changed, 142 insertions(+), 36 deletions(-) diff --git a/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift b/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift index d21ccf284..2e3ed676b 100644 --- a/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift +++ b/apps/macos/Codevisor/Features/Panes/ScreenSharingToolbar.swift @@ -10,7 +10,7 @@ struct ScreenSharingToolbar: ToolbarContent { var body: some ToolbarContent { ToolbarItem(id: "screenSharing.mode", placement: .principal) { - HStack(spacing: 8) { controlActions } + controlActions } ToolbarItem(id: "screenSharing.size", placement: .primaryAction) { Picker("Size", selection: Binding(get: { model.preferences.fitToWindow }, set: { model.setFitToWindow($0) })) { @@ -19,44 +19,34 @@ struct ScreenSharingToolbar: ToolbarContent { } .labelsHidden().frame(width: 110) } - if let clipboard = model.clipboard { - ToolbarItem(id: "screenSharing.clipboard", placement: .primaryAction) { - Menu { - Button("Send Clipboard to Mac") { clipboard.sendLocalText() } - Button("Get Clipboard from Mac") { clipboard.getRemoteText() } - } label: { - Image(systemName: "doc.on.clipboard") + ToolbarItem(id: "screenSharing.clipboard", placement: .primaryAction) { + Menu { + Group { + Button("Send Clipboard to Mac") { model.clipboard?.sendLocalText() } + Button("Get Clipboard from Mac") { model.clipboard?.getRemoteText() } } - .accessibilityLabel("Clipboard") - .help("Transfer plain text between clipboards") - .disabled(!clipboard.available || clipboard.busy) + .disabled(model.clipboard?.available != true || model.clipboard?.busy == true) + } label: { + Image(systemName: "doc.on.clipboard") } + .accessibilityLabel("Clipboard") + .help("Transfer plain text between clipboards") } ToolbarItem(id: "screenSharing.details", placement: .primaryAction) { ScreenSharingDetailsButton(model: model).id(ObjectIdentifier(model)) } } - @ViewBuilder private var controlActions: some View { - if model.phase == .viewing, let control = model.control { - Picker( - "Interaction mode", - selection: Binding( - get: { control.state != .viewing }, - set: { if $0 { control.request() } else { control.release() } }) - ) { - Text("View").tag(false) - Text("Control").tag(true) - } - .pickerStyle(.segmented).labelsHidden().fixedSize() - .disabled(!control.available) - .help("Send mouse, keyboard and app shortcuts to this Mac. Control–Option–Escape returns to viewing.") - if control.state == .requesting { - ProgressView().controlSize(.mini).accessibilityLabel("Requesting control") - } - } else { - Text("View").foregroundStyle(.secondary) + private var controlActions: some View { + Picker( + "Interaction mode", + selection: Binding(get: { model.interactionMode }, set: { model.setInteractionMode($0) }) + ) { + Text("View").tag(ScreenSharingViewerModel.InteractionMode.view) + Text("Control").tag(ScreenSharingViewerModel.InteractionMode.control) } + .pickerStyle(.segmented).labelsHidden().fixedSize() + .help("Send mouse, keyboard and app shortcuts to this Mac. Control–Option–Escape returns to viewing.") } } @@ -95,6 +85,13 @@ private struct ScreenSharingDetailsButton: View { Text(diagnostics.decoder).font(.caption).foregroundStyle(.secondary) } .font(.callout) + } else { + VStack(alignment: .leading, spacing: 10) { + Text("Connection Details").font(.headline) + Text(model.message ?? "Connection details will appear when the screen share is ready.") + .foregroundStyle(.secondary) + } + .font(.callout) } } } diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift index 75daf3d90..c95098fd2 100644 --- a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerControl.swift @@ -10,6 +10,7 @@ public final class ScreenSharingViewerControl { public private(set) var available = false public private(set) var message: String? @ObservationIgnored var onActiveChanged: ((Bool) -> Void)? + @ObservationIgnored var onReleased: (() -> Void)? @ObservationIgnored private let send: (ScreenSharingControlMessage) -> Bool @ObservationIgnored private let now: () -> TimeInterval @ObservationIgnored private var requestID: UUID? @@ -44,6 +45,7 @@ public final class ScreenSharingViewerControl { let oldLease = lease requestID = nil; lease = nil; state = .viewing; message = reason onActiveChanged?(false) + onReleased?() if let oldLease { _ = send(.release(lease: oldLease)) } } diff --git a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift index 1ef0555eb..dd74915de 100644 --- a/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift +++ b/packages/swift/CodevisorCoreMac/Sources/CodevisorCoreMac/ScreenSharing/ScreenSharingViewerModel.swift @@ -7,7 +7,9 @@ import Observation @Observable public final class ScreenSharingViewerModel { public enum Phase: Equatable { case idle, loading, ready, connecting, reconnecting, viewing, suspended, failed } + public enum InteractionMode: Hashable, Sendable { case view, control } public private(set) var phase: Phase = .idle + public private(set) var interactionMode: InteractionMode = .control public private(set) var message: String? public private(set) var displays: [ServerScreenSharingDisplay] = [] public private(set) var selectedDisplayId: String? @@ -24,7 +26,7 @@ public final class ScreenSharingViewerModel { @ObservationIgnored private let makePeer: (ServerScreenSharingConnectivity?) throws -> any ScreenSharingViewingPeer @ObservationIgnored private let sleep: @Sendable (Duration) async throws -> Void @ObservationIgnored private var task: Task? - @ObservationIgnored private var endpoint: (any ScreenSharingViewingPeer)? + private var endpoint: (any ScreenSharingViewingPeer)? @ObservationIgnored private var generation = 0 @ObservationIgnored private var visible = false @ObservationIgnored private var wantsConnection = false @@ -99,6 +101,14 @@ public final class ScreenSharingViewerModel { onPreferencesChanged?(preferences) } + /// Keep the user's choice while connecting; send the request only after video + /// and the control channel are ready. + public func setInteractionMode(_ mode: InteractionMode) { + interactionMode = mode + guard phase == .viewing else { return } + if mode == .control { control?.requestWhenAvailable() } else { control?.release() } + } + /// Apply a registry update without replacing the live surface or echoing /// the write. A display change from another client requires a new Connect. public func applyPreferences(_ preferences: ScreenSharingPanePreferences) { @@ -170,12 +180,16 @@ public final class ScreenSharingViewerModel { peer = created; endpoint = created; videoView = created.view created.fit(preferences.fitToWindow) created.onFocusChanged = { [weak self] in self?.onFocusChanged?($0) } + created.control.onReleased = { [weak self] in + guard let self, self.isCurrent(generation), self.phase == .viewing else { return } + self.interactionMode = .view + } phase = restarts == 0 ? .connecting : .reconnecting created.onReady = { [weak self] in guard let self, self.isCurrent(generation), [.connecting, .reconnecting].contains(self.phase) else { return } self.phase = .viewing self.message = nil - self.control?.requestWhenAvailable() + if self.interactionMode == .control { self.control?.requestWhenAvailable() } } created.onConnectionChanged = { [weak self] state in guard let self, self.isCurrent(generation), ["failed", "disconnected", "closed"].contains(state) else { return } diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift index 01c3302b3..0b8ad5efe 100644 --- a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ScreenSharingViewerTests.swift @@ -16,11 +16,13 @@ struct ScreenSharingViewerTests { peer.deliversVideo = false peer.channelAvailableOnAccept = channelFirst let model = makeModel(transport, clock: clock) { peer } + #expect(model.interactionMode == .control) model.setVisible(true) await awaitObserved { model.phase == .ready } model.connect() await clock.waitForSleep(.seconds(8)) #expect(model.phase == .connecting) + #expect(model.interactionMode == .control) #expect(peer.controlMessages.isEmpty) peer.onReady?() #expect(model.phase == .viewing) @@ -36,6 +38,7 @@ struct ScreenSharingViewerTests { peer.control.receive(.grant(request: request, lease: lease)) #expect(model.control?.state == .controlling) peer.control.release() + #expect(model.interactionMode == .view) peer.onReady?() model.setFitToWindow(false) peer.control.setAvailable(true) @@ -45,13 +48,101 @@ struct ScreenSharingViewerTests { #expect(clock.pendingCount == 0) } - @Test func networkRecoveryUsesFreshMediaAndTheSameSessionAndRequestsFreshControl() async throws { + @Test(arguments: [ScreenSharingViewerModel.InteractionMode.view, .control]) + func connectingHonorsTheLatestModeAndSizeChoice(mode: ScreenSharingViewerModel.InteractionMode) async { + let transport = SharingTransport(blockFirstStart: true) + let clock = TestClock() + let peer = SharingPeer() + let model = makeModel(transport, clock: clock) { peer } + model.setInteractionMode(.view) + #expect(model.interactionMode == .view && model.control == nil) + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await transport.started.wait() + #expect(model.phase == .connecting) + model.setInteractionMode(.control) + model.setInteractionMode(mode) + model.setFitToWindow(false) + #expect(model.interactionMode == mode) + #expect(peer.controlMessages.isEmpty) + transport.releaseFirstStart.signal() + await clock.waitForSleep(.seconds(8)) + #expect(model.phase == .viewing) + #expect(model.interactionMode == mode) + #expect(peer.control.state == (mode == .control ? .requesting : .viewing)) + #expect(peer.controlMessages.count == (mode == .control ? 1 : 0)) + #expect(!peer.fitToWindow) + await model.close() + #expect(clock.pendingCount == 0) + } + + @Test func choosingViewCancelsControlWaitingForTheChannel() async { + let transport = SharingTransport() + let clock = TestClock() + let peer = SharingPeer() + peer.channelAvailableOnAccept = false + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + #expect(model.phase == .viewing && model.interactionMode == .control) + #expect(peer.controlMessages.isEmpty) + model.setInteractionMode(.view) + peer.control.setAvailable(true) + #expect(model.interactionMode == .view) + #expect(peer.controlMessages.isEmpty) + model.setInteractionMode(.control) + #expect(peer.control.state == .requesting) + #expect(peer.controlMessages.count == 1) + await model.close() + #expect(clock.pendingCount == 0) + } + + @Test(arguments: [true, false]) + func deniedOrRevokedControlReturnsTheModeToView(grantFirst: Bool) async throws { + let transport = SharingTransport() + let clock = TestClock() + let peer = SharingPeer() + let model = makeModel(transport, clock: clock) { peer } + model.setVisible(true) + await awaitObserved { model.phase == .ready } + model.connect() + await clock.waitForSleep(.seconds(8)) + guard case .request(let request)? = peer.controlMessages.first else { + Issue.record("Missing control request"); await model.close(); return + } + if grantFirst { + let lease = UUID() + peer.control.receive(.grant(request: request, lease: lease)) + #expect(model.interactionMode == .control && peer.control.state == .controlling) + peer.control.receive(.revoked(lease: lease, reason: "Host ended control")) + } else { + peer.control.receive(.denied(request: request, reason: "Host denied control")) + } + #expect(model.interactionMode == .view && peer.control.state == .viewing) + let sent = peer.controlMessages + model.setFitToWindow(false) + peer.onReady?() + #expect(model.interactionMode == .view && peer.controlMessages == sent) + await model.close() + #expect(clock.pendingCount == 0) + } + + @Test(arguments: [ScreenSharingViewerModel.InteractionMode.view, .control]) + func networkRecoveryUsesFreshMediaAndTheSameSessionAndPreservesMode( + mode: ScreenSharingViewerModel.InteractionMode + ) + async throws + { let transport = SharingTransport() let clock = TestClock() var peers: [SharingPeer] = [] let model = makeModel(transport, clock: clock) { let peer = SharingPeer(); peers.append(peer); return peer } + model.setInteractionMode(mode) model.setVisible(true) await awaitObserved { model.phase == .ready } model.connect() @@ -72,9 +163,11 @@ struct ScreenSharingViewerTests { #expect(requests.filter { $0.operation == .stop }.isEmpty) #expect(requests.filter { $0.operation == .capabilities && $0.viewerId == start.viewerId }.count == 2) #expect(first.control.state == .viewing) - #expect(peers[1].control.state == .requesting) - #expect(first.controlMessages.count == 1 && peers[1].controlMessages.count == 1) - #expect(first.controlMessages != peers[1].controlMessages) + #expect(model.interactionMode == mode) + #expect(peers[1].control.state == (mode == .control ? .requesting : .viewing)) + let count = mode == .control ? 1 : 0 + #expect(first.controlMessages.count == count && peers[1].controlMessages.count == count) + if mode == .control { #expect(first.controlMessages != peers[1].controlMessages) } await model.close() #expect(peers.allSatisfy { $0.closed }) #expect(clock.pendingCount == 0) From e107326a58ddc5b3fc545b9c713d90da9dc8c74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 16:49:47 -0700 Subject: [PATCH 03/11] Document final MVP verification and distribution gates --- docs/plans/native-screen-sharing-mvp.md | 16 ++++++++++++++-- packages/swift/CodevisorScreenSharing/README.md | 6 +++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index 8458916d6..848d810b1 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -9,7 +9,7 @@ The intended MVP is one Codevisor Mac viewing and controlling an existing displa - ScreenCaptureKit capture, VideoToolbox H.264 encode/decode, encrypted WebRTC media and Metal rendering. - Authenticated workspace/pane signaling and a searchable display chooser matching New Tab. - Native toolbar with the machine name and display resolution, a persistent View/Control segment, Fit/Actual Size, clipboard and connection diagnostics. Closing the tab ends the session. -- Control is requested once when each connection becomes ready. Local menus and other focused controls suspend input forwarding and release held input without changing the selected mode. Ctrl-Option-Escape releases control explicitly. +- New panes start in Control mode. The View/Control selector remains interactive while connecting, and the latest choice applies when video and the control channel are ready. Reconnecting preserves that choice. Local menus and other focused controls suspend input forwarding and release held input without changing the selected mode. Ctrl-Option-Escape releases control explicitly. - While the video has focus in Control mode, system shortcuts such as Command-Space and Command-Q are forwarded to the host; bounded plain-text clipboard transfer is also available. - Chatless workspace navigation through the shared pane container, including conversion from New Tab and sidebar title updates. - Bounded media ownership, terminal renderer stop and owner-scoped cancellation of pending host starts. @@ -20,10 +20,22 @@ The intended MVP is one Codevisor Mac viewing and controlling an existing displa Actual-app checks on September 14 established remote video and control, including opening Spotlight, typing, and quitting the remote Calculator with Command-Q while the local viewer stayed connected. These checks used the pre-integration development build. A clean two-Mac acceptance run on the integrated source remains required; these checks are not a performance benchmark. -The final host stop-ordering source passed 167 CoreMac tests and serial macOS/iOS native builds on September 13, 2026. The sidebar and host-ordering fixes were compiled in isolated build outputs but have not yet been exercised in a newly launched two-Mac session. These are historical results, not acceptance evidence for the integrated branch. The September 14 integration adopts main's center-only workspace layout and focus-source ownership, and preserves chatless workspace behavior. Current build and full-check results are recorded in the PR. +Commit `dbdbcaea` passed the normal pre-commit hook in a clean validation worktree, without exclusions: the full JavaScript check chain, Swift formatting/lint, 2,119 Swift package tests, 30 macOS transcript tests, 11 macOS composer tests, the iOS build, and 8 iOS transcript tests. The macOS development app built successfully. The September 14 integration adopts main's center-only workspace layout and focus-source ownership, and preserves chatless workspace behavior. + +Actual-app checks with the final viewer established that the complete toolbar stays interactive during connection, View/Control choices survive connection and menu interactions, and switching modes no longer shows a spinner. Those checks used the earlier host build. The isolated host was subsequently rebuilt from the matching final source; all 2,864 staged source files were verified and the app passed strict signature verification. The fresh acceptance attempt reached the host Screen Recording permission error. The remaining two-Mac workflows and sustained-use check are pending reauthorization of this rebuilt development app; earlier results do not replace that acceptance pass. The local measurement archives contain raw session and machine-specific evidence and are intentionally excluded from this public checkpoint. Historical measurement links in the implementation notes refer to those local archives. No credentials, raw signaling, desktop captures or runner logs are required to build the feature. +## Dependency and distribution review + +The pinned stasel WebRTC 152.0.0 archive was downloaded again on September 14. Its SHA256 matches `scripts/webrtc-build.lock.json` and the SwiftPM checksum: `115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8`. All 387 regular files and 10 symlink targets match the resolved artifact. The XCFramework provides macOS arm64/x86_64, iOS device and simulator slices, and Mac Catalyst slices. + +The built macOS app includes the WebRTC license and privacy manifest. The app and its embedded WebRTC framework pass strict signature verification. The release script signs embedded frameworks before signing and verifying the enclosing app; Developer ID signing and notarization of this final revision have not been executed. + +The archive contains the main WebRTC license but no aggregate third-party notices. The pinned source build and graph-derived notice audit remain open. The recipe requires Xcode 26.5 and Python 3.12.14. The available Macs have Xcode 27 beta and 26.6; an isolated Python 3.12.14 installation is now prepared. The recipe correctly refuses the unmatched Xcode before fetching sources. No toolchain pin was relaxed and no replacement dependency was promoted. + +Pushing to `main` automatically starts the Alpha build and publication workflow, so the dependency/distribution gate cannot be deferred to a later manual release. The PR currently has no completed GitHub build/test workflow; its correctness check is skipped while in draft. The full-check result above is local verification, not a CI result. + ## Before MVP acceptance 1. Verify the final source in two actual apps: permissions, viewing, typing, clicking, dragging, scrolling, clipboard, scaling and display selection. diff --git a/packages/swift/CodevisorScreenSharing/README.md b/packages/swift/CodevisorScreenSharing/README.md index a7de0838e..0627d5306 100644 --- a/packages/swift/CodevisorScreenSharing/README.md +++ b/packages/swift/CodevisorScreenSharing/README.md @@ -6,7 +6,7 @@ The working path is ScreenCaptureKit (or a synthetic motion source) → native W ## Native workspace pane -Run `bun run dev:macos`. In a workspace hosted by a Mac, open New Tab → Screen Sharing, select a display and Connect. The host must run this version of the native app and allow Screen Recording. The native viewer supports Fit and Actual Size. iOS preserves the pane and shows an unavailable state until its viewer milestone. +Run `bun run dev:macos`. In a workspace hosted by a Mac, open New Tab → Screen Sharing and select a display from the searchable chooser to connect. The host must run this version of the native app and allow Screen Recording. The native toolbar shows the machine name and resolution, View/Control, Fit/Actual Size, clipboard and connection details. Closing the tab ends the session. iOS preserves the pane and shows an unavailable state until its viewer milestone. The viewer sends a receive-only offer through the existing machine-authenticated, relay-aware client to `POST /v1/screen-sharing`. The server validates the active workspace and native pane, then forwards the request over Codevisor’s authenticated Unix bridge to the app’s native host service. SDP stays transient; only the stable display UUID and fit preference enter pane metadata. Synced size preferences update the existing native surface; a display change from another client requires a fresh Connect. Websites are rejected even on loopback. @@ -16,7 +16,7 @@ The current host scales to fit 1920×1080, preserving aspect ratio, at up to 60 ## Native control -After video connects, choose **Control** to send mouse, keyboard and app shortcuts to the selected Mac display. The host also needs Accessibility permission (called Device Control and Data Access in macOS 27). Control is explicit for every new connection. **Control–Option–Escape** always returns to viewing; Stop Control, clicking outside the video, losing keyboard/window focus, hiding the pane, disconnecting and the host’s Stop Sharing action also release it. The host indicator changes to “Controlled”. +New panes start in **Control** mode and request control when video and the data channel are ready. The selector stays interactive while connecting; choosing **View** cancels that pending request. The selected mode survives menu interactions and reconnection. The host also needs Accessibility permission (called Device Control and Data Access in macOS 27). **Control–Option–Escape** returns to View. Clicking outside the video or losing keyboard/window focus suspends input forwarding and releases held input without changing the selected mode. Hiding or closing the pane, disconnecting, and the host’s Stop Sharing action tear down the connection and its control lease. The host indicator changes to “Controlled” while a control lease is active. A fresh host-issued control lease binds input to the current authenticated WebRTC peer and selected display. One-second heartbeats renew a separate three-second control deadline, checked every 250 ms. Expiry, permission loss, malformed messages and channel closure revoke the lease and release tracked keys/buttons. Delayed grants are released after viewer cancellation; input from an expired lease cannot revive control. No control lease, input or key content enters pane metadata or logs. @@ -24,7 +24,7 @@ The native ordered data channel carries key/button transitions, scroll deltas an The native surface maps through Fit/Actual Size letterboxing, scroll-document position and backing-pixel scale. The host maps normalized coordinates into Quartz display bounds, including negative origins. It uses public CGEvents from a private event source, with no automation delays or app activation. Tagged injected events are excluded from capture by a viewer on the same Mac. Fractional trackpad deltas accumulate before conversion to integer pixels. -Physical Mac key codes use the host’s keyboard layout and input method. OS-reserved shortcuts may remain local. Basic physical typing, modifiers, repeat, mouse buttons, dragging and scrolling are implemented; automatic clipboard synchronization, a local cursor overlay and dedicated Unicode composition UI remain follow-up work. Explicit plain-text clipboard transfer is available from the toolbar. The protocol has a bounded, separate text-insertion message exercised by the diagnostic sink. Rotated/external-display mapping, non-US layouts, IME behavior and real interactions on a second Mac still need hardware validation. +Physical Mac key codes use the host’s keyboard layout and input method. With viewer Accessibility permission, a focused video surface in Control mode forwards system shortcuts such as Command-Space and Command-Q. Basic physical typing, modifiers, repeat, mouse buttons, dragging and scrolling are implemented; automatic clipboard synchronization, a local cursor overlay and dedicated Unicode composition UI remain follow-up work. Explicit plain-text clipboard transfer is available from the toolbar. The protocol has a bounded, separate text-insertion message exercised by the diagnostic sink. Rotated/external-display mapping, non-US layouts, IME behavior and the complete final-source two-Mac acceptance checklist still need hardware validation. ## Clipboard, cursor and connection details From 51f66dbfedd18b7ea9d33880773716c4cf65c211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 17:36:52 -0700 Subject: [PATCH 04/11] Pin WebRTC source builds to Xcode 26.6 --- .github/workflows/webrtc-artifact.yml | 11 ++++++++--- docs/plans/native-screen-sharing-mvp.md | 2 +- docs/plans/native-screen-sharing.md | 2 +- packages/swift/CodevisorScreenSharing/README.md | 3 ++- scripts/webrtc-build.lock.json | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/webrtc-artifact.yml b/.github/workflows/webrtc-artifact.yml index 3f2f525c1..efc56834b 100644 --- a/.github/workflows/webrtc-artifact.yml +++ b/.github/workflows/webrtc-artifact.yml @@ -26,9 +26,14 @@ jobs: with: python-version: ${{ steps.tools.outputs.python }} - name: Build frameworks and dependency notices - env: - DEVELOPER_DIR: /Applications/Xcode_${{ steps.tools.outputs.xcode }}.app/Contents/Developer - run: node scripts/build-webrtc.mjs + shell: bash + run: | + developer_dir="/Applications/Xcode_${{ steps.tools.outputs.xcode }}.app/Contents/Developer" + if [[ ! -x "$developer_dir/usr/bin/xcodebuild" ]]; then + developer_dir="$(scripts/release/resolve-xcode.sh)" + fi + # The recipe still enforces the exact pinned version before fetching. + DEVELOPER_DIR="$developer_dir" node scripts/build-webrtc.mjs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: native-webrtc-candidate diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index 848d810b1..caec27d74 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -32,7 +32,7 @@ The pinned stasel WebRTC 152.0.0 archive was downloaded again on September 14. I The built macOS app includes the WebRTC license and privacy manifest. The app and its embedded WebRTC framework pass strict signature verification. The release script signs embedded frameworks before signing and verifying the enclosing app; Developer ID signing and notarization of this final revision have not been executed. -The archive contains the main WebRTC license but no aggregate third-party notices. The pinned source build and graph-derived notice audit remain open. The recipe requires Xcode 26.5 and Python 3.12.14. The available Macs have Xcode 27 beta and 26.6; an isolated Python 3.12.14 installation is now prepared. The recipe correctly refuses the unmatched Xcode before fetching sources. No toolchain pin was relaxed and no replacement dependency was promoted. +The archive contains the main WebRTC license but no aggregate third-party notices. The pinned source build and graph-derived notice audit remain open. On September 14, the Xcode pin was deliberately updated from 26.5 to the available 26.6 toolchain; Python remains pinned to 3.12.14. The earlier 26.5 refusal was the recipe's exact-version check, not an established upstream compatibility limit. The WebRTC source revision and installed dependency remain unchanged; the candidate artifact must pass its build, notice audit and validation before promotion. Pushing to `main` automatically starts the Alpha build and publication workflow, so the dependency/distribution gate cannot be deferred to a later manual release. The PR currently has no completed GitHub build/test workflow; its correctness check is skipped while in draft. The full-check result above is local verification, not a CI result. diff --git a/docs/plans/native-screen-sharing.md b/docs/plans/native-screen-sharing.md index 23ec71e50..385002f69 100644 --- a/docs/plans/native-screen-sharing.md +++ b/docs/plans/native-screen-sharing.md @@ -108,7 +108,7 @@ Milestones are sequential at their interfaces. Milestone 0 can be split into dep - [x] Explicit native HEVC/Main444 transport and NV24 rendering in the probe, Release builds, scoped capture picker and bounded pipeline experiments; see the [pipeline report](../measurements/native-screen-sharing-pipeline-2026-09-11/README.md). Performance parity and product codec promotion remain open. - [x] Keep requested recovery keyframes pending across encoder admission drops; explicit decoder-state-loss checks pass for H.264 and Main444, including 4K encoder backpressure. This is local fault injection, not loss acceptance on a real network. - [x] Pinned source-build recipe and manual artifact workflow prepared; published dependency checksum/slices verified. -- [ ] Complete the pinned source build with Xcode 26.5, audit generated third-party notices and validate the candidate binary before promotion. +- [ ] Complete the pinned source build with Xcode 26.6, audit generated third-party notices and validate the candidate binary before promotion. The toolchain pin was deliberately updated from 26.5 on September 14; the WebRTC source revision and installed artifact remain unchanged. - [ ] Run the specified two-Mac LAN editing/terminal/browser and input-to-photon checks, including rotated/external displays and non-US input methods. - [ ] Validate separate networks, forced TURN, UDP restrictions and route changes against a deployed relay. - [ ] Complete isolated wired-LAN and 30-minute resource/latency comparisons, matched 1080p Apple runs, quantitative text fidelity and external input-to-photon measurements. diff --git a/packages/swift/CodevisorScreenSharing/README.md b/packages/swift/CodevisorScreenSharing/README.md index 0627d5306..568fa7eee 100644 --- a/packages/swift/CodevisorScreenSharing/README.md +++ b/packages/swift/CodevisorScreenSharing/README.md @@ -237,6 +237,7 @@ Milestone 0 now includes exploratory two-Mac runs and a matched-scale 4K recordi | SHA-256 | `115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8` | | Required slices present | macOS arm64/x86_64; iOS arm64; iOS Simulator arm64/x86_64 | | Upstream workflow Xcode pin | `26.5` | +| Codevisor source-build Xcode pin | `26.6` (deliberate toolchain update, September 14) | SwiftPM validates the artifact checksum. The package and both Xcode workspaces pin the package revision. The reference checkout is [.repos/WebRTC](../../../.repos/WebRTC), added as a submodule. The downloaded archive checksum and required slices were inspected locally; this change uses the published binary rather than claiming a local WebRTC source build. @@ -244,7 +245,7 @@ The reference build script and release workflow are available at `.repos/WebRTC/ The Codevisor-owned recipe is [scripts/build-webrtc.mjs](../../../scripts/build-webrtc.mjs), with source, depot_tools, Python and Xcode pins in [webrtc-build.lock.json](../../../scripts/webrtc-build.lock.json). `node scripts/build-webrtc.mjs --plan` reviews its five architecture builds without changing files. The build disables depot_tools auto-update, records actual dependency revisions and GN arguments, generates notices from each platform's GN target graph, preserves dSYMs/privacy manifests, signs the local frameworks and writes an XCFramework archive with a SHA-256 manifest. Source and artifacts stay under a recipe-specific `tmp/webrtc-source/` directory; SwiftPM pins are unchanged. -A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Local preflight correctly rejected Xcode 27: the pinned upstream recipe requires Xcode 26.5, which is not installed here. The complete source build, notices review, binary smoke tests and candidate promotion remain unverified. Pinning inputs alone is not evidence of a byte-reproducible build. +A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Codevisor's recipe now pins Xcode 26.6, deliberately differing from the upstream workflow's 26.5 pin. The earlier refusal of Xcode 27 was an exact-version preflight check, not evidence that 26.6 is incompatible. Python remains pinned to 3.12.14. The complete source build, notices review, binary smoke tests and candidate promotion remain unverified. Pinning inputs alone is not evidence of a byte-reproducible build. The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The archive does not include a generated aggregate third-party license file: generate and audit the complete notices from the actual source-build dependency graph before product distribution. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. diff --git a/scripts/webrtc-build.lock.json b/scripts/webrtc-build.lock.json index d42e37e70..f1f818b9a 100644 --- a/scripts/webrtc-build.lock.json +++ b/scripts/webrtc-build.lock.json @@ -5,7 +5,7 @@ "packageVersion": "152.0.0", "packageRevision": "1d04692697cb642bfebf6ad2dd99fe52649c3d6d", "publishedSHA256": "115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8", - "xcodeVersion": "26.5", + "xcodeVersion": "26.6", "pythonVersion": "3.12.14", "gnArguments": [ "is_debug=false", From a015a78f77b0854d7a0d982811b31c71d6541231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 18:05:28 -0700 Subject: [PATCH 05/11] Validate the Xcode 26.6 WebRTC source recipe --- docs/plans/native-screen-sharing-mvp.md | 6 ++++-- docs/plans/native-screen-sharing.md | 3 ++- packages/swift/CodevisorScreenSharing/README.md | 6 +++--- scripts/build-webrtc.mjs | 3 +++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index caec27d74..28048b343 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -14,7 +14,7 @@ The intended MVP is one Codevisor Mac viewing and controlling an existing displa - Chatless workspace navigation through the shared pane container, including conversion from New Tab and sidebar title updates. - Bounded media ownership, terminal renderer stop and owner-scoped cancellation of pending host starts. - A default-off `CODEVISOR_SCREEN_SHARING_DIAGNOSTIC_PROFILE=paced15-worker` profile: 120 fps capture request at adaptive level zero, 1–15 ms receiver playout bounds, synchronized arrival rendering with two drawables and off-main preparation. A capture request is not an achieved frame rate. Process-wide WebRTC configuration requires an app restart to change. -- A standalone probe and a pinned WebRTC artifact-build recipe. The recipe's complete source build and generated dependency notices remain unverified. +- A standalone probe and a pinned WebRTC artifact-build recipe. The source build, generated-notice audit and isolated candidate tests now pass with Xcode 26.6; the candidate is not installed or promoted. ## Existing verification @@ -32,7 +32,9 @@ The pinned stasel WebRTC 152.0.0 archive was downloaded again on September 14. I The built macOS app includes the WebRTC license and privacy manifest. The app and its embedded WebRTC framework pass strict signature verification. The release script signs embedded frameworks before signing and verifying the enclosing app; Developer ID signing and notarization of this final revision have not been executed. -The archive contains the main WebRTC license but no aggregate third-party notices. The pinned source build and graph-derived notice audit remain open. On September 14, the Xcode pin was deliberately updated from 26.5 to the available 26.6 toolchain; Python remains pinned to 3.12.14. The earlier 26.5 refusal was the recipe's exact-version check, not an established upstream compatibility limit. The WebRTC source revision and installed dependency remain unchanged; the candidate artifact must pass its build, notice audit and validation before promotion. +The published archive contains the main WebRTC license but no aggregate third-party notices. On September 14, the Xcode pin was deliberately updated from 26.5 to 26.6 (17F113), with Python 3.12.14 unchanged. The first source-build attempt found a missing depot_tools bootstrap before GN; the corrected recipe initializes the pinned tools without updating their revision. The subsequent complete build passed for all five macOS/iOS architectures and packaged a candidate XCFramework (`cd477857489c347daddfbfa19bf0d905a2e2a143f3ee84543ba887b97cc13c45`). The generated notices match the exact source license texts for every mapped dependency in all five target graphs. Headers, slices, signatures, privacy manifests and matching dSYM UUIDs passed inspection; all 180 unchanged screen-sharing tests passed against the isolated candidate, as did factory/track/peer/data-channel creation and close. No capture or media connection was run with it. + +The WebRTC source revision and installed product dependency remain unchanged. The candidate includes audited notices, but the product resource bundle still needs their integration and packaging verification. Actual-app validation of the candidate, its promotion, and Developer ID signing/notarization remain open. A successful source build does not establish byte reproducibility or performance equivalence. Pushing to `main` automatically starts the Alpha build and publication workflow, so the dependency/distribution gate cannot be deferred to a later manual release. The PR currently has no completed GitHub build/test workflow; its correctness check is skipped while in draft. The full-check result above is local verification, not a CI result. diff --git a/docs/plans/native-screen-sharing.md b/docs/plans/native-screen-sharing.md index 385002f69..99499035b 100644 --- a/docs/plans/native-screen-sharing.md +++ b/docs/plans/native-screen-sharing.md @@ -108,7 +108,8 @@ Milestones are sequential at their interfaces. Milestone 0 can be split into dep - [x] Explicit native HEVC/Main444 transport and NV24 rendering in the probe, Release builds, scoped capture picker and bounded pipeline experiments; see the [pipeline report](../measurements/native-screen-sharing-pipeline-2026-09-11/README.md). Performance parity and product codec promotion remain open. - [x] Keep requested recovery keyframes pending across encoder admission drops; explicit decoder-state-loss checks pass for H.264 and Main444, including 4K encoder backpressure. This is local fault injection, not loss acceptance on a real network. - [x] Pinned source-build recipe and manual artifact workflow prepared; published dependency checksum/slices verified. -- [ ] Complete the pinned source build with Xcode 26.6, audit generated third-party notices and validate the candidate binary before promotion. The toolchain pin was deliberately updated from 26.5 on September 14; the WebRTC source revision and installed artifact remain unchanged. +- [x] Complete the pinned source build with Xcode 26.6 and audit generated third-party notices: all five architectures, source-license text checks, slices/signatures/privacy/dSYM inspection, 180 unchanged media tests against the isolated candidate, and a native factory/peer smoke passed September 14. The toolchain pin deliberately changed from 26.5; WebRTC source and the installed dependency are unchanged. See the [MVP checkpoint](native-screen-sharing-mvp.md). +- [ ] Integrate the audited notices into product packaging and validate the source-built candidate in actual apps before promotion. No candidate media/performance claim or byte-reproducibility claim is made. - [ ] Run the specified two-Mac LAN editing/terminal/browser and input-to-photon checks, including rotated/external displays and non-US input methods. - [ ] Validate separate networks, forced TURN, UDP restrictions and route changes against a deployed relay. - [ ] Complete isolated wired-LAN and 30-minute resource/latency comparisons, matched 1080p Apple runs, quantitative text fidelity and external input-to-photon measurements. diff --git a/packages/swift/CodevisorScreenSharing/README.md b/packages/swift/CodevisorScreenSharing/README.md index 568fa7eee..9ae1d41d3 100644 --- a/packages/swift/CodevisorScreenSharing/README.md +++ b/packages/swift/CodevisorScreenSharing/README.md @@ -243,11 +243,11 @@ SwiftPM validates the artifact checksum. The package and both Xcode workspaces p The reference build script and release workflow are available at `.repos/WebRTC/scripts/build.sh` and `.repos/WebRTC/.github/workflows/webrtc-release.yml`. They build native Objective-C codec hooks and enable H.265 packetization. The distribution provides no Apple HEVC encoder/decoder implementation. Our custom VideoToolbox adapters provide H.264 by default and explicit HEVC/Main444 selection for probe experiments. -The Codevisor-owned recipe is [scripts/build-webrtc.mjs](../../../scripts/build-webrtc.mjs), with source, depot_tools, Python and Xcode pins in [webrtc-build.lock.json](../../../scripts/webrtc-build.lock.json). `node scripts/build-webrtc.mjs --plan` reviews its five architecture builds without changing files. The build disables depot_tools auto-update, records actual dependency revisions and GN arguments, generates notices from each platform's GN target graph, preserves dSYMs/privacy manifests, signs the local frameworks and writes an XCFramework archive with a SHA-256 manifest. Source and artifacts stay under a recipe-specific `tmp/webrtc-source/` directory; SwiftPM pins are unchanged. +The Codevisor-owned recipe is [scripts/build-webrtc.mjs](../../../scripts/build-webrtc.mjs), with source, depot_tools, Python and Xcode pins in [webrtc-build.lock.json](../../../scripts/webrtc-build.lock.json). `node scripts/build-webrtc.mjs --plan` reviews its five architecture builds without changing files. The build disables depot_tools auto-update, explicitly bootstraps its pinned tools without advancing the checkout, records actual dependency revisions and GN arguments, generates notices from each platform's GN target graph, preserves dSYMs/privacy manifests, signs the local frameworks and writes an XCFramework archive with a SHA-256 manifest. Source and artifacts stay under a recipe-specific `tmp/webrtc-source/` directory; SwiftPM pins are unchanged. -A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Codevisor's recipe now pins Xcode 26.6, deliberately differing from the upstream workflow's 26.5 pin. The earlier refusal of Xcode 27 was an exact-version preflight check, not evidence that 26.6 is incompatible. Python remains pinned to 3.12.14. The complete source build, notices review, binary smoke tests and candidate promotion remain unverified. Pinning inputs alone is not evidence of a byte-reproducible build. +A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Codevisor's recipe now pins Xcode 26.6, deliberately differing from the upstream workflow's 26.5 pin. The earlier refusal of Xcode 27 was an exact-version preflight check, not evidence that 26.6 is incompatible. Python remains pinned to 3.12.14. The complete source build passed on September 14 with Xcode 26.6 (17F113) and Python 3.12.14: macOS arm64/x86_64, iOS arm64 and Simulator arm64/x86_64. Candidate archive SHA-256: `cd477857489c347daddfbfa19bf0d905a2e2a143f3ee84543ba887b97cc13c45`. Graph-derived notices were checked against the exact source license texts across all five builds; the macOS notice has 25 sections and the iOS notice 23, including WebRTC. All 86 macOS public headers match the published dependency byte-for-byte. Architecture slices, privacy manifests, signatures and dSYM UUIDs passed inspection. The unchanged media module compiled against the candidate in an isolated package, all 180 tests passed, and a native factory/source/track/peer/data-channel creation-and-close smoke passed without capture or connection. This is one successful build, not proof of byte reproducibility or a media/performance comparison. The installed SwiftPM dependency remains the published artifact; candidate promotion and actual-app validation remain open. -The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The archive does not include a generated aggregate third-party license file: generate and audit the complete notices from the actual source-build dependency graph before product distribution. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. +The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The published archive does not include an aggregate third-party license file. The source-built candidate now includes the audited generated notices, but they have not yet been integrated into the product resource bundle; notice packaging remains a distribution gate. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. Control API references: [WebRTC data channels](https://webrtc.org/getting-started/data-channels), [Apple’s private event-source state](https://developer.apple.com/documentation/coregraphics/cgeventsourcestateid), and [event-source user data](https://developer.apple.com/documentation/coregraphics/cgeventsource/userdata). Native channel signatures were checked against the pinned M152 framework headers. diff --git a/scripts/build-webrtc.mjs b/scripts/build-webrtc.mjs index f5153600c..8ad803b56 100644 --- a/scripts/build-webrtc.mjs +++ b/scripts/build-webrtc.mjs @@ -203,6 +203,9 @@ function build() { "https://chromium.googlesource.com/chromium/tools/depot_tools.git", lock.depotToolsRevision ) + // Disabling depot_tools updates also skips gclient's implicit bootstrap. + // Initialize its pinned tools without advancing the checkout revision. + run(join(depot, "ensure_bootstrap"), [], depot) checkout(source, "https://webrtc.googlesource.com/src.git", lock.sourceRevision) writeFileSync( join(workspace, ".gclient"), From acd462f846182bdd21f7c5facb806ac0ac410ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 19:43:22 -0700 Subject: [PATCH 06/11] Bundle WebRTC notices and validate pull requests --- .github/workflows/pull-request.yml | 65 + .../swift/CodevisorScreenSharing/README.md | 11 +- .../Resources/WebRTC-ThirdPartyNotices-iOS.md | 2207 +++++++++++++++ .../WebRTC-ThirdPartyNotices-macOS.md | 2443 +++++++++++++++++ packages/swift/Package.swift | 6 +- 5 files changed, 4730 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pull-request.yml create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-iOS.md create mode 100644 packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-macOS.md diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 000000000..811f36595 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,65 @@ +name: Pull Request Checks + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + BUN_VERSION: "1.3.13" + NODE_VERSION: "24.x" + +jobs: + javascript: + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: ${{ env.NODE_VERSION }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: ${{ env.BUN_VERSION }} + - run: bun install --frozen-lockfile + - run: bun run check:js + + native: + if: ${{ !github.event.pull_request.draft }} + # PR code runs on an ephemeral hosted machine, without signing secrets. + runs-on: macos-26 + timeout-minutes: 60 + env: + DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: ${{ env.NODE_VERSION }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: ${{ env.BUN_VERSION }} + - name: Record native tools + run: | + xcodebuild -version + xcrun swift --version + - run: bun install --frozen-lockfile + - run: bun run swift:format:check + - run: bun run swift:lint + - run: bun run swift:test + - run: bun run swift:test:macos-transcript + - run: bun run swift:test:macos-composer + - run: bun run build:macos + - run: bun run swift:build:ios + - run: bun run swift:test:ios-transcript diff --git a/packages/swift/CodevisorScreenSharing/README.md b/packages/swift/CodevisorScreenSharing/README.md index 9ae1d41d3..76c063bb5 100644 --- a/packages/swift/CodevisorScreenSharing/README.md +++ b/packages/swift/CodevisorScreenSharing/README.md @@ -247,7 +247,16 @@ The Codevisor-owned recipe is [scripts/build-webrtc.mjs](../../../scripts/build- A manual [artifact workflow](../../../.github/workflows/webrtc-artifact.yml) uploads a candidate for review without publishing it. Codevisor's recipe now pins Xcode 26.6, deliberately differing from the upstream workflow's 26.5 pin. The earlier refusal of Xcode 27 was an exact-version preflight check, not evidence that 26.6 is incompatible. Python remains pinned to 3.12.14. The complete source build passed on September 14 with Xcode 26.6 (17F113) and Python 3.12.14: macOS arm64/x86_64, iOS arm64 and Simulator arm64/x86_64. Candidate archive SHA-256: `cd477857489c347daddfbfa19bf0d905a2e2a143f3ee84543ba887b97cc13c45`. Graph-derived notices were checked against the exact source license texts across all five builds; the macOS notice has 25 sections and the iOS notice 23, including WebRTC. All 86 macOS public headers match the published dependency byte-for-byte. Architecture slices, privacy manifests, signatures and dSYM UUIDs passed inspection. The unchanged media module compiled against the candidate in an isolated package, all 180 tests passed, and a native factory/source/track/peer/data-channel creation-and-close smoke passed without capture or connection. This is one successful build, not proof of byte reproducibility or a media/performance comparison. The installed SwiftPM dependency remains the published artifact; candidate promotion and actual-app validation remain open. -The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The published archive does not include an aggregate third-party license file. The source-built candidate now includes the audited generated notices, but they have not yet been integrated into the product resource bundle; notice packaging remains a distribution gate. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. +The archive's WebRTC BSD notice is copied into the module resource bundle and the diagnostic app. The framework's supplied privacy manifest is preserved. The published archive does not include an aggregate third-party license file. The exact audited macOS and iOS notice files now ship in `CodevisorKit_CodevisorScreenSharing.bundle` through explicit SwiftPM copy resources, alongside the main WebRTC license. The probe copies that same resource bundle. The notices are generated from the pinned source and matching upstream build arguments; the product continues using the checksum-verified published binary. The existing macOS release script already signs and verifies embedded frameworks inside-out, including WebRTC. A signed/notarized release containing this feature has not been produced or published. + +Notice provenance (generated text is preserved byte-for-byte): + +| Resource | SHA-256 | +| --- | --- | +| `WebRTC-ThirdPartyNotices-macOS.md` | `b31e3534049987e366e146ed4197bd63feb87d99b7a456e1290ce7172b7662df` | +| `WebRTC-ThirdPartyNotices-iOS.md` | `c14b7a9740d38cf418f56d2189e1cf266e61b887166e89a570f7a8bef02c488e` | + +Regenerate with `node scripts/build-webrtc.mjs`; audit `artifacts/notices-macos/LICENSE.md` and `artifacts/notices-ios/LICENSE.md` against the recorded target graphs before copying them to `Sources/CodevisorScreenSharing/Resources/`. The main WebRTC source revision, dependency revisions, GN arguments and build tools are recorded in the recipe output. These notices cover the native macOS and iOS configurations; Mac Catalyst is not a Codevisor target. Control API references: [WebRTC data channels](https://webrtc.org/getting-started/data-channels), [Apple’s private event-source state](https://developer.apple.com/documentation/coregraphics/cgeventsourcestateid), and [event-source user data](https://developer.apple.com/documentation/coregraphics/cgeventsource/userdata). Native channel signatures were checked against the pinned M152 framework headers. diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-iOS.md b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-iOS.md new file mode 100644 index 000000000..5b0eedbe1 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-iOS.md @@ -0,0 +1,2207 @@ +# webrtc +``` +Copyright (c) 2011, The WebRTC project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# abseil-cpp +``` + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +``` + +# boringssl +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# dav1d +``` +Copyright © 2018, VideoLAN and dav1d authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# fft +``` +/* + * Copyright(c)1995,97 Mark Olesen <olesen@me.QueensU.CA> + * Queen's Univ at Kingston (Canada) + * + * Permission to use, copy, modify, and distribute this software for + * any purpose without fee is hereby granted, provided that this + * entire notice is included in all copies of any software which is + * or includes a copy or modification of this software and in all + * copies of the supporting documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR QUEEN'S + * UNIVERSITY AT KINGSTON MAKES ANY REPRESENTATION OR WARRANTY OF ANY + * KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS + * FITNESS FOR ANY PARTICULAR PURPOSE. + * + * All of which is to say that you can do what you like with this + * source code provided you don't try to sell it as your own and you + * include an unaltered copy of this message (including the + * copyright). + * + * It is also implicitly understood that bug fixes and improvements + * should make their way back to the general Internet community so + * that everyone benefits. + */ + +``` + +# fiat +``` +The Apache License, Version 2.0 (Apache-2.0) + +Copyright 2015-2020 the fiat-crypto authors (see the AUTHORS file) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + +# g711 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g711.h - In line A-law and u-law conversion routines + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2001 Steve Underwood + * + * Despite my general liking of the GPL, I place this code in the + * public domain for the benefit of all mankind - even the slimy + * ones who might try to proprietize my work and use it to my + * detriment. + */ + +``` + +# g722 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g722_decode.c - The ITU G.722 codec, decode part. + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2005 Steve Underwood + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based in part on a single channel G.722 codec which is: + * + * Copyright (c) CMU 1993 + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + */ + +``` + +# libaom +``` +Copyright (c) 2016, Alliance for Open Media. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libc++ +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The libc++ library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# libc++abi +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The libc++abi library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# libsrtp +``` +/* + * + * Copyright (c) 2001-2017 Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * Neither the name of the Cisco Systems, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +``` + +# libvpx +``` +Copyright (c) 2010, The WebM Project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google, nor the WebM Project, nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libyuv +``` +Copyright 2011 The LibYuv Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# llvm-libc +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2007-2019 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +``` + +# nasm +``` +NASM is now licensed under the 2-clause BSD license, also known as the +simplified BSD license. + +SPDX-License-Identifier: BSD-2-Clause + + Copyright 1996-2025 the NASM Authors - All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following + conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# ooura +``` +/* + * http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html + * Copyright Takuya OOURA, 1996-2001 + * + * You may use, copy, modify and distribute this code for any purpose (include + * commercial use) and without fee. Please refer to this package when you modify + * this code. + */ + +``` + +# opus +``` +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, + Jean-Marc Valin, Timothy B. Terriberry, + CSIRO, Gregory Maxwell, Mark Borgerding, + Erik de Castro Lopo, Mozilla, Amazon + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses which are +specified at: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ + +``` + +# perfetto +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +------------------ + +Files: * except those files noted below + + Copyright (c) 2017, The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/stdlib/chromium/*, protos/third_party/chromium/*, test/trace_processor/diff_tests/stdlib/chrome/* + + Copyright 2015 The Chromium Authors + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------ + +Files: src/trace_processor/perfetto_sql/syntaqlite/syntaqlite_perfetto.{c, h} + + Copyright 2025 The syntaqlite Authors. All rights reserved. + + Machine-generated amalgamation of syntaqlite runtime + Perfetto dialect + sources (https://github.com/LalitMaganti/syntaqlite). Portions derive + from SQLite's public-domain `parse.y` grammar. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.{c, h} + +The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: + +May you do good and not evil. +May you find forgiveness for yourself and forgive others. +May you share freely, never taking more than you give. + + +------------------ + +Files: src/base/intrusive_tree.cc + +$OpenBSD: tree.h,v 1.31 2023/03/08 04:43:09 guenther Exp $ + +Copyright 2002 Niels Provos <provos@citi.umich.edu> +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# pffft +``` +Copyright (c) 2013 Julien Pommier ( pommier@modartt.com ) + +Based on original fortran 77 code from FFTPACKv4 from NETLIB, +authored by Dr Paul Swarztrauber of NCAR, in 1985. + +As confirmed by the NCAR fftpack software curators, the following +FFTPACKv5 license applies to FFTPACKv4 sources. My changes are +released under the same terms. + +FFTPACK license: + +http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html + +Copyright (c) 2004 the University Corporation for Atmospheric +Research ("UCAR"). All rights reserved. Developed by NCAR's +Computational and Information Systems Laboratory, UCAR, +www.cisl.ucar.edu. + +Redistribution and use of the Software in source and binary forms, +with or without modification, is permitted provided that the +following conditions are met: + +- Neither the names of NCAR's Computational and Information Systems +Laboratory, the University Corporation for Atmospheric Research, +nor the names of its sponsors or contributors may be used to +endorse or promote products derived from this Software without +specific prior written permission. + +- Redistributions of source code must retain the above copyright +notices, this list of conditions, and the disclaimer below. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions, and the disclaimer below in the +documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +``` + +# protobuf +``` +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +``` + +# rnnoise +``` +Copyright (c) 2017, Mozilla +Copyright (c) 2007-2017, Jean-Marc Valin +Copyright (c) 2005-2017, Xiph.Org Foundation +Copyright (c) 2003-2004, Mark Borgerding + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of the Xiph.Org Foundation nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# spl_sqrt_floor +``` +/* + * Written by Wilco Dijkstra, 1996. The following email exchange establishes the + * license. + * + * From: Wilco Dijkstra <Wilco.Dijkstra@ntlworld.com> + * Date: Fri, Jun 24, 2011 at 3:20 AM + * Subject: Re: sqrt routine + * To: Kevin Ma <kma@google.com> + * Hi Kevin, + * Thanks for asking. Those routines are public domain (originally posted to + * comp.sys.arm a long time ago), so you can use them freely for any purpose. + * Cheers, + * Wilco + * + * ----- Original Message ----- + * From: "Kevin Ma" <kma@google.com> + * To: <Wilco.Dijkstra@ntlworld.com> + * Sent: Thursday, June 23, 2011 11:44 PM + * Subject: Fwd: sqrt routine + * Hi Wilco, + * I saw your sqrt routine from several web sites, including + * http://www.finesse.demon.co.uk/steven/sqrt.html. + * Just wonder if there's any copyright information with your Successive + * approximation routines, or if I can freely use it for any purpose. + * Thanks. + * Kevin + */ + +``` + diff --git a/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-macOS.md b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-macOS.md new file mode 100644 index 000000000..9baee6a87 --- /dev/null +++ b/packages/swift/CodevisorScreenSharing/Sources/CodevisorScreenSharing/Resources/WebRTC-ThirdPartyNotices-macOS.md @@ -0,0 +1,2443 @@ +# webrtc +``` +Copyright (c) 2011, The WebRTC project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# abseil-cpp +``` + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +``` + +# boringssl +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# dav1d +``` +Copyright © 2018, VideoLAN and dav1d authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# fft +``` +/* + * Copyright(c)1995,97 Mark Olesen <olesen@me.QueensU.CA> + * Queen's Univ at Kingston (Canada) + * + * Permission to use, copy, modify, and distribute this software for + * any purpose without fee is hereby granted, provided that this + * entire notice is included in all copies of any software which is + * or includes a copy or modification of this software and in all + * copies of the supporting documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR QUEEN'S + * UNIVERSITY AT KINGSTON MAKES ANY REPRESENTATION OR WARRANTY OF ANY + * KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS + * FITNESS FOR ANY PARTICULAR PURPOSE. + * + * All of which is to say that you can do what you like with this + * source code provided you don't try to sell it as your own and you + * include an unaltered copy of this message (including the + * copyright). + * + * It is also implicitly understood that bug fixes and improvements + * should make their way back to the general Internet community so + * that everyone benefits. + */ + +``` + +# fiat +``` +The Apache License, Version 2.0 (Apache-2.0) + +Copyright 2015-2020 the fiat-crypto authors (see the AUTHORS file) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + +# g711 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g711.h - In line A-law and u-law conversion routines + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2001 Steve Underwood + * + * Despite my general liking of the GPL, I place this code in the + * public domain for the benefit of all mankind - even the slimy + * ones who might try to proprietize my work and use it to my + * detriment. + */ + +``` + +# g722 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g722_decode.c - The ITU G.722 codec, decode part. + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2005 Steve Underwood + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based in part on a single channel G.722 codec which is: + * + * Copyright (c) CMU 1993 + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + */ + +``` + +# libaom +``` +Copyright (c) 2016, Alliance for Open Media. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libc++ +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The libc++ library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# libc++abi +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The libc++abi library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# libjpeg_turbo +``` +libjpeg-turbo Licenses +====================== + +libjpeg-turbo is covered by two compatible BSD-style open source licenses: + +- The IJG (Independent JPEG Group) License, which is listed in + [README.ijg](README.ijg) + + This license applies to the libjpeg API library and associated programs, + including any code inherited from libjpeg and any modifications to that + code. Note that the libjpeg-turbo SIMD source code bears the + [zlib License](https://opensource.org/licenses/Zlib), but in the context of + the overall libjpeg API library, the terms of the zlib License are subsumed + by the terms of the IJG License. + +- The Modified (3-clause) BSD License, which is listed below + + This license applies to the TurboJPEG API library and associated programs, as + well as the build system. Note that the TurboJPEG API library wraps the + libjpeg API library, so in the context of the overall TurboJPEG API library, + both the terms of the IJG License and the terms of the Modified (3-clause) + BSD License apply. + + +Complying with the libjpeg-turbo Licenses +========================================= + +This section provides a roll-up of the libjpeg-turbo licensing terms, to the +best of our understanding. This is not a license in and of itself. It is +intended solely for clarification. + +1. If you are distributing a modified version of the libjpeg-turbo source, + then: + + 1. You cannot alter or remove any existing copyright or license notices + from the source. + + **Origin** + - Clause 1 of the IJG License + - Clause 1 of the Modified BSD License + - Clauses 1 and 3 of the zlib License + + 2. You must add your own copyright notice to the header of each source + file you modified, so others can tell that you modified that file. (If + there is not an existing copyright header in that file, then you can + simply add a notice stating that you modified the file.) + + **Origin** + - Clause 1 of the IJG License + - Clause 2 of the zlib License + + 3. You must include the IJG README file, and you must not alter any of the + copyright or license text in that file. + + **Origin** + - Clause 1 of the IJG License + +2. If you are distributing only libjpeg-turbo binaries without the source, or + if you are distributing an application that statically links with + libjpeg-turbo, then: + + 1. Your product documentation must include a message stating: + + This software is based in part on the work of the Independent JPEG + Group. + + **Origin** + - Clause 2 of the IJG license + + 2. If your binary distribution includes or uses the TurboJPEG API, then + your product documentation must include the text of the Modified BSD + License (see below.) + + **Origin** + - Clause 2 of the Modified BSD License + +3. You cannot use the name of the IJG or The libjpeg-turbo Project or the + contributors thereof in advertising, publicity, etc. + + **Origin** + - IJG License + - Clause 3 of the Modified BSD License + +4. The IJG and The libjpeg-turbo Project do not warrant libjpeg-turbo to be + free of defects, nor do we accept any liability for undesirable + consequences resulting from your use of the software. + + **Origin** + - IJG License + - Modified BSD License + - zlib License + + +The Modified (3-clause) BSD License +=================================== + +Copyright (C)2009-2024 D. R. Commander. All Rights Reserved.<br> +Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +- Neither the name of the libjpeg-turbo Project nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS", +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +Why Two Licenses? +================= + +The zlib License could have been used instead of the Modified (3-clause) BSD +License, and since the IJG License effectively subsumes the distribution +conditions of the zlib License, this would have effectively placed +libjpeg-turbo binary distributions under the IJG License. However, the IJG +License specifically refers to the Independent JPEG Group and does not extend +attribution and endorsement protections to other entities. Thus, it was +desirable to choose a license that granted us the same protections for new code +that were granted to the IJG for code derived from their software. + +``` + +# libsrtp +``` +/* + * + * Copyright (c) 2001-2017 Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * Neither the name of the Cisco Systems, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +``` + +# libvpx +``` +Copyright (c) 2010, The WebM Project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google, nor the WebM Project, nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libyuv +``` +Copyright 2011 The LibYuv Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# llvm-libc +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2007-2019 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +``` + +# nasm +``` +NASM is now licensed under the 2-clause BSD license, also known as the +simplified BSD license. + +SPDX-License-Identifier: BSD-2-Clause + + Copyright 1996-2025 the NASM Authors - All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following + conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# ooura +``` +/* + * http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html + * Copyright Takuya OOURA, 1996-2001 + * + * You may use, copy, modify and distribute this code for any purpose (include + * commercial use) and without fee. Please refer to this package when you modify + * this code. + */ + +``` + +# opus +``` +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, + Jean-Marc Valin, Timothy B. Terriberry, + CSIRO, Gregory Maxwell, Mark Borgerding, + Erik de Castro Lopo, Mozilla, Amazon + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses which are +specified at: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ + +``` + +# perfetto +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +------------------ + +Files: * except those files noted below + + Copyright (c) 2017, The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/stdlib/chromium/*, protos/third_party/chromium/*, test/trace_processor/diff_tests/stdlib/chrome/* + + Copyright 2015 The Chromium Authors + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------ + +Files: src/trace_processor/perfetto_sql/syntaqlite/syntaqlite_perfetto.{c, h} + + Copyright 2025 The syntaqlite Authors. All rights reserved. + + Machine-generated amalgamation of syntaqlite runtime + Perfetto dialect + sources (https://github.com/LalitMaganti/syntaqlite). Portions derive + from SQLite's public-domain `parse.y` grammar. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.{c, h} + +The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: + +May you do good and not evil. +May you find forgiveness for yourself and forgive others. +May you share freely, never taking more than you give. + + +------------------ + +Files: src/base/intrusive_tree.cc + +$OpenBSD: tree.h,v 1.31 2023/03/08 04:43:09 guenther Exp $ + +Copyright 2002 Niels Provos <provos@citi.umich.edu> +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# pffft +``` +Copyright (c) 2013 Julien Pommier ( pommier@modartt.com ) + +Based on original fortran 77 code from FFTPACKv4 from NETLIB, +authored by Dr Paul Swarztrauber of NCAR, in 1985. + +As confirmed by the NCAR fftpack software curators, the following +FFTPACKv5 license applies to FFTPACKv4 sources. My changes are +released under the same terms. + +FFTPACK license: + +http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html + +Copyright (c) 2004 the University Corporation for Atmospheric +Research ("UCAR"). All rights reserved. Developed by NCAR's +Computational and Information Systems Laboratory, UCAR, +www.cisl.ucar.edu. + +Redistribution and use of the Software in source and binary forms, +with or without modification, is permitted provided that the +following conditions are met: + +- Neither the names of NCAR's Computational and Information Systems +Laboratory, the University Corporation for Atmospheric Research, +nor the names of its sponsors or contributors may be used to +endorse or promote products derived from this Software without +specific prior written permission. + +- Redistributions of source code must retain the above copyright +notices, this list of conditions, and the disclaimer below. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions, and the disclaimer below in the +documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +``` + +# portaudio +``` +/* + * $Id: pa_memorybarrier.h 1240 2007-07-17 13:05:07Z bjornroche $ + * Portable Audio I/O Library + * Memory barrier utilities + * + * Author: Bjorn Roche, XO Audio, LLC + * + * This program uses the PortAudio Portable Audio Library. + * For more information see: http://www.portaudio.com + * Copyright (c) 1999-2000 Ross Bencina and Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + +/* + * $Id: pa_ringbuffer.c 1421 2009-11-18 16:09:05Z bjornroche $ + * Portable Audio I/O Library + * Ring Buffer utility. + * + * Author: Phil Burk, http://www.softsynth.com + * modified for SMP safety on Mac OS X by Bjorn Roche + * modified for SMP safety on Linux by Leland Lucius + * also, allowed for const where possible + * modified for multiple-byte-sized data elements by Sven Fischer + * + * Note that this is safe only for a single-thread reader and a + * single-thread writer. + * + * This program uses the PortAudio Portable Audio Library. + * For more information see: http://www.portaudio.com + * Copyright (c) 1999-2000 Ross Bencina and Phil Burk + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * The text above constitutes the entire PortAudio license; however, + * the PortAudio community also makes the following non-binding requests: + * + * Any person wishing to distribute modifications to the Software is + * requested to send the modifications to the original developer so that + * they can be incorporated into the canonical version. It is also + * requested that these non-binding requests be included along with the + * license above. + */ + + +``` + +# protobuf +``` +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +``` + +# rnnoise +``` +Copyright (c) 2017, Mozilla +Copyright (c) 2007-2017, Jean-Marc Valin +Copyright (c) 2005-2017, Xiph.Org Foundation +Copyright (c) 2003-2004, Mark Borgerding + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of the Xiph.Org Foundation nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# spl_sqrt_floor +``` +/* + * Written by Wilco Dijkstra, 1996. The following email exchange establishes the + * license. + * + * From: Wilco Dijkstra <Wilco.Dijkstra@ntlworld.com> + * Date: Fri, Jun 24, 2011 at 3:20 AM + * Subject: Re: sqrt routine + * To: Kevin Ma <kma@google.com> + * Hi Kevin, + * Thanks for asking. Those routines are public domain (originally posted to + * comp.sys.arm a long time ago), so you can use them freely for any purpose. + * Cheers, + * Wilco + * + * ----- Original Message ----- + * From: "Kevin Ma" <kma@google.com> + * To: <Wilco.Dijkstra@ntlworld.com> + * Sent: Thursday, June 23, 2011 11:44 PM + * Subject: Fwd: sqrt routine + * Hi Wilco, + * I saw your sqrt routine from several web sites, including + * http://www.finesse.demon.co.uk/steven/sqrt.html. + * Just wonder if there's any copyright information with your Successive + * approximation routines, or if I can freely use it for any purpose. + * Thanks. + * Kevin + */ + +``` + diff --git a/packages/swift/Package.swift b/packages/swift/Package.swift index 14d3bb70d..7b8b4a172 100644 --- a/packages/swift/Package.swift +++ b/packages/swift/Package.swift @@ -38,7 +38,11 @@ let package = Package( name: "CodevisorScreenSharing", dependencies: [.product(name: "WebRTC", package: "WebRTC")], path: "CodevisorScreenSharing/Sources/CodevisorScreenSharing", - resources: [.copy("Resources/WebRTC-LICENSE.txt")], + resources: [ + .copy("Resources/WebRTC-LICENSE.txt"), + .copy("Resources/WebRTC-ThirdPartyNotices-macOS.md"), + .copy("Resources/WebRTC-ThirdPartyNotices-iOS.md"), + ], swiftSettings: [.swiftLanguageMode(.v6)] ), .executableTarget( From d627587d06abe644a9a4424986d2e717083f5391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 19:50:13 -0700 Subject: [PATCH 07/11] Install pinned SwiftLint for hosted native checks --- .github/workflows/pull-request.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 811f36595..69989861d 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -55,6 +55,16 @@ jobs: xcodebuild -version xcrun swift --version - run: bun install --frozen-lockfile + - name: Install the validated SwiftLint version + run: | + tool_dir="$RUNNER_TEMP/swiftlint-0.63.1" + mkdir -p "$tool_dir" + curl --fail --location --retry 3 --output "$tool_dir/swiftlint.zip" \ + https://github.com/realm/SwiftLint/releases/download/0.63.1/portable_swiftlint.zip + echo "0e6369741b694b701e9bd4e4fe9b408a5946d4bb076b79ea6ec2bce428475739 $tool_dir/swiftlint.zip" | shasum -a 256 -c - + ditto -x -k "$tool_dir/swiftlint.zip" "$tool_dir" + "$tool_dir/swiftlint" version + echo "$tool_dir" >> "$GITHUB_PATH" - run: bun run swift:format:check - run: bun run swift:lint - run: bun run swift:test From 88d074da8e3d9182b083232dd105d215aba64646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 19:57:20 -0700 Subject: [PATCH 08/11] Record MVP package verification and remaining gates --- docs/plans/native-screen-sharing-mvp.md | 12 +++++++++--- docs/plans/native-screen-sharing.md | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index 28048b343..b13cf71a1 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -22,6 +22,8 @@ Actual-app checks on September 14 established remote video and control, includin Commit `dbdbcaea` passed the normal pre-commit hook in a clean validation worktree, without exclusions: the full JavaScript check chain, Swift formatting/lint, 2,119 Swift package tests, 30 macOS transcript tests, 11 macOS composer tests, the iOS build, and 8 iOS transcript tests. The macOS development app built successfully. The September 14 integration adopts main's center-only workspace layout and focus-source ownership, and preserves chatless workspace behavior. +Main integration `166fce13` brings in `10bdc699`'s attachment-thumbnail framing fix without conflicts. That merge, notice/CI commit `acd462f8`, and CI tooling correction `d627587d` each passed the complete normal pre-commit hook without exclusions. The combined macOS build passed, and the resource bundle's two dependency-notice files match the audited source hashes. + Actual-app checks with the final viewer established that the complete toolbar stays interactive during connection, View/Control choices survive connection and menu interactions, and switching modes no longer shows a spinner. Those checks used the earlier host build. The isolated host was subsequently rebuilt from the matching final source; all 2,864 staged source files were verified and the app passed strict signature verification. The fresh acceptance attempt reached the host Screen Recording permission error. The remaining two-Mac workflows and sustained-use check are pending reauthorization of this rebuilt development app; earlier results do not replace that acceptance pass. The local measurement archives contain raw session and machine-specific evidence and are intentionally excluded from this public checkpoint. Historical measurement links in the implementation notes refer to those local archives. No credentials, raw signaling, desktop captures or runner logs are required to build the feature. @@ -30,13 +32,17 @@ The local measurement archives contain raw session and machine-specific evidence The pinned stasel WebRTC 152.0.0 archive was downloaded again on September 14. Its SHA256 matches `scripts/webrtc-build.lock.json` and the SwiftPM checksum: `115cb9944248a3302c0c8af17462e2576a28ccc7adef9f6a1fe66ee75d9e1cc8`. All 387 regular files and 10 symlink targets match the resolved artifact. The XCFramework provides macOS arm64/x86_64, iOS device and simulator slices, and Mac Catalyst slices. -The built macOS app includes the WebRTC license and privacy manifest. The app and its embedded WebRTC framework pass strict signature verification. The release script signs embedded frameworks before signing and verifying the enclosing app; Developer ID signing and notarization of this final revision have not been executed. +The product now bundles the main WebRTC license, audited macOS/iOS third-party notices and the framework's privacy manifest. The notices are explicit SwiftPM copy resources, with provenance and hashes in the [library README](../../packages/swift/CodevisorScreenSharing/README.md). The installed binary dependency is unchanged. The published archive contains the main WebRTC license but no aggregate third-party notices. On September 14, the Xcode pin was deliberately updated from 26.5 to 26.6 (17F113), with Python 3.12.14 unchanged. The first source-build attempt found a missing depot_tools bootstrap before GN; the corrected recipe initializes the pinned tools without updating their revision. The subsequent complete build passed for all five macOS/iOS architectures and packaged a candidate XCFramework (`cd477857489c347daddfbfa19bf0d905a2e2a143f3ee84543ba887b97cc13c45`). The generated notices match the exact source license texts for every mapped dependency in all five target graphs. Headers, slices, signatures, privacy manifests and matching dSYM UUIDs passed inspection; all 180 unchanged screen-sharing tests passed against the isolated candidate, as did factory/track/peer/data-channel creation and close. No capture or media connection was run with it. -The WebRTC source revision and installed product dependency remain unchanged. The candidate includes audited notices, but the product resource bundle still needs their integration and packaging verification. Actual-app validation of the candidate, its promotion, and Developer ID signing/notarization remain open. A successful source build does not establish byte reproducibility or performance equivalence. +An unpublished Developer ID packaging run of `acd462f8` completed through the ordinary release script with version `0.0.0`. Both architecture-specific ZIPs and DMGs passed verification. The apps and embedded WebRTC framework have Developer ID signatures, hardened runtime and secure timestamps; both ZIPs preserve the exact audited notice bytes and privacy manifest, and both DMG checksums/signatures passed. The signed ARM runtime loaded the screen-sharing API and exercised an in-memory SQLite database. No app UI or media session was launched by this verification. + +This was a local packaging check using the installed Xcode 27 beta; the hosted native PR check is pinned to Xcode 26.6. The ARM runtime was rebuilt from this revision. For Intel, only the unchanged native Node/dependency tree was reused from main's successful CI run at `10bdc699`, with current JavaScript and resources. All dependency manifests and the lockfile matched (the root manifest differs only by the development probe script); 2,019 current non-native files and all eight Intel Mach-O files were checked. Intel execution was unavailable without Rosetta. These review artifacts were not published and do not replace native release builds. Notarization was not attempted because this session has no notarization credentials. + +The source-built WebRTC candidate remains isolated and unpromoted; actual-app validation is required before replacing the installed dependency. A successful source build does not establish byte reproducibility or performance equivalence. -Pushing to `main` automatically starts the Alpha build and publication workflow, so the dependency/distribution gate cannot be deferred to a later manual release. The PR currently has no completed GitHub build/test workflow; its correctness check is skipped while in draft. The full-check result above is local verification, not a CI result. +Pushing to `main` automatically starts the Alpha build and publication workflow, so final CI, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is now ready for review, with non-publishing GitHub-hosted JavaScript and native checks. Native validation uses Xcode 26.6 and checksum-pinned SwiftLint 0.63.1; the first hosted run exposed the missing linter, which was corrected. Check the final head's live results on the PR rather than treating local hooks as CI. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. The PR has not been merged. ## Before MVP acceptance diff --git a/docs/plans/native-screen-sharing.md b/docs/plans/native-screen-sharing.md index 99499035b..f574e3928 100644 --- a/docs/plans/native-screen-sharing.md +++ b/docs/plans/native-screen-sharing.md @@ -109,7 +109,8 @@ Milestones are sequential at their interfaces. Milestone 0 can be split into dep - [x] Keep requested recovery keyframes pending across encoder admission drops; explicit decoder-state-loss checks pass for H.264 and Main444, including 4K encoder backpressure. This is local fault injection, not loss acceptance on a real network. - [x] Pinned source-build recipe and manual artifact workflow prepared; published dependency checksum/slices verified. - [x] Complete the pinned source build with Xcode 26.6 and audit generated third-party notices: all five architectures, source-license text checks, slices/signatures/privacy/dSYM inspection, 180 unchanged media tests against the isolated candidate, and a native factory/peer smoke passed September 14. The toolchain pin deliberately changed from 26.5; WebRTC source and the installed dependency are unchanged. See the [MVP checkpoint](native-screen-sharing-mvp.md). -- [ ] Integrate the audited notices into product packaging and validate the source-built candidate in actual apps before promotion. No candidate media/performance claim or byte-reproducibility claim is made. +- [x] Integrate the audited notices into product packaging. The combined macOS build and unpublished Developer ID ZIP/DMG verification passed; notices retain the audited hashes. See the [MVP checkpoint](native-screen-sharing-mvp.md) for toolchain, Intel-runtime and notarization limits. +- [ ] Validate the isolated source-built candidate in actual apps before replacing the installed dependency. No candidate media/performance claim or byte-reproducibility claim is made. - [ ] Run the specified two-Mac LAN editing/terminal/browser and input-to-photon checks, including rotated/external displays and non-US input methods. - [ ] Validate separate networks, forced TURN, UDP restrictions and route changes against a deployed relay. - [ ] Complete isolated wired-LAN and 30-minute resource/latency comparisons, matched 1080p Apple runs, quantitative text fidelity and external input-to-photon measurements. From 1b49def4fd1d9a940d2e25357d855167fec231c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 20:07:45 -0700 Subject: [PATCH 09/11] Await video-writer test callbacks without blocking Swift workers --- .github/workflows/pull-request.yml | 1 + docs/plans/native-screen-sharing-mvp.md | 2 +- .../ComputerUseVideoWriterTests.swift | 35 +++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 69989861d..1f5aef7f2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -68,6 +68,7 @@ jobs: - run: bun run swift:format:check - run: bun run swift:lint - run: bun run swift:test + timeout-minutes: 15 - run: bun run swift:test:macos-transcript - run: bun run swift:test:macos-composer - run: bun run build:macos diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index b13cf71a1..a3abe7cdb 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -42,7 +42,7 @@ This was a local packaging check using the installed Xcode 27 beta; the hosted n The source-built WebRTC candidate remains isolated and unpromoted; actual-app validation is required before replacing the installed dependency. A successful source build does not establish byte reproducibility or performance equivalence. -Pushing to `main` automatically starts the Alpha build and publication workflow, so final CI, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is now ready for review, with non-publishing GitHub-hosted JavaScript and native checks. Native validation uses Xcode 26.6 and checksum-pinned SwiftLint 0.63.1; the first hosted run exposed the missing linter, which was corrected. Check the final head's live results on the PR rather than treating local hooks as CI. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. The PR has not been merged. +Pushing to `main` automatically starts the Alpha build and publication workflow, so final CI, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is now ready for review, with non-publishing GitHub-hosted JavaScript and native checks. Native validation uses Xcode 26.6 and checksum-pinned SwiftLint 0.63.1; the first hosted run exposed the missing linter, which was corrected. Check the final head's live results on the PR rather than treating local hooks as CI. A hosted run also exposed a timeout in the existing MP4 integration test. Its fixture now awaits SDK callbacks asynchronously, propagates startup errors and cancels the writer during cleanup; it retains all MP4 duration/dimension/final-frame assertions. This removes blocking bridge-worker waits from an async test but does not by itself establish the hosted failure's cause. The package-test step has a 15-minute deadlock bound. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. The PR has not been merged. ## Before MVP acceptance diff --git a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ComputerUseVideoWriterTests.swift b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ComputerUseVideoWriterTests.swift index 8e11c464e..c7e3703c4 100644 --- a/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ComputerUseVideoWriterTests.swift +++ b/packages/swift/CodevisorCoreMac/Tests/CodevisorCoreMacTests/ComputerUseVideoWriterTests.swift @@ -4,18 +4,26 @@ import Foundation import Testing @testable import CodevisorCoreMac -@Suite("Computer Use video files") +@Suite("Computer Use video files", .timeLimit(.minutes(1))) struct ComputerUseVideoWriterTests { @Test("A static screen remains visible for the full recording duration in the MP4") func staticDuration() async throws { let url = FileManager.default.temporaryDirectory.appendingPathComponent("video-\(UUID().uuidString).mp4") defer { try? FileManager.default.removeItem(at: url) } - let completed = ComputerUseRecordingResult() - let started = ComputerUseRecordingResult() + let completed = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + let started = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(1)) let writer = try ComputerUseVideoWriter( url: url, size: CGSize(width: 640, height: 480), fps: 30, - onStarted: { started.complete(.success(true)) }, onFinished: { completed.complete(.success($0)) }, - onError: { completed.complete(.failure($0)) }) + onStarted: { + started.continuation.yield(true); started.continuation.finish() + }, + onFinished: { + completed.continuation.yield($0); completed.continuation.finish() + }, + onError: { + started.continuation.finish(throwing: $0); completed.continuation.finish(throwing: $0) + }) + defer { writer.cancel(); writer.queue.sync {} } var pixel: CVPixelBuffer? #expect(CVPixelBufferCreate(nil, 640, 480, kCVPixelFormatType_32BGRA, nil, &pixel) == kCVReturnSuccess) let frame = try #require(pixel) @@ -23,9 +31,9 @@ struct ComputerUseVideoWriterTests { memset(CVPixelBufferGetBaseAddress(frame), 128, CVPixelBufferGetDataSize(frame)) CVPixelBufferUnlockBaseAddress(frame, []) writer.queue.sync { writer.append(frame, at: CMTime(seconds: 100, preferredTimescale: 600)) } - #expect(try started.wait()) + #expect(try await started.stream.first(where: { _ in true }) == true) writer.finish(duration: 3) - #expect(try completed.wait() >= 3) + #expect(try #require(try await completed.stream.first(where: { _ in true })) >= 3) let asset = AVURLAsset(url: url) let duration = try await asset.load(.duration) #expect(abs(CMTimeGetSeconds(duration) - 3) < 0.1) @@ -37,14 +45,19 @@ struct ComputerUseVideoWriterTests { } @Test("A recording with no captured frames fails instead of publishing an empty MP4") - func noFrames() throws { + func noFrames() async throws { let url = FileManager.default.temporaryDirectory.appendingPathComponent("video-\(UUID().uuidString).mp4") defer { try? FileManager.default.removeItem(at: url) } - let completed = ComputerUseRecordingResult() + let completed = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingNewest(1)) let writer = try ComputerUseVideoWriter( url: url, size: CGSize(width: 640, height: 480), fps: 30, - onStarted: {}, onFinished: { completed.complete(.success($0)) }, onError: { completed.complete(.failure($0)) }) + onStarted: {}, + onFinished: { + completed.continuation.yield($0); completed.continuation.finish() + }, + onError: { completed.continuation.finish(throwing: $0) }) + defer { writer.cancel(); writer.queue.sync {} } writer.finish(duration: 3) - #expect(throws: BridgeError.self) { try completed.wait() } + await #expect(throws: BridgeError.self) { try await completed.stream.first(where: { _ in true }) } } } From d455c353e8f1bc0f40082b20ba50150a3f47a79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 20:30:30 -0700 Subject: [PATCH 10/11] Suspend asynchronous test gates without blocking Swift workers --- docs/plans/native-screen-sharing-mvp.md | 2 + .../CloudDirectPathControllerTests.swift | 2 +- .../CloudHubConnectionTests.swift | 135 +--------------- .../CloudHubFlowControlTests.swift | 2 +- .../CloudHubReconnectTests.swift | 148 ++++++++++++++++++ .../CloudRelayCancellationTests.swift | 2 +- .../CloudRelayLoopbackBridgeTests.swift | 2 +- .../CountingCredentialStore.swift | 3 +- .../ScriptedHttpMachine.swift | 2 +- .../LatestValuePreparationWorker.swift | 6 +- packages/swift/Package.swift | 1 + .../Views/StreamingSegmenter.swift | 7 +- .../LatestValuePreparationWorkerTests.swift | 8 +- .../StreamingSegmenterTests.swift | 14 +- .../TranscriptActiveProjectionWorker.swift | 4 +- ...ranscriptActiveProjectionWorkerTests.swift | 14 +- 16 files changed, 186 insertions(+), 166 deletions(-) create mode 100644 packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubReconnectTests.swift diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index a3abe7cdb..6a8409b10 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -44,6 +44,8 @@ The source-built WebRTC candidate remains isolated and unpromoted; actual-app va Pushing to `main` automatically starts the Alpha build and publication workflow, so final CI, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is now ready for review, with non-publishing GitHub-hosted JavaScript and native checks. Native validation uses Xcode 26.6 and checksum-pinned SwiftLint 0.63.1; the first hosted run exposed the missing linter, which was corrected. Check the final head's live results on the PR rather than treating local hooks as CI. A hosted run also exposed a timeout in the existing MP4 integration test. Its fixture now awaits SDK callbacks asynchronously, propagates startup errors and cancels the writer during cleanup; it retains all MP4 duration/dimension/final-frame assertions. This removes blocking bridge-worker waits from an async test but does not by itself establish the hosted failure's cause. The package-test step has a 15-minute deadlock bound. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. The PR has not been merged. +A local single-worker reproduction subsequently identified two independent test scheduling defects: preparation fixtures blocked Swift workers with semaphores while waiting for other Swift tasks, and cloud fixtures retried closed sockets with zero virtual delay. Preparation seams now allow asynchronous test gates; cloud fixtures suspend on a positive virtual backoff that reconnect tests advance explicitly. Production parsing, latest-value selection and reconnect policy are unchanged. The full package suite completes with `LIBDISPATCH_COOPERATIVE_POOL_STRICT=1`; this reproducer result is separate from the final hosted CI verdict. + ## Before MVP acceptance 1. Verify the final source in two actual apps: permissions, viewing, typing, clicking, dragging, scrolling, clipboard, scaling and display selection. diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudDirectPathControllerTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudDirectPathControllerTests.swift index 106212030..3f25ebe8a 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudDirectPathControllerTests.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudDirectPathControllerTests.swift @@ -167,7 +167,7 @@ struct CloudDirectPathControllerTests { webSocketTransport: FakeWebSocketTransport { _ in scriptedHub.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) let relayEndpoint = CloudRelayEndpoint( hub: hub, diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubConnectionTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubConnectionTests.swift index 487a6e4fb..c8811da74 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubConnectionTests.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubConnectionTests.swift @@ -39,7 +39,7 @@ struct CloudHubConnectionTests { webSocketTransport: FakeWebSocketTransport { _ in scripted.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) try await hub.waitUntilReady() @@ -61,39 +61,6 @@ struct CloudHubConnectionTests { await hub.shutdown() } - @Test("Hub reconnects reuse the credential snapshot") - func reconnectUsesCredentialSnapshot() async throws { - let first = ScriptedCloudHub() - let second = ScriptedCloudHub() - let sockets = SocketQueue([first.socket, second.socket]) - let transport = FakeWebSocketTransport { _ in sockets.next() } - let memory = InMemoryCloudCredentialStore(token: "session-token") - try memory.saveAppDeviceId("app-device") - try memory.saveAppSecretKey(Data(repeating: 7, count: 32)) - let store = CountingCredentialStore(base: memory) - let hub = CloudHubConnection( - serverURL: URL(string: "https://cloud.example.com")!, - credentialStore: store, - deviceName: "Test App", - deviceOS: "macOS", - webSocketTransport: transport, - readyTimeout: .seconds(3), - sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } - ) - - try await hub.waitUntilReady() - first.socket.disconnect() - #expect(await waitUntil { transport.requests.count == 2 }) - try await hub.waitUntilReady() - - let counts = store.readCounts - #expect(counts.token == 1) - #expect(counts.deviceId == 1) - #expect(counts.secretKey == 1) - await hub.shutdown() - } - @Test("Channel opens park while a known machine is offline") func offlineMachineParksChannelOpen() async throws { let machine = ScriptedRelayMachine() @@ -259,39 +226,6 @@ struct CloudHubConnectionTests { await hub.shutdown() } - @Test("A missing heartbeat pong replaces a half-open socket") - func heartbeatTimeoutReconnects() async throws { - let first = ScriptedCloudHub() - first.respondsToPing = false - let second = ScriptedCloudHub() - let sockets = SocketQueue([first.socket, second.socket]) - let transport = FakeWebSocketTransport { _ in sockets.next() } - let store = InMemoryCloudCredentialStore(token: "session-token") - let clock = TestClock() - let hub = CloudHubConnection( - serverURL: URL(string: "https://cloud.example.com")!, - credentialStore: store, - deviceName: "Test App", - deviceOS: "macOS", - webSocketTransport: transport, - readyTimeout: .seconds(2), - heartbeatInterval: .seconds(30), - heartbeatTimeout: .seconds(10), - sleep: clock.sleep, - reconnectDelay: { _ in .zero } - ) - - try await hub.waitUntilReady() - await clock.waitForSleep(.seconds(30)) - clock.advance(by: .seconds(30)) - await clock.waitForSleep(.seconds(10)) - #expect(transport.requests.count == 1) - clock.advance(by: .seconds(10)) - #expect(await waitUntil { transport.requests.count == 2 }) - try await hub.waitUntilReady() - await hub.shutdown() - } - @Test("Keepalive pongs record the relay RTT") func keepaliveMeasuresRtt() async throws { let scripted = ScriptedCloudHub() @@ -307,7 +241,7 @@ struct CloudHubConnectionTests { heartbeatTimeout: .seconds(2), sleep: clock.sleep, now: { clock.now }, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) try await hub.waitUntilReady() @@ -321,65 +255,6 @@ struct CloudHubConnectionTests { await hub.shutdown() } - @Test("An outbound send failure replaces the hub socket") - func sendFailureReconnects() async throws { - let machine = ScriptedRelayMachine() - let first = ScriptedCloudHub(machines: [machine.presence]) - let second = ScriptedCloudHub(machines: [machine.presence]) - let sockets = SocketQueue([first.socket, second.socket]) - let transport = FakeWebSocketTransport { _ in sockets.next() } - let store = InMemoryCloudCredentialStore(token: "session-token") - let hub = CloudHubConnection( - serverURL: URL(string: "https://cloud.example.com")!, - credentialStore: store, - deviceName: "Test App", - deviceOS: "macOS", - webSocketTransport: transport, - readyTimeout: .seconds(2), - sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } - ) - - try await hub.waitUntilReady() - first.socket.failsSends = true - _ = try await hub.openChannel( - machineDeviceId: machine.deviceId, - machinePublicKey: machine.publicKey, - channelType: "test", - params: nil, - onMessage: { _ in }, - onClosed: { _ in } - ) - #expect(await waitUntil { transport.requests.count >= 2 }) - try await hub.waitUntilReady() - await hub.shutdown() - } - - @Test("Lifecycle reconnect immediately replaces the current socket") - func lifecycleReconnect() async throws { - let first = ScriptedCloudHub() - let second = ScriptedCloudHub() - let sockets = SocketQueue([first.socket, second.socket]) - let transport = FakeWebSocketTransport { _ in sockets.next() } - let store = InMemoryCloudCredentialStore(token: "session-token") - let hub = CloudHubConnection( - serverURL: URL(string: "https://cloud.example.com")!, - credentialStore: store, - deviceName: "Test App", - deviceOS: "macOS", - webSocketTransport: transport, - readyTimeout: .seconds(2), - sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } - ) - - try await hub.waitUntilReady() - await hub.reconnect() - #expect(await waitUntil { transport.requests.count >= 2 }) - try await hub.waitUntilReady() - await hub.shutdown() - } - @Test("Fatal hub close codes stop reconnecting") func fatalCloseCode() async throws { let scripted = ScriptedCloudHub() @@ -407,7 +282,7 @@ struct CloudHubConnectionTests { webSocketTransport: FakeWebSocketTransport { _ in scripted.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) await #expect(throws: CloudHubConnectionError.notSignedIn) { try await hub.waitUntilReady() @@ -428,7 +303,7 @@ struct CloudHubConnectionTests { webSocketTransport: transport, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) try await hub.waitUntilReady() #expect(transport.requests.first?.absoluteString == "wss://cloud.example.com/connect?token=session-token") @@ -453,7 +328,7 @@ struct CloudHubConnectionTests { webSocketTransport: transport, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) try await hub.waitUntilReady() #expect( diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubFlowControlTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubFlowControlTests.swift index 787617fa9..d7cd0364b 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubFlowControlTests.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubFlowControlTests.swift @@ -46,7 +46,7 @@ struct CloudHubFlowControlTests { webSocketTransport: FakeWebSocketTransport { _ in scripted.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) let recorder = Recorder() let closed = TestSignal() diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubReconnectTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubReconnectTests.swift new file mode 100644 index 000000000..ef5f0884a --- /dev/null +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudHubReconnectTests.swift @@ -0,0 +1,148 @@ +import CodevisorTestSupport +import Foundation +import Testing +import CodevisorClient +@testable import CodevisorCloud + +@Suite("CloudHub reconnect scheduling") +struct CloudHubReconnectTests { + @Test("Hub reconnects reuse the credential snapshot") + func reconnectUsesCredentialSnapshot() async throws { + let first = ScriptedCloudHub() + let second = ScriptedCloudHub() + let sockets = SocketQueue([first.socket, second.socket]) + let transport = FakeWebSocketTransport { _ in sockets.next() } + let memory = InMemoryCloudCredentialStore(token: "session-token") + try memory.saveAppDeviceId("app-device") + try memory.saveAppSecretKey(Data(repeating: 7, count: 32)) + let store = CountingCredentialStore(base: memory) + let clock = TestClock() + let hub = CloudHubConnection( + serverURL: URL(string: "https://cloud.example.com")!, + credentialStore: store, + deviceName: "Test App", + deviceOS: "macOS", + webSocketTransport: transport, + readyTimeout: .seconds(3), + sleep: clock.sleep, + reconnectDelay: { _ in .seconds(1) } + ) + + try await hub.waitUntilReady() + first.socket.disconnect() + await clock.waitForSleep(.seconds(1)) + #expect(transport.requests.count == 1) + clock.advance(by: .seconds(1)) + #expect(await waitUntil { transport.requests.count == 2 }) + try await hub.waitUntilReady() + + let counts = store.readCounts + #expect(counts.token == 1) + #expect(counts.deviceId == 1) + #expect(counts.secretKey == 1) + await hub.shutdown() + } + + @Test("A missing heartbeat pong replaces a half-open socket") + func heartbeatTimeoutReconnects() async throws { + let first = ScriptedCloudHub() + first.respondsToPing = false + let second = ScriptedCloudHub() + let sockets = SocketQueue([first.socket, second.socket]) + let transport = FakeWebSocketTransport { _ in sockets.next() } + let store = InMemoryCloudCredentialStore(token: "session-token") + let clock = TestClock() + let hub = CloudHubConnection( + serverURL: URL(string: "https://cloud.example.com")!, + credentialStore: store, + deviceName: "Test App", + deviceOS: "macOS", + webSocketTransport: transport, + readyTimeout: .seconds(2), + heartbeatInterval: .seconds(30), + heartbeatTimeout: .seconds(10), + sleep: clock.sleep, + reconnectDelay: { _ in .seconds(1) } + ) + + try await hub.waitUntilReady() + await clock.waitForSleep(.seconds(30)) + clock.advance(by: .seconds(30)) + await clock.waitForSleep(.seconds(10)) + #expect(transport.requests.count == 1) + clock.advance(by: .seconds(10)) + await clock.waitForSleep(.seconds(1)) + #expect(transport.requests.count == 1) + clock.advance(by: .seconds(1)) + #expect(await waitUntil { transport.requests.count == 2 }) + try await hub.waitUntilReady() + await hub.shutdown() + } + + @Test("An outbound send failure replaces the hub socket") + func sendFailureReconnects() async throws { + let machine = ScriptedRelayMachine() + let first = ScriptedCloudHub(machines: [machine.presence]) + let second = ScriptedCloudHub(machines: [machine.presence]) + let sockets = SocketQueue([first.socket, second.socket]) + let transport = FakeWebSocketTransport { _ in sockets.next() } + let store = InMemoryCloudCredentialStore(token: "session-token") + let clock = TestClock() + let hub = CloudHubConnection( + serverURL: URL(string: "https://cloud.example.com")!, + credentialStore: store, + deviceName: "Test App", + deviceOS: "macOS", + webSocketTransport: transport, + readyTimeout: .seconds(2), + sleep: clock.sleep, + reconnectDelay: { _ in .seconds(1) } + ) + + try await hub.waitUntilReady() + first.socket.failsSends = true + _ = try await hub.openChannel( + machineDeviceId: machine.deviceId, + machinePublicKey: machine.publicKey, + channelType: "test", + params: nil, + onMessage: { _ in }, + onClosed: { _ in } + ) + await clock.waitForSleep(.seconds(1)) + #expect(transport.requests.count == 1) + clock.advance(by: .seconds(1)) + #expect(await waitUntil { transport.requests.count >= 2 }) + try await hub.waitUntilReady() + await hub.shutdown() + } + + @Test("Lifecycle reconnect immediately replaces the current socket") + func lifecycleReconnect() async throws { + let first = ScriptedCloudHub() + let second = ScriptedCloudHub() + let sockets = SocketQueue([first.socket, second.socket]) + let transport = FakeWebSocketTransport { _ in sockets.next() } + let store = InMemoryCloudCredentialStore(token: "session-token") + let clock = TestClock() + let hub = CloudHubConnection( + serverURL: URL(string: "https://cloud.example.com")!, + credentialStore: store, + deviceName: "Test App", + deviceOS: "macOS", + webSocketTransport: transport, + readyTimeout: .seconds(2), + sleep: clock.sleep, + reconnectDelay: { _ in .seconds(1) } + ) + + try await hub.waitUntilReady() + await hub.reconnect() + await clock.waitForSleep(.seconds(1)) + #expect(transport.requests.count == 1) + clock.advance(by: .seconds(1)) + #expect(await waitUntil { transport.requests.count >= 2 }) + try await hub.waitUntilReady() + await hub.shutdown() + } +} diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayCancellationTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayCancellationTests.swift index 509b2ebcc..7ef54c7e0 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayCancellationTests.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayCancellationTests.swift @@ -15,7 +15,7 @@ struct CloudRelayCancellationTests { credentialStore: InMemoryCloudCredentialStore(token: "test-token"), deviceName: "Test", deviceOS: "macOS", webSocketTransport: FakeWebSocketTransport { _ in upstream }, - readyTimeout: .seconds(20), sleep: clock.sleep, reconnectDelay: { _ in .zero }) + readyTimeout: .seconds(20), sleep: clock.sleep, reconnectDelay: { _ in .seconds(1) }) let endpoint = CloudRelayEndpoint(hub: hub, machineDeviceId: "machine", machinePublicKey: "unused") let socket = CloudRelayWebSocketTransport(endpoint: endpoint).connect( URLRequest(url: URL(string: "https://cloud-relay.invalid/v1/sessions/chat/events/socket")!), diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayLoopbackBridgeTests.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayLoopbackBridgeTests.swift index 9315fb7bc..870b44f5c 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayLoopbackBridgeTests.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CloudRelayLoopbackBridgeTests.swift @@ -256,7 +256,7 @@ struct CloudRelayLoopbackBridgeTests { webSocketTransport: FakeWebSocketTransport { _ in scriptedMachine.hub.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) return ( CloudRelayLoopbackBridge( diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CountingCredentialStore.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CountingCredentialStore.swift index b546552fa..b17665158 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CountingCredentialStore.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/CountingCredentialStore.swift @@ -145,7 +145,8 @@ func makeHub( heartbeatInterval: heartbeatInterval, heartbeatTimeout: heartbeatTimeout, sleep: TestClock().sleep, - reconnectDelay: { _ in .zero }, + // A closed fake socket must suspend between attempts, even with one Swift worker. + reconnectDelay: { _ in .seconds(1) }, onMachineWait: onMachineWait ) return (hub, store) diff --git a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/ScriptedHttpMachine.swift b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/ScriptedHttpMachine.swift index 3895b66f6..a41426f68 100644 --- a/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/ScriptedHttpMachine.swift +++ b/packages/swift/CodevisorCloud/Tests/CodevisorCloudTests/ScriptedHttpMachine.swift @@ -215,7 +215,7 @@ func makeRelayEndpoint( webSocketTransport: FakeWebSocketTransport { _ in scripted.socket }, readyTimeout: .seconds(2), sleep: TestClock().sleep, - reconnectDelay: { _ in .zero } + reconnectDelay: { _ in .seconds(1) } ) let endpoint = CloudRelayEndpoint( hub: hub, diff --git a/packages/swift/MarkdownCore/Sources/MarkdownCore/LatestValuePreparationWorker.swift b/packages/swift/MarkdownCore/Sources/MarkdownCore/LatestValuePreparationWorker.swift index 43189d369..a84b2b315 100644 --- a/packages/swift/MarkdownCore/Sources/MarkdownCore/LatestValuePreparationWorker.swift +++ b/packages/swift/MarkdownCore/Sources/MarkdownCore/LatestValuePreparationWorker.swift @@ -11,12 +11,12 @@ public final class LatestValuePreparationWorker) -> Void } - private let prepare: @Sendable (Input) throws -> Output + private let prepare: @Sendable (Input) async throws -> Output private var generation: UInt64 = 0 private var pending: Work? private var processing: Task? - public init(prepare: @escaping @Sendable (Input) throws -> Output) { + public init(prepare: @escaping @Sendable (Input) async throws -> Output) { self.prepare = prepare } @@ -42,7 +42,7 @@ public final class LatestValuePreparationWorker [MarkdownSegment] + typealias SnapshotParser = @Sendable (String, Bool) async -> [MarkdownSegment] struct Presentation: Sendable { let text: String @@ -77,10 +77,11 @@ final class StreamingMarkdownParseCoordinator: ObservableObject { requestedText = text requestedIsComplete = isComplete self.snapshotParser = snapshotParser + // Only streamed updates use the asynchronous parser seam; mounting stays synchronous. let segments = isComplete ? MarkdownSegmentCache.shared.segments(for: text) - : snapshotParser(text, false) + : StreamingSegmenter.parseSnapshot(text: text, isComplete: false) let renderSegments = MarkdownRenderSegment.initial(segments) nextSegmentID = UInt64(renderSegments.count) presentation = Presentation( @@ -104,7 +105,7 @@ final class StreamingMarkdownParseCoordinator: ObservableObject { let snapshotParser = snapshotParser let parsed = await Task.detached(priority: .userInitiated) { - snapshotParser(text, isComplete) + await snapshotParser(text, isComplete) }.value guard !Task.isCancelled else { return } publish(parsed, text: text, isComplete: isComplete, generation: requestGeneration) diff --git a/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/LatestValuePreparationWorkerTests.swift b/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/LatestValuePreparationWorkerTests.swift index 623eabea2..d8bea251d 100644 --- a/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/LatestValuePreparationWorkerTests.swift +++ b/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/LatestValuePreparationWorkerTests.swift @@ -9,11 +9,11 @@ struct LatestValuePreparationWorkerTests { @Test func runningWorkPublishesAndOnlyLatestPendingInputRuns() async { let started = TestSignal() let published = TestSignal() - let release = DispatchSemaphore(value: 0) + let release = TestSignal() let inputs = Mutex<[Int]>([]) let worker = LatestValuePreparationWorker { value in inputs.withLock { $0.append(value) } - if value == 1 { started.signal(); release.wait() } + if value == 1 { started.signal(); await release.wait() } return value } var outputs: [Int] = [] @@ -35,9 +35,9 @@ struct LatestValuePreparationWorkerTests { @Test func cancellationRejectsOldOutputAndAllowsANewPresentation() async { let started = TestSignal() let published = TestSignal() - let release = DispatchSemaphore(value: 0) + let release = TestSignal() let worker = LatestValuePreparationWorker { value in - if value == 1 { started.signal(); release.wait() } + if value == 1 { started.signal(); await release.wait() } return value } worker.submit(1) { _, _ in Issue.record("Cancelled presentation published") } diff --git a/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/StreamingSegmenterTests.swift b/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/StreamingSegmenterTests.swift index 2aef0fc3e..f093624d4 100644 --- a/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/StreamingSegmenterTests.swift +++ b/packages/swift/StreamMarkdown/Tests/StreamMarkdownTests/StreamingSegmenterTests.swift @@ -1,3 +1,4 @@ +import CodevisorTestSupport import Foundation import Testing @testable import StreamMarkdown @@ -184,27 +185,22 @@ struct StreamingSegmenterTests { @Test("Async parsing discards an out-of-order stale snapshot") func staleAsyncSnapshot() async { - let slowStarted = DispatchSemaphore(value: 0) - let releaseSlow = DispatchSemaphore(value: 0) + let slowStarted = TestSignal() + let releaseSlow = TestSignal() let coordinator = StreamingMarkdownParseCoordinator( text: "initial", isComplete: false, snapshotParser: { text, _ in if text == "slow" { slowStarted.signal() - releaseSlow.wait() + await releaseSlow.wait() } return [.textRun([.paragraph(text)])] } ) async let slow: Void = coordinator.update(text: "slow", isComplete: false) - await withCheckedContinuation { continuation in - DispatchQueue.global().async { - slowStarted.wait() - continuation.resume() - } - } + await slowStarted.wait() await coordinator.update(text: "fast", isComplete: false) releaseSlow.signal() _ = await slow diff --git a/packages/swift/TranscriptKit/Sources/TranscriptKit/TranscriptActiveProjectionWorker.swift b/packages/swift/TranscriptKit/Sources/TranscriptKit/TranscriptActiveProjectionWorker.swift index d536a9a03..743a6dfc8 100644 --- a/packages/swift/TranscriptKit/Sources/TranscriptKit/TranscriptActiveProjectionWorker.swift +++ b/packages/swift/TranscriptKit/Sources/TranscriptKit/TranscriptActiveProjectionWorker.swift @@ -30,7 +30,7 @@ public final class TranscriptActiveProjectionWorker { public let rows: [TranscriptPresentationRow] } - typealias Projector = @Sendable (ConversationItem, String?) -> [TranscriptPresentationRow] + typealias Projector = @Sendable (ConversationItem, String?) async -> [TranscriptPresentationRow] private let worker: LatestValuePreparationWorker @@ -42,7 +42,7 @@ public final class TranscriptActiveProjectionWorker { init(projector: @escaping Projector) { worker = LatestValuePreparationWorker { request in - projector(request.item, request.waitingOnBackgroundTask) + await projector(request.item, request.waitingOnBackgroundTask) } } diff --git a/packages/swift/TranscriptKit/Tests/TranscriptKitTests/TranscriptActiveProjectionWorkerTests.swift b/packages/swift/TranscriptKit/Tests/TranscriptKitTests/TranscriptActiveProjectionWorkerTests.swift index ef042f07a..133032e20 100644 --- a/packages/swift/TranscriptKit/Tests/TranscriptKitTests/TranscriptActiveProjectionWorkerTests.swift +++ b/packages/swift/TranscriptKit/Tests/TranscriptKitTests/TranscriptActiveProjectionWorkerTests.swift @@ -1,3 +1,4 @@ +import CodevisorTestSupport import Foundation import Synchronization import Testing @@ -7,15 +8,15 @@ import Testing struct TranscriptActiveProjectionWorkerTests { @Test("Projection keeps one in-flight parse and replaces the waiting snapshot") func latestPendingSnapshotWins() async { - let started = DispatchSemaphore(value: 0) - let release = DispatchSemaphore(value: 0) + let started = TestSignal() + let release = TestSignal() let projectedMarkdown = Mutex<[String]>([]) let worker = TranscriptActiveProjectionWorker { item, _ in let markdown = Self.markdown(in: item) projectedMarkdown.withLock { $0.append(markdown) } if markdown == "first" { started.signal() - release.wait() + await release.wait() } return [] } @@ -25,12 +26,7 @@ struct TranscriptActiveProjectionWorkerTests { #expect(output.request.revision == 1) firstPublished.withLock { $0 = true } } - await withCheckedContinuation { continuation in - DispatchQueue.global().async { - started.wait() - continuation.resume() - } - } + await started.wait() worker.submit(request(revision: 2, markdown: "second")) { _ in Issue.record("A replaced waiting projection must not publish") } From 0225decda4c8d07ebfabbb845c66dfe13cd51e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20=C5=A2urcanu?= Date: Mon, 14 Sep 2026 20:45:50 -0700 Subject: [PATCH 11/11] Use pre-commit validation instead of a pull request workflow --- .github/workflows/pull-request.yml | 76 ------------------------- docs/plans/native-screen-sharing-mvp.md | 14 +++-- 2 files changed, 8 insertions(+), 82 deletions(-) delete mode 100644 .github/workflows/pull-request.yml diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml deleted file mode 100644 index 1f5aef7f2..000000000 --- a/.github/workflows/pull-request.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Pull Request Checks - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -permissions: - contents: read - -concurrency: - group: pull-request-${{ github.event.pull_request.number }} - cancel-in-progress: true - -env: - BUN_VERSION: "1.3.13" - NODE_VERSION: "24.x" - -jobs: - javascript: - if: ${{ !github.event.pull_request.draft }} - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: ${{ env.NODE_VERSION }} - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: ${{ env.BUN_VERSION }} - - run: bun install --frozen-lockfile - - run: bun run check:js - - native: - if: ${{ !github.event.pull_request.draft }} - # PR code runs on an ephemeral hosted machine, without signing secrets. - runs-on: macos-26 - timeout-minutes: 60 - env: - DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: ${{ env.NODE_VERSION }} - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: ${{ env.BUN_VERSION }} - - name: Record native tools - run: | - xcodebuild -version - xcrun swift --version - - run: bun install --frozen-lockfile - - name: Install the validated SwiftLint version - run: | - tool_dir="$RUNNER_TEMP/swiftlint-0.63.1" - mkdir -p "$tool_dir" - curl --fail --location --retry 3 --output "$tool_dir/swiftlint.zip" \ - https://github.com/realm/SwiftLint/releases/download/0.63.1/portable_swiftlint.zip - echo "0e6369741b694b701e9bd4e4fe9b408a5946d4bb076b79ea6ec2bce428475739 $tool_dir/swiftlint.zip" | shasum -a 256 -c - - ditto -x -k "$tool_dir/swiftlint.zip" "$tool_dir" - "$tool_dir/swiftlint" version - echo "$tool_dir" >> "$GITHUB_PATH" - - run: bun run swift:format:check - - run: bun run swift:lint - - run: bun run swift:test - timeout-minutes: 15 - - run: bun run swift:test:macos-transcript - - run: bun run swift:test:macos-composer - - run: bun run build:macos - - run: bun run swift:build:ios - - run: bun run swift:test:ios-transcript diff --git a/docs/plans/native-screen-sharing-mvp.md b/docs/plans/native-screen-sharing-mvp.md index 6a8409b10..775817add 100644 --- a/docs/plans/native-screen-sharing-mvp.md +++ b/docs/plans/native-screen-sharing-mvp.md @@ -22,7 +22,7 @@ Actual-app checks on September 14 established remote video and control, includin Commit `dbdbcaea` passed the normal pre-commit hook in a clean validation worktree, without exclusions: the full JavaScript check chain, Swift formatting/lint, 2,119 Swift package tests, 30 macOS transcript tests, 11 macOS composer tests, the iOS build, and 8 iOS transcript tests. The macOS development app built successfully. The September 14 integration adopts main's center-only workspace layout and focus-source ownership, and preserves chatless workspace behavior. -Main integration `166fce13` brings in `10bdc699`'s attachment-thumbnail framing fix without conflicts. That merge, notice/CI commit `acd462f8`, and CI tooling correction `d627587d` each passed the complete normal pre-commit hook without exclusions. The combined macOS build passed, and the resource bundle's two dependency-notice files match the audited source hashes. +Main integration `166fce13` brings in `10bdc699`'s attachment-thumbnail framing fix without conflicts. The integration, bundled notices and subsequent test scheduling fixes passed the complete normal pre-commit hook without exclusions. The combined macOS build passed, and the resource bundle's two dependency-notice files match the audited source hashes. Actual-app checks with the final viewer established that the complete toolbar stays interactive during connection, View/Control choices survive connection and menu interactions, and switching modes no longer shows a spinner. Those checks used the earlier host build. The isolated host was subsequently rebuilt from the matching final source; all 2,864 staged source files were verified and the app passed strict signature verification. The fresh acceptance attempt reached the host Screen Recording permission error. The remaining two-Mac workflows and sustained-use check are pending reauthorization of this rebuilt development app; earlier results do not replace that acceptance pass. @@ -36,22 +36,24 @@ The product now bundles the main WebRTC license, audited macOS/iOS third-party n The published archive contains the main WebRTC license but no aggregate third-party notices. On September 14, the Xcode pin was deliberately updated from 26.5 to 26.6 (17F113), with Python 3.12.14 unchanged. The first source-build attempt found a missing depot_tools bootstrap before GN; the corrected recipe initializes the pinned tools without updating their revision. The subsequent complete build passed for all five macOS/iOS architectures and packaged a candidate XCFramework (`cd477857489c347daddfbfa19bf0d905a2e2a143f3ee84543ba887b97cc13c45`). The generated notices match the exact source license texts for every mapped dependency in all five target graphs. Headers, slices, signatures, privacy manifests and matching dSYM UUIDs passed inspection; all 180 unchanged screen-sharing tests passed against the isolated candidate, as did factory/track/peer/data-channel creation and close. No capture or media connection was run with it. -An unpublished Developer ID packaging run of `acd462f8` completed through the ordinary release script with version `0.0.0`. Both architecture-specific ZIPs and DMGs passed verification. The apps and embedded WebRTC framework have Developer ID signatures, hardened runtime and secure timestamps; both ZIPs preserve the exact audited notice bytes and privacy manifest, and both DMG checksums/signatures passed. The signed ARM runtime loaded the screen-sharing API and exercised an in-memory SQLite database. No app UI or media session was launched by this verification. +An unpublished Developer ID packaging run of `d455c353` completed through the ordinary release script with version `0.0.0`. Both architecture-specific ZIPs and DMGs passed verification. The apps and embedded WebRTC framework have Developer ID signatures, hardened runtime and secure timestamps; both ZIPs preserve the exact audited notice bytes and privacy manifest, and both DMG checksums/signatures passed. The signed ARM runtime loaded the screen-sharing API and exercised an in-memory SQLite database. No app UI or media session was launched by this verification. -This was a local packaging check using the installed Xcode 27 beta; the hosted native PR check is pinned to Xcode 26.6. The ARM runtime was rebuilt from this revision. For Intel, only the unchanged native Node/dependency tree was reused from main's successful CI run at `10bdc699`, with current JavaScript and resources. All dependency manifests and the lockfile matched (the root manifest differs only by the development probe script); 2,019 current non-native files and all eight Intel Mach-O files were checked. Intel execution was unavailable without Rosetta. These review artifacts were not published and do not replace native release builds. Notarization was not attempted because this session has no notarization credentials. +This was a local packaging check using the installed Xcode 27 beta. The isolated WebRTC source-build recipe uses Xcode 26.6. The ARM runtime was rebuilt from this revision. For Intel, only the unchanged native Node/dependency tree was reused from main's successful CI run at `10bdc699`, with current JavaScript and resources. All dependency manifests and the lockfile matched (the root manifest differs only by the development probe script); 2,019 current non-native files and all eight Intel Mach-O files were checked. Intel execution was unavailable without Rosetta. These review artifacts were not published and do not replace native release builds. Notarization was not attempted because this session has no notarization credentials. The source-built WebRTC candidate remains isolated and unpromoted; actual-app validation is required before replacing the installed dependency. A successful source build does not establish byte reproducibility or performance equivalence. -Pushing to `main` automatically starts the Alpha build and publication workflow, so final CI, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is now ready for review, with non-publishing GitHub-hosted JavaScript and native checks. Native validation uses Xcode 26.6 and checksum-pinned SwiftLint 0.63.1; the first hosted run exposed the missing linter, which was corrected. Check the final head's live results on the PR rather than treating local hooks as CI. A hosted run also exposed a timeout in the existing MP4 integration test. Its fixture now awaits SDK callbacks asynchronously, propagates startup errors and cancels the writer during cleanup; it retains all MP4 duration/dimension/final-frame assertions. This removes blocking bridge-worker waits from an async test but does not by itself establish the hosted failure's cause. The package-test step has a 15-minute deadlock bound. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. The PR has not been merged. +Pushing to `main` automatically starts the Alpha build and publication workflow, so pre-commit validation, distribution and actual-app acceptance must be reviewed before merging. [PR #7](https://github.com/851-labs/codevisor/pull/7) is ready for review and has not been merged. Validation uses the existing pre-commit hook: typography and the complete `bun run check` chain, without exclusions. The added PR validation workflow and its uncommitted watchdog were removed at the user's request; hosted PR checks are not an MVP gate. Existing release and dependency-artifact workflows are unchanged. Macroscope's automatic review is unavailable because of a workspace billing/usage issue; no automatic review approval is claimed. -A local single-worker reproduction subsequently identified two independent test scheduling defects: preparation fixtures blocked Swift workers with semaphores while waiting for other Swift tasks, and cloud fixtures retried closed sockets with zero virtual delay. Preparation seams now allow asynchronous test gates; cloud fixtures suspend on a positive virtual backoff that reconnect tests advance explicitly. Production parsing, latest-value selection and reconnect policy are unchanged. The full package suite completes with `LIBDISPATCH_COOPERATIVE_POOL_STRICT=1`; this reproducer result is separate from the final hosted CI verdict. +The MP4 integration fixture awaits SDK callbacks asynchronously, propagates startup errors and cancels its writer during cleanup while retaining all duration, dimension and final-frame assertions. + +A local single-worker reproduction subsequently identified two independent test scheduling defects: preparation fixtures blocked Swift workers with semaphores while waiting for other Swift tasks, and cloud fixtures retried closed sockets with zero virtual delay. Preparation seams now allow asynchronous test gates; cloud fixtures suspend on a positive virtual backoff that reconnect tests advance explicitly. Production parsing, latest-value selection and reconnect policy are unchanged. The full package suite completes with `LIBDISPATCH_COOPERATIVE_POOL_STRICT=1`, and the complete normal pre-commit hook passes. ## Before MVP acceptance 1. Verify the final source in two actual apps: permissions, viewing, typing, clicking, dragging, scrolling, clipboard, scaling and display selection. 2. Verify input release and capture cleanup on focus loss, pane hide/close, disconnect, host stop and reconnect; verify no stale start revives a stopped session. 3. Complete a visible sustained-use check for responsiveness, resources and bounded queues using the configuration intended for users. -4. Review the pinned dependency, distributed notices, signing/packaging and clean CI results. Any revision to the source-build prerequisite in the original plan must be explicit. +4. Review the pinned dependency, distributed notices, signing/packaging and the full pre-commit check results. Any revision to the source-build prerequisite in the original plan must be explicit. 5. Review the integrated shared workspace-navigation changes for regressions, including chatless workspaces, tab selection and focus ownership. Apple parity, a production-default tuning change, iOS viewing, audio, virtual displays, multiple viewers and internet/forced-relay guarantees are outside this MVP checkpoint. Existing connectivity code does not establish those guarantees.