diff --git a/.github/workflows/codemode-evals.yml b/.github/workflows/codemode-evals.yml index 18c7eda..1fb2b54 100644 --- a/.github/workflows/codemode-evals.yml +++ b/.github/workflows/codemode-evals.yml @@ -26,7 +26,9 @@ jobs: # unchanged main added no signal for these offline checks. deterministic: name: Deterministic evals - runs-on: macos-latest + # Pinned, not macos-latest: the package declares a macOS 15 floor, and a + # silently-rolling runner would move that floor without a code change. + runs-on: macos-15 steps: - name: Checkout uses: actions/checkout@v4 @@ -49,6 +51,18 @@ jobs: - name: Test package run: swift test + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "22" + + # The generated TypeScript declarations go into model prompts and host + # type-checkers, so they are compiled for real: every platform surface and + # every per-capability fragment, under `tsc --strict`, TypeScript pinned. + # The Swift tests only catch failure shapes someone already thought of. + - name: Type-check generated TypeScript declarations + run: scripts/check-typescript-declarations.sh "$RUNNER_TEMP/codemode-dts" + - name: Build eval CLI run: swift build --package-path Tools/CodeModeEval @@ -57,7 +71,7 @@ jobs: platform-build: name: Build (${{ matrix.platform }}) - runs-on: macos-latest + runs-on: macos-15 strategy: fail-fast: false matrix: @@ -107,7 +121,7 @@ jobs: # live+compare steps with a warning, instead of pretending or failing noisily. llm-live: name: LLM eval (live regression gate) - runs-on: macos-latest + runs-on: macos-15 needs: deterministic if: github.event_name == 'workflow_dispatch' env: diff --git a/Package.swift b/Package.swift index b1b3f42..3ceeeb2 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,11 @@ let package = Package( .testTarget( name: "CodeModeAuthoringTests", dependencies: [ + // Declared directly, not only via CodeModeAuthoring: these tests use + // CodeModeConfiguration/CodeModeAgentTools, and without the direct + // edge SwiftPM did not rebuild them when CodeMode's types changed + // layout — leaving stale objects that failed to link or crashed. + "CodeMode", "CodeModeAuthoring", "CodeModeMacros", .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), diff --git a/README.md b/README.md index 29bf64f..6006bae 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,40 @@ # CodeMode.swift -`CodeMode.swift` is a Swift package that implements [CodeMode](https://blog.cloudflare.com/code-mode/) for Apple platform APIs by exposing agents to two tools: - -- `searchJavaScriptAPI`: code-driven discovery of the bundled JavaScript wrapped Apple API surface for the current host platform. -- `executeJavaScript`: constrained JavaScript execution with capability allowlisting and structured errors. - -GitHub: [velos/CodeMode.swift](https://github.com/velos/CodeMode.swift) - -## Highlights - -- Platforms: `iOS 18+`, `macOS 15+`, `visionOS 2+` -- watchOS is audited but not a runtime target in this package because the current runtime depends on JavaScriptCore. -- Typed Swift host API through `CodeModeAgentTools` -- Streaming execution via `JavaScriptExecutionCall` -- Structured failures via `CodeModeToolError` -- Hybrid JS surface: - - web-style globals: `fetch`, `URL`, `URLSearchParams`, `setTimeout`, `console` - - cross-platform Apple namespaces: `apple.keychain`, `apple.location`, `apple.weather`, `apple.calendar`, `apple.reminders`, `apple.contacts`, `apple.photos`, `apple.vision`, `apple.notifications`, `apple.health`, `apple.home`, `apple.media`, `apple.fs`, `apple.cloudkit`, `apple.maps`, `apple.storekit`, `apple.speech`, `apple.appIntents`, `apple.activity`, `apple.foundationModels`, `apple.music`, `apple.wallet` - - platform-specific namespaces when needed: `ios.alarm` -- iOS/visionOS system UI helpers through an injected presenter, including alerts, calendar editors, photo/contact/document pickers, share sheets, Quick Look previews, web authentication, and iOS-only camera/scan/mail/message compose flows -- Node-style aliases for file operations through `globalThis.fs.promises` -- Sandboxed filesystem policy with allowed roots: `tmp`, `caches`, `documents` -- Search and execution only expose helpers supported on the current host platform -- `apple.*` is the canonical cross-Apple namespace, not a promise that every helper exists on every Apple OS -- Library-only package surface +A Swift package that implements [Code Mode](https://blog.cloudflare.com/code-mode/) for Apple platform APIs. -## Installation +Instead of handing an agent one tool per API, CodeMode gives it two: -Add `CodeMode.swift` with Swift Package Manager: +- **`searchJavaScriptAPI`**: the agent writes a small JavaScript function that filters a catalog of available helpers. +- **`executeJavaScript`**: the agent writes one script that does the whole job — read the calendar, filter it, write a file, return a summary — against helpers like `apple.calendar.read` and `apple.fs.write`. -```swift -.package(url: "https://github.com/velos/CodeMode.swift", branch: "main") -``` +Models are better at writing code than at chaining dozens of tool calls, and a script that does its filtering locally returns the answer rather than every intermediate result. One execution replaces a round trip per operation. -No release has been tagged yet, so depend on `main` for now. Once `0.1.0` is tagged, prefer the versioned form: +Platforms: iOS 18+, macOS 15+, visionOS 2+. watchOS is out of scope because the runtime depends on JavaScriptCore. + +## Installation ```swift -.package(url: "https://github.com/velos/CodeMode.swift", from: "0.1.0") +.package(url: "https://github.com/velos/CodeMode.swift", branch: "main") ``` -Then add the product to your target: +No release is tagged yet; once `0.1.0` is, prefer `from: "0.1.0"`. Add the product to your target: ```swift .product(name: "CodeMode", package: "CodeMode") ``` -## Public API - -- `CodeModeAgentTools` -- `searchJavaScriptAPI(_:) async throws -> JavaScriptAPISearchResponse` -- `executeJavaScript(_:) async throws -> JavaScriptExecutionCall` -- `JavaScriptExecutionCall.events` -- `JavaScriptExecutionCall.result` -- `JavaScriptExecutionCall.cancel()` -- `CodeModeToolError` -- `CodeModeConfiguration` -- `CodeModeFileSystem` -- `LocalCodeModeFileSystem` -- `CodeModeAgentToolDescriptions` -- `CodeModeEvaluation` SwiftPM product for deterministic scenario evaluation -- `SystemUIPresenter` -- `UIKitSystemUIPresenter` on iOS/visionOS - -## Quick Start +## Quick start ```swift import CodeMode -let tools = CodeModeAgentTools() - -let searchResponse = try await tools.searchJavaScriptAPI( - JavaScriptAPISearchRequest( - code: """ - async () => { - return api.references - .filter(ref => ref.tags.includes("reminders")) - .map(ref => ({ - capability: ref.capability, - jsNames: ref.jsNames, - summary: ref.summary - })); - } - """ +let tools = CodeModeAgentTools( + config: CodeModeConfiguration( + // The host's ceiling on what any script may use. See "Security model". + capabilityGrant: .only([.fsRead, .fsWrite]) ) ) -print(searchResponse.result ?? .null) - let call = try await tools.executeJavaScript( JavaScriptExecutionRequest( code: """ @@ -98,314 +46,167 @@ let call = try await tools.executeJavaScript( ) for await event in call.events { - switch event { - case .log(let entry): - print(entry.message) - case .diagnostic(let diagnostic): - print(diagnostic.message) - case .syntaxError(let error), - .functionNotFound(let error), - .thrownError(let error), - .toolError(let error): - print("\(error.code): \(error.message)") - case .finished: - break - } + if case .log(let entry) = event { print(entry.message) } } -let result = try await call.result -print(result.output ?? .null) +let result = try await call.result // throws CodeModeToolError on failure +print(result.output ?? .null) // "hello" ``` -## Filesystem Integration +In an agent, the model writes `code` and `allowedCapabilities`; your host passes them through. -CodeMode keeps its JavaScript filesystem API stable while allowing hosts to provide the underlying operations: +## How it works -```swift -let tools = CodeModeAgentTools( - config: CodeModeConfiguration( - fileSystem: MyCodeModeFileSystem() - ) -) -``` +1. **Search.** `searchJavaScriptAPI` runs the model's function against an `api` object listing every helper available on this platform: names, arguments, types, examples, and a TypeScript declaration for each. The model gets back only what its function returns. +2. **Execute.** Each execution gets a fresh `JSContext` with this platform's helpers installed. The script is compiled as the body of an async function, so it can use top-level `await` and `return`. +3. **Bridge.** Each `apple.*` call crosses into Swift, is checked against the capability grant, path policy, and network policy, handles any system permission prompt, and calls the real framework (EventKit, CoreLocation, HealthKit, and so on). +4. **Report.** Logs, diagnostics, and errors stream as events while the script runs. The final result is JSON, or a `CodeModeToolError` with a code, the failing helper, a line, and repair suggestions the model can act on. -`CodeModeFileSystem` receives paths after `PathPolicy` resolution, so sandbox root enforcement stays in CodeMode while the host can route reads, writes, listings, moves, copies, deletes, and stats through another backing implementation. `LocalCodeModeFileSystem` preserves the default `FileManager` behavior. +Calls into native code are synchronous, so `Promise.all` over several helpers runs them one at a time. `setTimeout` and `clearTimeout` are real timers, so `await new Promise(r => setTimeout(r, 1000))` actually waits. -## Network Access Policy +### The JavaScript surface -`network.fetch` egress is governed by `CodeModeConfiguration.networkAccessPolicy`: +- **Web-style globals:** `fetch`, `URL`, `URLSearchParams`, `setTimeout`/`clearTimeout`, `console`, and Node-style `fs.promises` and `path`. +- **`apple.*` namespaces:** `keychain`, `location`, `weather`, `calendar`, `reminders`, `contacts`, `photos`, `vision`, `notifications`, `health`, `home`, `media`, `fs`, `cloudkit`, `maps`, `storekit`, `speech`, `appIntents`, `activity`, `foundationModels`, `music`, `wallet`. +- **Platform-specific namespaces** such as `ios.alarm` (AlarmKit). +- **System UI** on iOS and visionOS: alerts and prompts, pickers for photos, contacts, documents, and calendars, share sheets, Quick Look, web authentication, and on iOS camera capture, document scanning, and mail and message compose. These need a `SystemUIPresenter`, such as `UIKitSystemUIPresenter`, and fail with `UI_PRESENTER_UNAVAILABLE` without one. -- The default `NetworkAccessPolicy.standard` refuses loopback, RFC 1918, link-local (including cloud metadata addresses such as `169.254.169.254`), CGNAT, and unique-local destinations, plus `localhost` and `.local`/`.localhost`/`.internal` names, and caps buffered response bodies at 10 MB. -- Redirect targets are re-validated against the policy before they are followed. -- Refusals throw structured `NETWORK_POLICY_VIOLATION` errors and are written to the audit logger along with successful fetch destinations. -- `allowedHosts` restricts fetch to an explicit list (entries match the host and its subdomains, and deliberately allowlisted private hosts such as `localhost` are honored); `blockedHosts` refuses specific hosts; `NetworkAccessPolicy.permissive` restores unrestricted behavior. -- Matching is by URL host only; DNS resolution is not performed, so a public hostname that resolves to a private address is not detected. Hosts that need stricter guarantees should set `allowedHosts`. +`apple.*` is the shared cross-platform namespace, not a promise that every helper exists on every OS. Search and execution only ever expose what the current host supports. -```swift -let tools = CodeModeAgentTools( - config: CodeModeConfiguration( - networkAccessPolicy: NetworkAccessPolicy( - allowedHosts: ["api.example.com"], - maxResponseBytes: 2_000_000 - ) - ) -) -``` +## Wiring it into an agent -## Search - -`searchJavaScriptAPI` accepts `JavaScriptAPISearchRequest`: - -- `code`: JavaScript source that evaluates to an async function - -Behavior: - -- empty search input throws `CodeModeToolError(code: "INVALID_REQUEST", ...)` -- search executes your async function against a preloaded `api` object -- `api` only contains capabilities and JS names supported on the current host platform -- returned output must be JSON-serializable -- syntax errors, invalid search programs, timeouts, and runtime failures throw `CodeModeToolError` -- responses include `result: JSONValue?` plus non-fatal diagnostics - -Available in search code: - -```ts -interface JavaScriptAPIReference { - capability: string; - capabilityKey: string; - builtInCapability: string | null; - jsNames: string[]; - summary: string; - tags: string[]; - example: string; - requiredArguments: string[]; - optionalArguments: string[]; - argumentTypes: Record; - argumentHints: Record; - argumentConstraints: { allowedStringValues: Record }; - resultSummary: string; -} +Three pieces, all generated from the same registry, so they cannot disagree with what actually runs: -declare const api: { - references: JavaScriptAPIReference[]; - byCapability: Record; - byJSName: Record; -}; -``` - -Example search programs: +```swift +// Tool definitions: names, descriptions, and JSON parameter schemas. +CodeModeAgentToolDescriptions.searchJavaScriptAPI +CodeModeAgentToolDescriptions.executeJavaScript -```javascript -async () => { - return api.references - .filter(ref => ref.tags.includes("media")) - .map(ref => ({ capability: ref.capability, jsNames: ref.jsNames })); -} -``` +// How to use the tools: one script per task, request only what you call, +// repair from the structured error. +let guidance = CodeModeAgentGuidance.systemPrompt(.standard) -```javascript -async () => { - return api.byJSName["apple.fs.read"]; -} +// What the helpers are: TypeScript declarations for the granted surface. +let declarations = tools.typeDeclarations() ``` -## Execution - -`executeJavaScript` accepts `JavaScriptExecutionRequest`: +**Guidance** matters more than it looks. The tool descriptions tell a model what helpers are called, not what shape its work should take, and without it agents tend to use `executeJavaScript` as a thin RPC, one call per operation. It comes in three tiers. Each is self-contained, and a smaller tier drops examples, never rules: -- `code` -- `allowedCapabilities` -- `timeoutMs` -- `context` +| Tier | ~Tokens | Adds | +| --- | --- | --- | +| `.brief` | 160 | One call runs the whole task; request only the capabilities you use. | +| `.standard` | 620 | The search → write → repair workflow, with a worked example. | +| `.full` | 1000 | Partial-failure handling, sequential native calls, and anti-patterns. | -It returns a `JavaScriptExecutionCall` immediately. +`CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 800)` picks the largest tier that fits, falling back to `.brief`. Counts are estimated at four characters per token. -Cross-platform privileged helpers are installed under `apple.*`. Platform-specific helpers are installed only where supported, for example `ios.alarm.*` on iOS hosts that support AlarmKit. +**Declarations** follow the host's `CapabilityGrant`, so they never advertise a helper the model would be denied. Size is the reason to grant narrowly: the unrestricted surface is about 21,000 tokens on iOS, while a grant of a few helpers is about 1,000. `typeDeclarations(for:)` renders an explicit subset, and every search result carries its own `dts`, so a model can also fetch declarations through search instead of having them all in its prompt: -`apple.location.requestPermission()` is currently exposed only on iOS hosts. Other Apple platforms can expose `apple.location.*` helpers when supported, but the explicit permission-request helper is intentionally hidden outside iOS for now. - -System UI helpers are installed only on supported UI platforms. Shared iOS/visionOS helpers include `apple.ui.presentAlert`, `apple.ui.presentPrompt`, `apple.settings.open`, `apple.calendar.pickCalendar`, `apple.calendar.presentEvent`, `apple.calendar.presentNewEvent`, `apple.contacts.pick`, `apple.contacts.presentContact`, `apple.contacts.presentNewContact`, `apple.photos.pick`, `apple.photos.presentLimitedLibraryPicker`, `apple.documents.pick`, `apple.documents.export`, `apple.documents.save`, `apple.documents.openIn`, `apple.share.present`, `apple.quicklook.preview`, `apple.camera.scanData`, `apple.print.present`, `apple.web.present`, and `apple.auth.webAuthenticate`. iOS-only helpers include `apple.camera.capture`, `apple.documents.scan`, `apple.mail.compose`, and `apple.messages.compose`. Host apps must provide a `SystemUIPresenter`; otherwise UI-presenting helpers fail with `UI_PRESENTER_UNAVAILABLE`. - -`call.events` is a non-throwing `AsyncStream` that can emit: +```javascript +async () => api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n") +``` -- `.log(ExecutionLog)` -- `.diagnostic(ToolDiagnostic)` -- `.syntaxError(CodeModeToolError)` -- `.functionNotFound(CodeModeToolError)` -- `.thrownError(CodeModeToolError)` -- `.toolError(CodeModeToolError)` -- `.finished` +Constrained arguments become string-literal unions and argument hints become doc comments. CI compiles every generated declaration with `tsc --strict`. -`call.result` is the throwing boundary: +## Security model -- on success it returns `JavaScriptExecutionResult` -- on failure it throws `CodeModeToolError` -- `call.cancel()` interrupts in-flight JavaScript +Scripts are written by a model, which means they may be written by whoever controls the model's input. The defaults assume that. -### Timeout and cancellation +**Capabilities.** `allowedCapabilities` is part of the tool schema, so the model fills it in. It's a useful least-privilege declaration, but not a sandbox on its own. `CodeModeConfiguration.capabilityGrant` is the host-owned ceiling, and a script gets `requested ∩ granted`: -`timeoutMs` and `cancel()` are enforced preemptively: a JavaScriptCore execution -time limit terminates CPU-bound scripts (for example `while (true) {}`) instead -of relying on a wall-clock check that cannot interrupt running JavaScript. +```swift +CodeModeConfiguration( + capabilityGrant: .only( + [.fsRead, .fsWrite, .networkFetch], + capabilityKeys: ["myapp.tasks.complete"] // custom providers + ) +) +``` -- `timeoutMs` bounds total wall-clock for the execution, including the - synchronous portion of the script and any time already elapsed while a bridge - call blocked on native I/O (network, a UI picker). Hosts that present - long-running UI or issue slow requests should size `timeoutMs` accordingly; - the default is `10000`. -- Preemption uses `JSContextGroupSetExecutionTimeLimit`, which JavaScriptCore - exports but declares only in a private WebKit header. CodeMode reaches it - through a thin C shim (`CCodeModeJSC`). This is JavaScriptCore's only - mechanism for interrupting runaway scripts; hosts submitting to the App Store - should be aware they rely on this exported-but-private symbol. +The default is `.unrestricted`, for source compatibility; set a grant whenever tool JSON comes from a model. A capability the grant withholds fails with `CAPABILITY_DENIED`, marked as not fixable by retrying. Built-in capabilities and custom provider keys are separate lists, so a provider key can never name a built-in. -`CodeModeToolError` includes structured fields such as: +**Filesystem.** Paths are scoped to the `tmp:`, `caches:`, and `documents:` roots. `fs.move`, `fs.copy`, and `fs.delete` refuse a root as source or destination, and replacing an existing file needs `overwrite: true`. If the operation fails, the original file is restored. Reads, writes, and copies are capped at 16 MB by default (`FileSystemLimits`). Provide a `CodeModeFileSystem` to back these operations with your own storage. Paths reach it already resolved by `PathPolicy`. A custom `PathPolicy` must declare its `allowedRoots`. -- `code` -- `message` -- `functionName` -- `capability` -- `line` -- `column` -- `suggestions` -- `diagnostics` -- `logs` -- `permissionEvents` +**Network.** `NetworkAccessPolicy.standard` blocks: -For hosts that vend these methods as LLM tool calls, use `CodeModeAgentToolDescriptions.searchJavaScriptAPI` and `CodeModeAgentToolDescriptions.executeJavaScript` as the canonical tool descriptions. +- loopback, private, link-local, CGNAT, and unique-local addresses, including cloud metadata endpoints; +- `localhost`, `.local`, `.internal`, and single-label hosts. -## Host App Permissions and Capabilities +It also re-checks redirect targets and caps response bodies at 10 MB. Matching is by hostname without DNS resolution, so set `allowedHosts` if you need a strict list. `fetch` runs on an isolated ephemeral session, with no cookies, stored credentials, or shared cache, so a script can't use whatever your app is signed in to. Scripts can't set `Cookie`, `Authorization`, or `Proxy-*` headers unless you set `allowsCredentialHeaders`. -Host apps must provide privacy usage strings for bridged APIs that request protected resources. +**Resources.** `timeoutMs` (default 10 s, maximum 60 s) bounds the whole execution, including time spent in native calls. It is enforced preemptively, so `while (true) {}` is terminated, not waited out. `ExecutionLimits` bounds result size, logs, diagnostics, buffered events, and concurrent executions (8 by default). Each limit truncates with a diagnostic rather than failing. -Required Info.plist keys by capability: +> **App Store note.** Preemption uses `JSContextGroupSetExecutionTimeLimit`. JavaScriptCore exports it but declares it only in a private header, and CodeMode reaches it through a small C shim (`CCodeModeJSC`). It is JavaScriptCore's only way to interrupt a runaway script. -- Location read (`location.read`): `NSLocationWhenInUseUsageDescription` -- Location permission request (`location.permission.request`, iOS-only): `NSLocationWhenInUseUsageDescription` -- Contacts (`contacts.read`, `contacts.search`, `contacts.ui.presentContact`, `contacts.ui.presentNewContact`): `NSContactsUsageDescription` -- Calendar read (`calendar.read`): `NSCalendarsFullAccessUsageDescription` -- Calendar event detail UI (`calendar.ui.presentEvent`): `NSCalendarsFullAccessUsageDescription` -- Calendar write-only (`calendar.write`): `NSCalendarsWriteOnlyAccessUsageDescription` -- Calendar event editor/chooser UI (`calendar.ui.presentNewEvent`, `calendar.ui.pickCalendar`): `NSCalendarsWriteOnlyAccessUsageDescription` -- Reminders (`reminders.read`, `reminders.write`): `NSRemindersFullAccessUsageDescription` -- Photos (`photos.read`, `photos.export`, `photos.ui.presentLimitedLibraryPicker`): `NSPhotoLibraryUsageDescription` -- Camera UI (`camera.ui.capture`, `camera.ui.scanData`, `documents.ui.scan`): `NSCameraUsageDescription` -- Camera video capture (`camera.ui.capture`): `NSMicrophoneUsageDescription` -- AlarmKit (`alarm.permission.request`, `alarm.read`, `alarm.schedule`, `alarm.cancel`): `NSAlarmKitUsageDescription` -- HealthKit read (`health.permission.request`, `health.read`): `NSHealthShareUsageDescription` -- HealthKit write (`health.permission.request`, `health.write`): `NSHealthUpdateUsageDescription` -- HomeKit (`home.read`, `home.write`): `NSHomeKitUsageDescription` +## Adding your own helpers -Notifications: +Expose app functionality alongside the built-ins with the `@CodeMode` macro from the `CodeModeAuthoring` product: -- Local notification scheduling and management (`notifications.*`) requires runtime authorization via `apple.notifications.requestPermission()` -- No additional Info.plist privacy string is required for `UNUserNotificationCenter` authorization prompts +```swift +import CodeModeAuthoring -AlarmKit: +@CodeMode(path: "myapp.tasks.complete", description: "Mark a task complete.") +struct TaskComplete: Sendable { + let store: TaskStore -- `alarm.*` requires `iOS 26+` and runtime authorization via `ios.alarm.requestPermission()` + struct Arguments { + @CodeModeParam("Task identifier") var id: String + } -HealthKit: + func call(arguments: Arguments) async throws { + try await store.complete(id: arguments.id) + } +} -- `health.*` requires the HealthKit entitlement and runtime authorization via `apple.health.requestPermission(...)` +let tools = CodeModeAgentTools( + config: CodeModeConfiguration(codeModeProviders: [TaskComplete(store: store)]) +) +``` -Weather: +Scripts call it as `await myapp.tasks.complete({ id: "task-123" })`, and the model requests it through `allowedCapabilityKeys`. It appears in search and in the TypeScript declarations like any built-in. `register(provider:)` adds providers after construction. See [the CodeModeAuthoring README](Sources/CodeModeAuthoring/README.md) for supported types and structured errors. -- `weather.read` requires enabling the WeatherKit capability on the app target -- Weather does not require a separate privacy prompt in this package because `weather.read` expects explicit coordinates +## Host app setup -Programmatic validation helper: +Bridged APIs need the usual privacy strings and entitlements. `HostConfigurationValidator` checks them for you: ```swift -import CodeMode - -let required: Set = [ - .locationRead, - .contactsSearch, - .weatherRead, -] - -let issues = HostConfigurationValidator.validate(requiredCapabilities: required) -for issue in issues { - print("[\(issue.severity.rawValue)] \(issue.key): \(issue.message)") -} +let issues = HostConfigurationValidator.validate(requiredCapabilities: [.locationRead, .contactsSearch]) + + HostConfigurationValidator.validate(configuration: config) // e.g. warns on an unrestricted grant ``` -## Development +| Capabilities | Info.plist key | +| --- | --- | +| `location.read`, `location.permission.request` (iOS) | `NSLocationWhenInUseUsageDescription` | +| `contacts.read`, `contacts.search`, `contacts.ui.*` | `NSContactsUsageDescription` | +| `calendar.read`, `calendar.ui.presentEvent` | `NSCalendarsFullAccessUsageDescription` | +| `calendar.write`, `calendar.ui.presentNewEvent`, `calendar.ui.pickCalendar` | `NSCalendarsWriteOnlyAccessUsageDescription` | +| `reminders.read`, `reminders.write` | `NSRemindersFullAccessUsageDescription` | +| `photos.read`, `photos.export`, `photos.ui.presentLimitedLibraryPicker` | `NSPhotoLibraryUsageDescription` | +| `camera.ui.capture`, `camera.ui.scanData`, `documents.ui.scan` | `NSCameraUsageDescription` | +| `camera.ui.capture` (video) | `NSMicrophoneUsageDescription` | +| `alarm.*` (iOS 26+) | `NSAlarmKitUsageDescription` | +| `health.permission.request`, `health.read` / `health.write` | `NSHealthShareUsageDescription` / `NSHealthUpdateUsageDescription` | +| `home.read`, `home.write` | `NSHomeKitUsageDescription` | + +HealthKit also needs its entitlement, and `weather.read` needs the WeatherKit capability. Notifications, alarms, and HealthKit authorize at runtime through their `requestPermission` helpers. -- Run the deterministic CodeMode eval harness: - -```sh -swift run --package-path Tools/CodeModeEval codemode-eval -swift run --package-path Tools/CodeModeEval codemode-eval list -swift run --package-path Tools/CodeModeEval codemode-eval run fs.round-trip --show-code -swift run --package-path Tools/CodeModeEval codemode-eval run --json -``` - -The eval harness runs 49 built-in user-style scenarios through the same -`searchJavaScriptAPI` and `executeJavaScript` APIs that host apps expose to -agents. It validates tool order, discovered catalog output, generated JavaScript -fragments, exact `allowedCapabilities`, structured errors, repair suggestions, -console logs, diagnostics, and final output. The scenarios cover filesystem -workflows, capability minimization, path policy failures, permission failures, -catalog search behavior, helper suggestions, API argument-shape confusion, -recovery after structured tool errors, execution timeouts, and catalog coverage -for the expanded Apple API families. - -- Preview LLM eval suites and work with saved live reports: +## Development ```sh -swift run --package-path Tools/CodeModeEval codemode-eval plan --suite core --repeat 5 --request-delay-ms 1000 -swift run --package-path Tools/CodeModeEval codemode-eval summarize Tools/CodeModeEval/.build/reports/core-baseline.json --output Tools/CodeModeEval/.build/reports/core-summary.json -swift run --package-path Tools/CodeModeEval codemode-eval report Tools/CodeModeEval/.build/reports/core-baseline.json --output Tools/CodeModeEval/.build/reports/core-baseline.md -swift run --package-path Tools/CodeModeEval codemode-eval compare Tools/CodeModeEval/.build/reports/core-baseline.json Tools/CodeModeEval/.build/reports/core-candidate.json +swift test # unit and integration tests +swift run --package-path Tools/CodeModeEval codemode-eval # 52 deterministic agent scenarios +scripts/check-typescript-declarations.sh # tsc --strict over every declaration ``` -The default eval package intentionally avoids private Wavelike dependencies, so -`swift build --package-path Tools/CodeModeEval` works in public or unauthenticated -CI. Use `plan` to preview request budgets, `summarize` to strip raw transcripts -before committing baselines, `report` to generate Markdown diagnostics with -tool-attempt retry traces, then `compare` to fail on pass-rate, exact-capability, -retry, or turn-count regressions. Tolerances are configurable with -`--pass-rate-tolerance`, `--capability-rate-tolerance`, `--retry-tolerance`, and -`--turn-tolerance`. The saved JSON report envelope includes raw run results plus -aggregate pass rate, average turns, retry count, exact/minimal capability -success, per-scenario metrics, and failure categories such as `wrong_tool`, -`wrong_js`, `overbroad_capability`, `failed_recovery`, and `no_final_answer`. -See [EVALS.md](EVALS.md) for CI/nightly policy, baseline handling, and -recommended commands. -The CLI lives in `Tools/CodeModeEval` so library consumers do not resolve -ArgumentParser when they use the `CodeMode` product. - -- License: MIT. See [LICENSE](LICENSE) +The eval scenarios drive the two tools the way an agent would. They grade the tool order, the generated code, whether the requested capabilities were exactly the minimal set, structured errors, and recovery after a failure. The same CLI plans, summarizes, and compares live-model runs; see [EVALS.md](EVALS.md). It lives in `Tools/CodeModeEval`, so library users don't resolve ArgumentParser. ## Acknowledgements -The "Code Mode" framing and the tool-oriented search and execution model in this repository were influenced by Cloudflare's Code Mode work: - -- [Cloudflare Code Mode API reference](https://developers.cloudflare.com/agents/api-reference/codemode/) -- [Cloudflare Code Mode announcement](https://blog.cloudflare.com/code-mode/) -- [Cloudflare Code Mode MCP](https://blog.cloudflare.com/code-mode-mcp/) - -This repository is an independent implementation and is not affiliated with Cloudflare. - -## Development Process +The Code Mode framing and its search-and-execute model come from Cloudflare's work: the [announcement](https://blog.cloudflare.com/code-mode/), the [API reference](https://developers.cloudflare.com/agents/api-reference/codemode/), and [Code Mode MCP](https://blog.cloudflare.com/code-mode-mcp/). This is an independent implementation, not affiliated with Cloudflare. Development of `CodeMode.swift` was done exclusively with Codex, initiated by an interactively built plan, executed by the model after the plan was finalized. -## Shipped Expanded Families - -These API families are represented in the catalog and local bridge layer on this -branch: - -- APNs / remote-notification token and settings lifecycle under `apple.notifications.*` -- PassKit wallet APIs under `apple.wallet.*` -- Speech, MusicKit, Foundation Models, CloudKit, Maps, StoreKit, App Intents, and ActivityKit namespaces - -## Deferred in v1 - -The package intentionally defers these to later phases: - -- Production host entitlement provisioning and app-specific UX for newly bridged frameworks -- Broader executable eval coverage for every catalog-only capability family -- Behavior-preserving refactors that generate more JavaScript bindings from registration metadata +License: MIT. See [LICENSE](LICENSE). diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index e922dd9..f3432ef 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -3,7 +3,15 @@ import Foundation public struct CodeModeConfiguration: Sendable { public var pathPolicy: any PathPolicy public var networkAccessPolicy: NetworkAccessPolicy + /// Host-owned ceiling on the model-authored `allowedCapabilities` / + /// `allowedCapabilityKeys`. Defaults to `.unrestricted`; see `CapabilityGrant`. + public var capabilityGrant: CapabilityGrant public var fileSystem: any CodeModeFileSystem + /// Byte ceilings applied by the filesystem bridge to `fs.read` / `fs.write`. + public var fileSystemLimits: FileSystemLimits + /// Bounds on retained result/log/event volume per execution, and on how + /// many executions run concurrently. + public var executionLimits: ExecutionLimits public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker public var auditLogger: any AuditLogger @@ -25,7 +33,10 @@ public struct CodeModeConfiguration: Sendable { public init( pathPolicy: any PathPolicy = DefaultPathPolicy(), networkAccessPolicy: NetworkAccessPolicy = .standard, + capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, + executionLimits: ExecutionLimits = .standard, artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), auditLogger: any AuditLogger = SyncAuditLogger(), @@ -46,7 +57,10 @@ public struct CodeModeConfiguration: Sendable { ) { self.pathPolicy = pathPolicy self.networkAccessPolicy = networkAccessPolicy + self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem + self.fileSystemLimits = fileSystemLimits + self.executionLimits = executionLimits self.artifactStore = artifactStore self.permissionBroker = permissionBroker self.auditLogger = auditLogger @@ -89,6 +103,10 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { public var argumentHints: [String: String] public var argumentConstraints: CapabilityArgumentConstraints public var resultSummary: String + /// TypeScript declaration for this helper, generated from the fields above. + /// Return it from `searchJavaScriptAPI` to give the model real types rather + /// than a coarse `argumentTypes` map plus prose. + public var dts: String public init( capability: String, @@ -103,7 +121,8 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { argumentTypes: [String: CapabilityArgumentType], argumentHints: [String: String], argumentConstraints: CapabilityArgumentConstraints = .none, - resultSummary: String + resultSummary: String, + dts: String = "" ) { self.capability = capability self.capabilityKey = capabilityKey ?? CodeModeCapabilityKey(rawValue: capability) @@ -118,6 +137,38 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { self.argumentHints = argumentHints self.argumentConstraints = argumentConstraints self.resultSummary = resultSummary + self.dts = dts + } + + private enum CodingKeys: String, CodingKey { + case capability, capabilityKey, builtInCapability, jsNames, summary, tags, example + case requiredArguments, optionalArguments, argumentTypes, argumentHints + case argumentConstraints, resultSummary, dts + } + + /// `dts` is optional on the way in. References encoded before it existed — + /// cached search results, eval fixtures — would otherwise fail to decode with + /// `keyNotFound`; when it is absent, the declaration is regenerated from the + /// decoded fields rather than left empty. + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.capability = try container.decode(String.self, forKey: .capability) + self.capabilityKey = try container.decode(CodeModeCapabilityKey.self, forKey: .capabilityKey) + self.builtInCapability = try container.decodeIfPresent(CapabilityID.self, forKey: .builtInCapability) + self.jsNames = try container.decode([String].self, forKey: .jsNames) + self.summary = try container.decode(String.self, forKey: .summary) + self.tags = try container.decode([String].self, forKey: .tags) + self.example = try container.decode(String.self, forKey: .example) + self.requiredArguments = try container.decode([String].self, forKey: .requiredArguments) + self.optionalArguments = try container.decode([String].self, forKey: .optionalArguments) + self.argumentTypes = try container.decode([String: CapabilityArgumentType].self, forKey: .argumentTypes) + self.argumentHints = try container.decode([String: String].self, forKey: .argumentHints) + self.argumentConstraints = try container.decode(CapabilityArgumentConstraints.self, forKey: .argumentConstraints) + self.resultSummary = try container.decode(String.self, forKey: .resultSummary) + self.dts = try container.decodeIfPresent(String.self, forKey: .dts) ?? "" + if self.dts.isEmpty { + self.dts = TypeScriptDeclarations.declaration(for: self) + } } } @@ -132,6 +183,13 @@ public struct JavaScriptAPISearchResponse: Sendable, Codable, Equatable { } public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { + /// Bounds advertised in `executeJavaScriptParameterSchema`. Values outside + /// them are clamped rather than rejected: a model that asks for a 10-minute + /// budget should get the ceiling and a running script, not a hard failure. + public static let minimumTimeoutMs = 1 + public static let maximumTimeoutMs = 60_000 + public static let defaultTimeoutMs = 10_000 + public var code: String public var allowedCapabilities: [CapabilityID] public var allowedCapabilityKeys: [CodeModeCapabilityKey] @@ -142,16 +200,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { code: String, allowedCapabilities: [CapabilityID], allowedCapabilityKeys: [CodeModeCapabilityKey] = [], - timeoutMs: Int = 10_000, + timeoutMs: Int = JavaScriptExecutionRequest.defaultTimeoutMs, context: ExecutionContext = .init() ) { self.code = code self.allowedCapabilities = allowedCapabilities self.allowedCapabilityKeys = allowedCapabilityKeys - self.timeoutMs = timeoutMs + self.timeoutMs = Self.clampTimeoutMs(timeoutMs) self.context = context } + static func clampTimeoutMs(_ value: Int) -> Int { + min(max(value, minimumTimeoutMs), maximumTimeoutMs) + } + private enum CodingKeys: String, CodingKey { case code case allowedCapabilities @@ -194,10 +256,13 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { self.allowedCapabilities = capabilities self.allowedCapabilityKeys = try container.decodeIfPresent([CodeModeCapabilityKey].self, forKey: .allowedCapabilityKeys) ?? [] - self.timeoutMs = try Self.decodeTimeoutMs(from: container) ?? 10_000 + self.timeoutMs = Self.clampTimeoutMs(try Self.decodeTimeoutMs(from: container) ?? Self.defaultTimeoutMs) self.context = try container.decodeIfPresent(ExecutionContext.self, forKey: .context) ?? .init() } + // Every conversion here is total. `Int.init(_: Double)` traps on non-finite + // and out-of-range input, and this runs on model-authored JSON before any + // script does — `{"timeoutMs": 1e300}` would kill the host process in-process. private static func decodeTimeoutMs(from container: KeyedDecodingContainer) throws -> Int? { guard container.contains(.timeoutMs), try !container.decodeNil(forKey: .timeoutMs) else { return nil @@ -206,11 +271,11 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { return value } if let value = try? container.decode(Double.self, forKey: .timeoutMs) { - return Int(value) + return try requireRepresentable(value, in: container) } if let string = try? container.decode(String.self, forKey: .timeoutMs), let value = Double(string.trimmingCharacters(in: .whitespaces)) { - return Int(value) + return try requireRepresentable(value, in: container) } throw DecodingError.dataCorruptedError( forKey: .timeoutMs, @@ -218,6 +283,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { debugDescription: "timeoutMs must be an integer or a numeric string." ) } + + private static func requireRepresentable( + _ value: Double, + in container: KeyedDecodingContainer + ) throws -> Int { + guard let converted = JSONValue.exactInt(from: value) else { + throw DecodingError.dataCorruptedError( + forKey: .timeoutMs, + in: container, + debugDescription: "timeoutMs must be a finite number representable as an integer; received \(value)." + ) + } + return converted + } } public enum JavaScriptExecutionEvent: Sendable, Equatable { @@ -299,22 +378,32 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { public let events: AsyncStream private let resultTask: Task + /// Keeps the execution alive while this handle is; the events stream holds + /// the same token. When both are gone the execution is cancelled, so a call + /// nobody can observe does not keep touching the filesystem, the network, + /// and system UI on the user's behalf. + private let observation: ExecutionObservationToken private let cancelImpl: @Sendable () -> Void init( events: AsyncStream, resultTask: Task, + observation: ExecutionObservationToken, cancelImpl: @escaping @Sendable () -> Void ) { self.events = events self.resultTask = resultTask + self.observation = observation self.cancelImpl = cancelImpl } public var result: JavaScriptExecutionResult { get async throws { let outcome = await withTaskCancellationHandler { - await waitForResultOutcome() + // `resultTask.result` already detaches the await from the calling + // task's cancellation, so the previous `Task.detached` per access + // bought nothing but an extra task. + await resultTask.result } onCancel: { self.cancel() } @@ -324,8 +413,7 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { return result case let .failure(error as CodeModeToolError): throw error - case let .failure(error as CancellationError): - _ = error + case .failure(is CancellationError): throw CodeModeToolError(code: "CANCELLED", message: "Execution cancelled") case let .failure(error): throw error @@ -337,18 +425,6 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { cancelImpl() resultTask.cancel() } - - private func waitForResultOutcome() async -> Result { - await withCheckedContinuation { continuation in - Task.detached { - do { - continuation.resume(returning: .success(try await self.resultTask.value)) - } catch { - continuation.resume(returning: .failure(error)) - } - } - } - } } public struct ExecutionContext: Sendable, Codable, Equatable { diff --git a/Sources/CodeMode/API/SystemAppleServiceClients.swift b/Sources/CodeMode/API/SystemAppleServiceClients.swift index 1f3db40..c501be0 100644 --- a/Sources/CodeMode/API/SystemAppleServiceClients.swift +++ b/Sources/CodeMode/API/SystemAppleServiceClients.swift @@ -819,8 +819,14 @@ enum SystemMapsMapping { private func openSystemURL(_ url: URL) -> Bool { #if canImport(UIKit) if Thread.isMainThread { - UIApplication.shared.open(url, options: [:], completionHandler: nil) - return true + // `UIApplication.shared` and `open` are main-actor-isolated. We have + // already established we are on the main thread, so state that to the + // compiler rather than hopping — a hop here would return before the open + // has been requested. + return MainActor.assumeIsolated { + UIApplication.shared.open(url, options: [:], completionHandler: nil) + return true + } } let semaphore = DispatchSemaphore(value: 0) let opened = LockedBox(false) diff --git a/Sources/CodeMode/Bridges/AlarmBridge.swift b/Sources/CodeMode/Bridges/AlarmBridge.swift index 4736f96..6931d19 100644 --- a/Sources/CodeMode/Bridges/AlarmBridge.swift +++ b/Sources/CodeMode/Bridges/AlarmBridge.swift @@ -139,13 +139,7 @@ public final class AlarmBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text else { return nil } - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: text) { - return date - } - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: text) + CodeModeDate.parse(text) } #if canImport(AlarmKit) && canImport(SwiftUI) && os(iOS) diff --git a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift index 55090e8..1160d8c 100644 --- a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift +++ b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift @@ -111,7 +111,12 @@ public final class SpeechBridge: @unchecked Sendable { try ensurePermission(.speechRecognition, context: context) let path = try requireNonEmptyString("path", in: arguments, capability: "speech.file.transcribe") let resolved = try context.pathPolicy.resolve(path: path) + // The raw, script-supplied `path` is replaced rather than accompanied: + // leaving both in the dictionary made every host adapter one + // `arguments.string("path")` away from a path-policy bypass, with nothing + // in the code saying which key was the safe one. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) try validateOptionalPositiveNumber("timeoutMs", in: arguments, capability: "speech.file.transcribe") return try client.transcribeFile(arguments: clientArguments) @@ -348,7 +353,9 @@ public final class PassKitBridge: @unchecked Sendable { public func addPass(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let path = try requireNonEmptyString("path", in: arguments, capability: "passkit.pass.add") let resolved = try context.pathPolicy.resolve(path: path) + // As above: the raw path does not survive into the client dictionary. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) return try client.addPass(arguments: clientArguments) } diff --git a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift index 706b4af..1c210e3 100644 --- a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift +++ b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift @@ -14,7 +14,7 @@ extension DefaultCapabilityRegistrationBuilder { descriptor: .init( id: .networkFetch, title: "Fetch HTTP resource", - summary: "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + summary: "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", tags: ["network", "http", "fetch"], example: "await fetch('https://api.example.com/data').then(r => r.json())", requiredArguments: ["url"], @@ -29,7 +29,7 @@ extension DefaultCapabilityRegistrationBuilder { argumentHints: [ "url": "Absolute HTTP(S) URL string.", "options.method": "HTTP method; defaults to GET.", - "options.headers": "Object of header key/value string pairs.", + "options.headers": "Object of header key/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.body": "UTF-8 request body string.", "options.bodyBase64": "Base64-encoded request body. Mutually exclusive with options.body.", "options.timeoutMs": "Request and bridge wait timeout in milliseconds; defaults to 30000.", diff --git a/Sources/CodeMode/Bridges/CodeModeFileSystem.swift b/Sources/CodeMode/Bridges/CodeModeFileSystem.swift index 02057ea..1c8699d 100644 --- a/Sources/CodeMode/Bridges/CodeModeFileSystem.swift +++ b/Sources/CodeMode/Bridges/CodeModeFileSystem.swift @@ -5,12 +5,16 @@ public struct CodeModeFileSystemEntry: Sendable, Equatable { public var path: String public var isDirectory: Bool public var size: Int + /// Hidden from display listings (`fs.list`), but still a real entry: it is + /// copied, moved, and counted against limits like any other. + public var isHidden: Bool - public init(name: String, path: String, isDirectory: Bool, size: Int) { + public init(name: String, path: String, isDirectory: Bool, size: Int, isHidden: Bool = false) { self.name = name self.path = path self.isDirectory = isDirectory self.size = size + self.isHidden = isHidden } } @@ -39,6 +43,14 @@ public struct CodeModeFileSystemAccess: Sendable, Equatable { } public protocol CodeModeFileSystem: Sendable { + /// Every entry in the directory, *including hidden ones* — mark those with + /// `isHidden` rather than omitting them. + /// + /// The filesystem bridge measures directory trees through this listing to + /// enforce `FileSystemLimits` on `fs.copy`, while `copyItem` copies + /// everything. A listing that skips hidden entries therefore lets a hidden + /// file or subtree of any size through the limit. Hiding is a display + /// concern; `fs.list` filters on `isHidden` itself. func listDirectory(at url: URL) throws -> [CodeModeFileSystemEntry] func readData(at url: URL) throws -> Data func writeData(_ data: Data, to url: URL) throws @@ -59,19 +71,24 @@ public final class LocalCodeModeFileSystem: CodeModeFileSystem, @unchecked Senda } public func listDirectory(at url: URL) throws -> [CodeModeFileSystemEntry] { + // No `.skipsHiddenFiles`: see the protocol requirement. Skipping them + // here let hidden entries bypass the copy limit. let values = try fileManager.contentsOfDirectory( at: url, - includingPropertiesForKeys: [.isDirectoryKey, .fileSizeKey], - options: [.skipsHiddenFiles] + includingPropertiesForKeys: [.isDirectoryKey, .fileSizeKey, .isHiddenKey], + options: [] ) return try values.map { item in - let resourceValues = try item.resourceValues(forKeys: [.isDirectoryKey, .fileSizeKey]) + let resourceValues = try item.resourceValues(forKeys: [.isDirectoryKey, .fileSizeKey, .isHiddenKey]) return CodeModeFileSystemEntry( name: item.lastPathComponent, path: item.path, isDirectory: resourceValues.isDirectory ?? false, - size: resourceValues.fileSize ?? 0 + size: resourceValues.fileSize ?? 0, + // The same test `.skipsHiddenFiles` applied: dot-prefixed names + // and the filesystem's hidden flag. + isHidden: resourceValues.isHidden ?? item.lastPathComponent.hasPrefix(".") ) } } diff --git a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift index 96eb81e..3985f55 100644 --- a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift +++ b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift @@ -3,6 +3,7 @@ import Foundation public enum DefaultCapabilityLoader { public static func loadAllRegistrations( fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, networkAccessPolicy: NetworkAccessPolicy = .standard, eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), cloudKitClient: any CloudKitClient = UnavailableCloudKitClient(), @@ -18,6 +19,7 @@ public enum DefaultCapabilityLoader { ) -> [CapabilityRegistration] { DefaultCapabilityRegistrationBuilder( fileSystem: fileSystem, + fileSystemLimits: fileSystemLimits, networkAccessPolicy: networkAccessPolicy, eventInbox: eventInbox, cloudKitClient: cloudKitClient, @@ -64,6 +66,7 @@ struct DefaultCapabilityRegistrationBuilder { init( fileSystem: any CodeModeFileSystem, + fileSystemLimits: FileSystemLimits, networkAccessPolicy: NetworkAccessPolicy, eventInbox: any CodeModeEventInbox, cloudKitClient: any CloudKitClient, @@ -77,7 +80,7 @@ struct DefaultCapabilityRegistrationBuilder { passKitClient: any PassKitClient, storeKitClient: any StoreKitClient ) { - self.fs = FileSystemBridge(fileSystem: fileSystem) + self.fs = FileSystemBridge(fileSystem: fileSystem, limits: fileSystemLimits) self.network = NetworkBridge(policy: networkAccessPolicy) self.keychain = KeychainBridge() self.location = LocationBridge() @@ -91,7 +94,7 @@ struct DefaultCapabilityRegistrationBuilder { self.health = HealthBridge() self.home = HomeBridge() self.media = MediaBridge() - self.systemUI = SystemUIBridge() + self.systemUI = SystemUIBridge(networkAccessPolicy: networkAccessPolicy) self.eventInbox = eventInbox self.cloudKit = CloudKitBridge(client: cloudKitClient, eventInbox: eventInbox) self.remoteNotifications = RemoteNotificationsBridge(client: remoteNotificationsClient, eventInbox: eventInbox) diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index 279c31b..9750c4a 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -5,6 +5,8 @@ import EventKit #endif public final class EventKitBridge: @unchecked Sendable { + static let reminderFetchTimeoutSeconds: TimeInterval = 15 + public init() {} public func readEvents(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -15,8 +17,13 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let start = isoDate(arguments.string("start")) ?? Date() - let end = isoDate(arguments.string("end")) ?? Calendar.current.date(byAdding: .day, value: 14, to: start) ?? start + // A value that is present but unparseable is an error, not a reason to + // fall back to now/+14d: that handed the script a plausible but wrong + // window with no diagnostic to notice it by. + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "calendar.read") ?? Date() + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "calendar.read") + ?? Calendar.current.date(byAdding: .day, value: 14, to: start) + ?? start let limit = arguments.int("limit") ?? 50 let calendars = try resolveCalendars( from: arguments, @@ -97,11 +104,9 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - var result: [JSONValue] = [] let includeCompleted = arguments.bool("includeCompleted") ?? false - let start = isoDate(arguments.string("start")) - let end = isoDate(arguments.string("end")) + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "reminders.read") + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "reminders.read") let limit = max(1, arguments.int("limit") ?? 50) let calendars = try resolveCalendars( from: arguments, @@ -113,23 +118,30 @@ public final class EventKitBridge: @unchecked Sendable { let predicate = includeCompleted ? store.predicateForReminders(in: calendars) : store.predicateForIncompleteReminders(withDueDateStarting: start, ending: end, calendars: calendars) - store.fetchReminders(matching: predicate) { reminders in - let filtered = (reminders ?? []) - .filter { reminder in - guard includeCompleted else { return true } - guard let dueDate = reminder.dueDateComponents?.date else { - return start == nil && end == nil + // Previously a `var` written from EventKit's completion queue, read after + // a semaphore wait whose timeout result was discarded — a data race with + // the late callback, and an empty array on timeout that the script could + // not tell apart from "no reminders". + let result = try CompletionWait.value( + timeout: Self.reminderFetchTimeoutSeconds, + operationName: "reminders.read" + ) { complete in + store.fetchReminders(matching: predicate) { reminders in + let filtered = (reminders ?? []) + .filter { reminder in + guard includeCompleted else { return true } + guard let dueDate = reminder.dueDateComponents?.date else { + return start == nil && end == nil + } + if let start, dueDate < start { return false } + if let end, dueDate > end { return false } + return true } - if let start, dueDate < start { return false } - if let end, dueDate > end { return false } - return true - } - .prefix(limit) - result = filtered.map { Self.reminderJSON($0) } - semaphore.signal() + .prefix(limit) + complete(filtered.map { Self.reminderJSON($0) }) + } } - _ = semaphore.wait(timeout: .now() + 15) return .array(result) #else _ = arguments @@ -191,8 +203,7 @@ public final class EventKitBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text, text.isEmpty == false else { return nil } - return ISO8601DateFormatter().date(from: text) + CodeModeDate.parse(text) } private func eventOperation(_ arguments: [String: JSONValue]) throws -> CalendarWriteOperation { @@ -480,6 +491,17 @@ public final class EventKitBridge: @unchecked Sendable { return span.ekSpan } + /// Test seam: these serializers are the only EventKit logic reachable + /// without a live, permission-granted store, and they are where the + /// implicitly-unwrapped `calendar`/`title` crashes lived. + static func eventJSONForTesting(_ event: EKEvent) -> JSONValue { + eventJSON(event) + } + + static func reminderJSONForTesting(_ reminder: EKReminder) -> JSONValue { + reminderJSON(reminder) + } + private static func eventJSON(_ event: EKEvent) -> JSONValue { .object([ "identifier": .string(event.eventIdentifier ?? ""), @@ -487,8 +509,11 @@ public final class EventKitBridge: @unchecked Sendable { "startDate": .string(event.startDate.ISO8601Format()), "endDate": .string(event.endDate.ISO8601Format()), "notes": .string(event.notes ?? ""), - "calendarIdentifier": .string(event.calendar.calendarIdentifier), - "calendarTitle": .string(event.calendar.title), + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned + // event — every neighboring field here is nil-coalesced, and this one + // crashed the execution thread. + "calendarIdentifier": .string(event.calendar?.calendarIdentifier ?? ""), + "calendarTitle": .string(event.calendar?.title ?? ""), "location": .string(event.location ?? ""), "url": .string(event.url?.absoluteString ?? ""), "isAllDay": .bool(event.isAllDay), @@ -498,7 +523,8 @@ public final class EventKitBridge: @unchecked Sendable { private static func reminderJSON(_ reminder: EKReminder) -> JSONValue { var object: [String: JSONValue] = [ "identifier": .string(reminder.calendarItemIdentifier), - "title": .string(reminder.title), + // `EKReminder.title` is `String!`. + "title": .string(reminder.title ?? ""), "isCompleted": .bool(reminder.isCompleted), "dueDate": .string(reminder.dueDateComponents?.date?.ISO8601Format() ?? ""), "completionDate": .string(reminder.completionDate?.ISO8601Format() ?? ""), diff --git a/Sources/CodeMode/Bridges/FileSystemBridge.swift b/Sources/CodeMode/Bridges/FileSystemBridge.swift index a4b8be9..93b0d25 100644 --- a/Sources/CodeMode/Bridges/FileSystemBridge.swift +++ b/Sources/CodeMode/Bridges/FileSystemBridge.swift @@ -1,14 +1,40 @@ import Foundation +/// Byte ceilings for the filesystem bridge. +/// +/// A sandbox root can hold files far larger than a script's result budget — a +/// 2 GB video in `documents:` read whole and base64-serialized is an immediate +/// jetsam on iOS — so reads and writes are bounded and fail with a diagnostic +/// instead of exhausting memory. +public struct FileSystemLimits: Sendable, Equatable { + public var maxReadBytes: Int + public var maxWriteBytes: Int + + public init(maxReadBytes: Int = 16 * 1_024 * 1_024, maxWriteBytes: Int = 16 * 1_024 * 1_024) { + self.maxReadBytes = maxReadBytes + self.maxWriteBytes = maxWriteBytes + } + + public static let standard = FileSystemLimits() + + /// No ceiling. Only appropriate when the host has its own quota. + public static let unlimited = FileSystemLimits(maxReadBytes: .max, maxWriteBytes: .max) +} + public final class FileSystemBridge: @unchecked Sendable { private let fileSystem: any CodeModeFileSystem + private let limits: FileSystemLimits - public init(fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem()) { + public init( + fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + limits: FileSystemLimits = .standard + ) { self.fileSystem = fileSystem + self.limits = limits } - public convenience init(fileManager: FileManager) { - self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager)) + public convenience init(fileManager: FileManager, limits: FileSystemLimits = .standard) { + self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager), limits: limits) } public func list(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -17,7 +43,10 @@ public final class FileSystemBridge: @unchecked Sendable { } let url = try context.pathPolicy.resolve(path: path) - let entries: [JSONValue] = try fileSystem.listDirectory(at: url).map { item in + // Hidden entries stay out of the listing, as before; the filtering just + // lives here now rather than in the storage layer, where it also hid + // them from copy-limit accounting. + let entries: [JSONValue] = try fileSystem.listDirectory(at: url).filter { !$0.isHidden }.map { item in return .object([ "name": .string(item.name), "path": .string(item.path), @@ -36,11 +65,23 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) + + // Checked against stat first: reading and then rejecting has already spent + // the memory the cap exists to protect. Re-checked after, because a + // custom CodeModeFileSystem's stat may be absent or stale. + try enforceReadLimit(try? fileSystem.attributesOfItem(at: url).size, at: url) let data = try fileSystem.readData(at: url) + try enforceReadLimit(data.count, at: url) switch encoding.lowercased() { case "utf8", "utf-8": - let text = String(data: data, encoding: .utf8) ?? "" + // Silently substituting "" for undecodable bytes tells the script the + // file is empty, which is worse than a failure it can act on. + guard let text = String(data: data, encoding: .utf8) else { + throw BridgeError.invalidArguments( + "fs.read could not decode \(url.lastPathComponent) as UTF-8. Retry with encoding: 'base64' for binary data." + ) + } return .object([ "path": .string(url.path), "text": .string(text), @@ -63,9 +104,6 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) - let parent = url.deletingLastPathComponent() - try fileSystem.createDirectory(at: parent, recursive: true) - let data: Data switch encoding.lowercased() { case "utf8", "utf-8": @@ -80,6 +118,13 @@ public final class FileSystemBridge: @unchecked Sendable { throw BridgeError.invalidArguments("Unsupported encoding: \(encoding)") } + guard data.count <= limits.maxWriteBytes else { + throw Self.overLimit("fs.write", url: url, bytes: data.count, limit: limits.maxWriteBytes) + } + + // Only after the payload is known good, so a rejected write leaves no + // directories behind. + try fileSystem.createDirectory(at: url.deletingLastPathComponent(), recursive: true) try fileSystem.writeData(data, to: url) return .object([ "path": .string(url.path), @@ -94,12 +139,11 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) + try validateTransfer(from: fromURL, to: toURL, operation: "fs.move", arguments: arguments, context: context) - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) + try replacing(toURL, operation: "fs.move", arguments: arguments) { + try fileSystem.moveItem(at: fromURL, to: toURL) } - - try fileSystem.moveItem(at: fromURL, to: toURL) return .object([ "from": .string(fromURL.path), "to": .string(toURL.path), @@ -113,18 +157,154 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) + try validateTransfer(from: fromURL, to: toURL, operation: "fs.copy", arguments: arguments, context: context) - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) + // A copy writes new bytes, so it answers to the write limit — otherwise + // copying a large tree is a way around it, and fills the disk. + let bytes = try totalSize(of: fromURL, stoppingAbove: limits.maxWriteBytes) + guard bytes <= limits.maxWriteBytes else { + throw Self.overLimit("fs.copy", url: fromURL, bytes: bytes, limit: limits.maxWriteBytes) } - try fileSystem.copyItem(at: fromURL, to: toURL) + try replacing(toURL, operation: "fs.copy", arguments: arguments) { + try fileSystem.copyItem(at: fromURL, to: toURL) + } return .object([ "from": .string(fromURL.path), "to": .string(toURL.path), ]) } + private func enforceReadLimit(_ bytes: Int?, at url: URL) throws { + guard let bytes, bytes > limits.maxReadBytes else { + return + } + throw Self.overLimit("fs.read", url: url, bytes: bytes, limit: limits.maxReadBytes) + } + + private static func overLimit(_ operation: String, url: URL, bytes: Int, limit: Int) -> BridgeError { + .invalidArguments( + "\(operation) refused \(url.lastPathComponent): \(bytes) bytes exceeds the \(limit)-byte limit. Use a smaller payload, or have the host raise CodeModeConfiguration.fileSystemLimits." + ) + } + + /// Everything that must hold before a move or copy touches the disk. + /// + /// Containment alone admits a sandbox root: `documents:` resolves to the + /// documents root itself. On the destination side that let + /// `fs.move({ to: 'documents:' })` delete Documents; on the source side + /// `fs.move({ from: 'documents:', to: 'tmp:x' })` moved Documents into a + /// purgeable directory, and `fs.copy` from a root duplicated the whole tree. + /// Both sides are refused, matching `fs.delete`. + private func validateTransfer( + from fromURL: URL, + to toURL: URL, + operation: String, + arguments: [String: JSONValue], + context: BridgeInvocationContext + ) throws { + guard context.pathPolicy.isAllowedRoot(fromURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its source. Name a path inside the root, for example 'documents:archive'." + ) + } + guard context.pathPolicy.isAllowedRoot(toURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its destination. Name a path inside the root, for example 'documents:archive/file.txt'." + ) + } + + // Checked before anything is removed. Without it, a missing source with + // overwrite: true deleted the destination and then failed. + guard fileSystem.itemExists(at: fromURL) else { + throw BridgeError.invalidArguments("\(operation) source does not exist: \(fromURL.lastPathComponent)") + } + + // Same path, or a destination inside the source: the operation can never + // succeed, and with overwrite: true the old order removed the destination + // first — for `from == to`, that was the source itself. + let fromPath = fromURL.standardizedFileURL.path + let toPath = toURL.standardizedFileURL.path + guard toPath != fromPath, toPath.hasPrefix(fromPath + "/") == false else { + throw BridgeError.invalidArguments("\(operation) destination cannot be the source or a path inside it.") + } + + guard fileSystem.itemExists(at: toURL) else { + return + } + + guard arguments.bool("overwrite") == true else { + throw BridgeError.invalidArguments( + "\(operation) destination already exists. Pass overwrite: true to replace it, or choose a different 'to' path." + ) + } + + let isDirectory = (try? fileSystem.attributesOfItem(at: toURL))?.isDirectory ?? false + if isDirectory, arguments.bool("recursive") != true { + throw BridgeError.invalidArguments( + "\(operation) destination is a directory. Pass recursive: true along with overwrite: true to replace it and everything under it." + ) + } + } + + /// Runs `operation`, replacing an existing destination only if it succeeds. + /// + /// The destination is moved aside rather than deleted, the operation runs, + /// and only then is the old item removed; on failure it is moved back. The + /// previous remove-then-act order meant any failure after the remove — a + /// missing source, a full disk, a permissions error — lost the destination + /// along with the operation. + private func replacing( + _ toURL: URL, + operation: String, + arguments: [String: JSONValue], + _ body: () throws -> Void + ) throws { + guard fileSystem.itemExists(at: toURL) else { + try body() + return + } + + // Same directory, so the aside move is a rename, not a copy. + let aside = toURL + .deletingLastPathComponent() + .appendingPathComponent(".\(toURL.lastPathComponent).codemode-replacing-\(UUID().uuidString)") + try fileSystem.moveItem(at: toURL, to: aside) + + do { + try body() + } catch { + try? fileSystem.moveItem(at: aside, to: toURL) + throw error + } + try? fileSystem.removeItem(at: aside) + } + + /// Total bytes under `url`, stopping as soon as the running total exceeds + /// `limit` so a huge tree is not walked in full just to be refused. + private func totalSize(of url: URL, stoppingAbove limit: Int) throws -> Int { + let attributes = try fileSystem.attributesOfItem(at: url) + guard attributes.isDirectory else { + return attributes.size + } + + var total = 0 + var pending = [url] + while let directory = pending.popLast() { + for entry in try fileSystem.listDirectory(at: directory) { + if entry.isDirectory { + pending.append(URL(fileURLWithPath: entry.path)) + } else { + total += entry.size + if total > limit { + return total + } + } + } + } + return total + } + public func delete(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { guard let path = arguments.string("path") else { throw BridgeError.invalidArguments("fs.delete requires 'path'") @@ -137,6 +317,12 @@ public final class FileSystemBridge: @unchecked Sendable { return .object(["deleted": .bool(false), "path": .string(url.path)]) } + guard context.pathPolicy.isAllowedRoot(url) == false else { + throw BridgeError.pathViolation( + "fs.delete refuses a sandbox root. Delete entries inside it individually." + ) + } + let attrs = try fileSystem.attributesOfItem(at: url) if attrs.isDirectory, recursive == false { throw BridgeError.invalidArguments("fs.delete requires recursive=true for directories") diff --git a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift index 650190b..ca296cb 100644 --- a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift @@ -1,7 +1,16 @@ import Foundation -// Filesystem capabilities. Flat arguments, no constrained values, so no -// CodeModeStringEnum is needed here. +// Filesystem capabilities. + +/// The encodings `fs.read`/`fs.write` accept. Declared as an enum so the catalog +/// advertises `allowedStringValues` instead of leaving the constraint in prose, +/// where the model cannot discover it. +enum FileEncoding: String, CodeModeStringEnum { + case utf8 + case base64 + + static let codeModeAliases: [String: FileEncoding] = ["utf-8": .utf8] +} @BuiltInCodeMode(.fsList, path: "apple.fs.list", aliases: ["fs.promises.readdir"]) struct FSListTool: BuiltInCodeModeTool { @@ -35,8 +44,8 @@ struct FSReadTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("File path using allowed root prefix.") var path: String - @ToolParam("utf8 (default) or base64.") - var encoding: String? + @ToolParam("utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.") + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -61,7 +70,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @ToolParam("UTF-8 text or base64 string depending on encoding.") var data: String? @ToolParam("utf8 (default) or base64.") - var encoding: String? + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -75,7 +84,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsMove, path: "apple.fs.move", aliases: ["fs.promises.rename"]) struct FSMoveTool: BuiltInCodeModeTool { static let codeModeTitle = "Move file" - static let codeModeSummary = "Move file/directory within allowed sandbox roots." + static let codeModeSummary = "Move file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -83,8 +92,12 @@ struct FSMoveTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } @@ -98,7 +111,7 @@ struct FSMoveTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsCopy, path: "apple.fs.copy", aliases: ["fs.promises.copyFile"]) struct FSCopyTool: BuiltInCodeModeTool { static let codeModeTitle = "Copy file" - static let codeModeSummary = "Copy file/directory within allowed sandbox roots." + static let codeModeSummary = "Copy file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -106,8 +119,12 @@ struct FSCopyTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } diff --git a/Sources/CodeMode/Bridges/HealthBridge.swift b/Sources/CodeMode/Bridges/HealthBridge.swift index 0732e72..7f584ba 100644 --- a/Sources/CodeMode/Bridges/HealthBridge.swift +++ b/Sources/CodeMode/Bridges/HealthBridge.swift @@ -26,11 +26,21 @@ public final class HealthBridge: @unchecked Sendable { let writeTypes = try requestedWriteTypes(from: writeNames) try ensureAuthorization(toShare: writeTypes, read: readTypes, context: context, forceRequest: true) + // `requestAuthorization`'s `success` means the sheet flow completed, not + // that anything was granted — HealthKit deliberately refuses to disclose + // read authorization so an app cannot infer a health condition from a + // denial. Reporting `granted: true` here was a fabrication that left the + // transcript claiming access while reads returned []. Share types are the + // only ones with an answerable status, so those are the only ones + // reported. return .object([ - "status": .string(PermissionStatus.granted.rawValue), - "granted": .bool(true), + "status": .string("completed"), + "authorizationFlowCompleted": .bool(true), "readTypes": .array(readNames.map(JSONValue.string)), "writeTypes": .array(writeNames.map(JSONValue.string)), + "writeAuthorization": .object(writeAuthorizationStatuses(for: writeNames)), + "readAuthorizationDisclosed": .bool(false), + "note": .string("HealthKit does not disclose read authorization. An empty health.read result may mean the user denied that type rather than that no samples exist."), ]) #else _ = arguments @@ -150,15 +160,7 @@ public final class HealthBridge: @unchecked Sendable { } private func parseISODate(_ value: String?) -> Date? { - guard let value else { return nil } - - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: value) { - return date - } - - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: value) + CodeModeDate.parse(value) } #if canImport(HealthKit) @@ -320,6 +322,44 @@ public final class HealthBridge: @unchecked Sendable { return types } + /// Per-type share authorization, which is the only part HealthKit answers + /// honestly. Read authorization is deliberately undisclosed by the framework. + private func writeAuthorizationStatuses(for names: [String]) -> [String: JSONValue] { + var statuses: [String: JSONValue] = [:] + for name in names { + guard let spec = try? writeSpec(for: name), + let quantityType = HKObjectType.quantityType(forIdentifier: spec.identifier) + else { + continue + } + statuses[name] = .string(Self.shareStatusName(healthStore.authorizationStatus(for: quantityType))) + } + return statuses + } + + /// `.granted` only when every requested share type is actually authorized; + /// otherwise `.notDetermined`, because HealthKit will not say more. + private func resolvedShareAuthorization(for toShare: Set) -> PermissionStatus { + guard toShare.isEmpty == false else { + return .notDetermined + } + let allShared = toShare.allSatisfy { healthStore.authorizationStatus(for: $0) == .sharingAuthorized } + return allShared ? .granted : .notDetermined + } + + private static func shareStatusName(_ status: HKAuthorizationStatus) -> String { + switch status { + case .sharingAuthorized: + return PermissionStatus.granted.rawValue + case .sharingDenied: + return PermissionStatus.denied.rawValue + case .notDetermined: + return PermissionStatus.notDetermined.rawValue + @unknown default: + return PermissionStatus.unavailable.rawValue + } + } + private func ensureAuthorization( toShare: Set, read: Set, @@ -340,7 +380,12 @@ public final class HealthBridge: @unchecked Sendable { let requestStatus = try authorizationRequestStatus(toShare: toShare, read: read) switch requestStatus { case .unnecessary: - context.recordPermission(.healthKit, status: .granted) + // `.unnecessary` means "there is nothing left to ask the user", not + // "access was granted" — for read types the user may well have + // denied. Record what is actually known: granted only when every + // share type says so, otherwise leave it undetermined rather than + // writing a granted claim into the transcript. + context.recordPermission(.healthKit, status: resolvedShareAuthorization(for: toShare)) case .shouldRequest, .unknown: context.recordPermission(.healthKit, status: .notDetermined) let granted = try requestAuthorization(toShare: toShare, read: read) diff --git a/Sources/CodeMode/Bridges/KeychainBridge.swift b/Sources/CodeMode/Bridges/KeychainBridge.swift index 1668d48..3f03c72 100644 --- a/Sources/CodeMode/Bridges/KeychainBridge.swift +++ b/Sources/CodeMode/Bridges/KeychainBridge.swift @@ -8,6 +8,22 @@ public final class KeychainBridge: @unchecked Sendable { self.service = service } + /// A failure that says what went wrong, not just which OSStatus it was. + /// + /// "failed with status -34018" told neither the model nor the host + /// developer anything. -34018 in particular — `errSecMissingEntitlement` — + /// is a host configuration problem (no keychain access group; routine in an + /// unsigned test bundle), and the model should know a retry cannot fix it. + static func failure(_ operation: String, status: OSStatus) -> BridgeError { + let description = (SecCopyErrorMessageString(status, nil) as String?) ?? "OSStatus \(status)" + if status == errSecMissingEntitlement { + return .nativeFailure( + "\(operation) failed: the host app has no keychain entitlement (\(status)). This is a host configuration issue — the app needs a keychain access group — and retrying will not help." + ) + } + return .nativeFailure("\(operation) failed: \(description) (\(status))") + } + public func read(arguments: [String: JSONValue]) throws -> JSONValue { guard let key = arguments.string("key"), key.isEmpty == false else { throw BridgeError.invalidArguments("keychain.read requires 'key'") @@ -32,7 +48,7 @@ public final class KeychainBridge: @unchecked Sendable { case errSecItemNotFound: return .null default: - throw BridgeError.nativeFailure("keychain.read failed with status \(status)") + throw Self.failure("keychain.read", status: status) } } @@ -62,12 +78,12 @@ public final class KeychainBridge: @unchecked Sendable { if status == errSecDuplicateItem { let updateStatus = SecItemUpdate(baseQuery as CFDictionary, attrs as CFDictionary) guard updateStatus == errSecSuccess else { - throw BridgeError.nativeFailure("keychain.write update failed with status \(updateStatus)") + throw Self.failure("keychain.write", status: updateStatus) } return .object(["key": .string(key), "written": .bool(true)]) } - throw BridgeError.nativeFailure("keychain.write failed with status \(status)") + throw Self.failure("keychain.write", status: status) } public func delete(arguments: [String: JSONValue]) throws -> JSONValue { @@ -86,6 +102,6 @@ public final class KeychainBridge: @unchecked Sendable { return .object(["key": .string(key), "deleted": .bool(true)]) } - throw BridgeError.nativeFailure("keychain.delete failed with status \(status)") + throw Self.failure("keychain.delete", status: status) } } diff --git a/Sources/CodeMode/Bridges/NetworkBridge.swift b/Sources/CodeMode/Bridges/NetworkBridge.swift index ef33703..01234a9 100644 --- a/Sources/CodeMode/Bridges/NetworkBridge.swift +++ b/Sources/CodeMode/Bridges/NetworkBridge.swift @@ -6,7 +6,27 @@ public final class NetworkBridge: @unchecked Sendable { private let session: URLSession private let policy: NetworkAccessPolicy - public init(session: URLSession = .shared, policy: NetworkAccessPolicy = .standard) { + /// The session `network.fetch` uses unless the host supplies its own. + /// + /// Deliberately *not* `URLSession.shared`: that session reads + /// `HTTPCookieStorage.shared` and `URLCredentialStorage.shared`, so every + /// script request would ride whatever the host app is already logged in to, + /// and a `Set-Cookie` in a script-fetched response would poison the app's + /// cookie jar. The sandbox's premise is that script traffic carries no + /// ambient authority, so this session has no cookie jar, no credential + /// store, and no shared cache. + public static let isolatedSession: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.urlCache = nil + configuration.httpShouldSetCookies = false + configuration.httpCookieAcceptPolicy = .never + configuration.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: configuration) + }() + + public init(session: URLSession = NetworkBridge.isolatedSession, policy: NetworkAccessPolicy = .standard) { self.session = session self.policy = policy } @@ -43,12 +63,23 @@ public final class NetworkBridge: @unchecked Sendable { var request = URLRequest(url: url) request.httpMethod = options.string("method")?.uppercased() ?? "GET" request.timeoutInterval = TimeInterval(timeoutMs) / 1_000 + // Belt and braces alongside the isolated session: even a host-supplied + // session must not attach its cookie jar to script traffic. + request.httpShouldHandleCookies = false if let headers = options.object("headers") { - for (key, value) in headers { - if let string = value.stringValue { - request.setValue(string, forHTTPHeaderField: key) + for key in headers.keys.sorted() { + guard let string = headers[key]?.stringValue else { + continue } + if let reason = policy.headerViolationReason(for: key) { + context.auditLogger.log(AuditEvent( + capability: CapabilityID.networkFetch.rawValue, + message: "denied \(urlString): \(reason)" + )) + throw BridgeError.networkPolicyViolation(reason) + } + request.setValue(string, forHTTPHeaderField: key) } } diff --git a/Sources/CodeMode/Bridges/NotificationsBridge.swift b/Sources/CodeMode/Bridges/NotificationsBridge.swift index da6fc2a..178f605 100644 --- a/Sources/CodeMode/Bridges/NotificationsBridge.swift +++ b/Sources/CodeMode/Bridges/NotificationsBridge.swift @@ -228,7 +228,7 @@ public final class NotificationsBridge: @unchecked Sendable { private func isoDate(_ text: String?) -> Date? { guard let text else { return nil } - return ISO8601DateFormatter().date(from: text) + return CodeModeDate.parse(text) } #if canImport(UserNotifications) diff --git a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift index 194d853..0066ce5 100644 --- a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift +++ b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift @@ -39,7 +39,7 @@ private struct KeychainReadTool: BuiltInCodeModeTool { static let codeModeTitle = "Read Keychain value" static let codeModeSummary = "Read a string value from app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.get('auth_token')" + static let codeModeExample = "await apple.keychain.get({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), ] @@ -67,7 +67,7 @@ private struct KeychainWriteTool: BuiltInCodeModeTool { static let codeModeTitle = "Write Keychain value" static let codeModeSummary = "Store or update a string value in app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.set('auth_token', token)" + static let codeModeExample = "await apple.keychain.set({ key: 'auth_token', value: token })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), BuiltInToolArgument("value", .string, optional: true, hint: "Secret string value. Defaults to empty string when omitted."), @@ -102,7 +102,7 @@ private struct KeychainDeleteTool: BuiltInCodeModeTool { static let codeModeTitle = "Delete Keychain value" static let codeModeSummary = "Delete an app-scoped Keychain value." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.delete('auth_token')" + static let codeModeExample = "await apple.keychain.delete({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for value removal."), ] diff --git a/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift b/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift index e5c97d4..0258ef7 100644 --- a/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemServicesCodeModeTools.swift @@ -289,7 +289,7 @@ struct HealthPermissionRequestTool: BuiltInCodeModeTool { static let codeModeSummary = "Request HealthKit authorization for requested read/write types." static let codeModeTags = ["healthkit", "health", "permission"] static let codeModeExample = "await apple.health.requestPermission({ readTypes: ['stepCount', 'heartRate'], writeTypes: ['stepCount'] })" - static let codeModeResultSummary = "Object with status/granted fields and requested type arrays." + static let codeModeResultSummary = "Object with status ('completed'), per-type writeAuthorization, and the requested type arrays. Read authorization is never disclosed by HealthKit, so an empty health.read may mean denial." struct Arguments: Sendable { @ToolParam("Optional array of type names to read, e.g. stepCount, heartRate, activeEnergyBurned.") diff --git a/Sources/CodeMode/Bridges/SystemUIBridge.swift b/Sources/CodeMode/Bridges/SystemUIBridge.swift index a5b59bb..1d63159 100644 --- a/Sources/CodeMode/Bridges/SystemUIBridge.swift +++ b/Sources/CodeMode/Bridges/SystemUIBridge.swift @@ -1,7 +1,11 @@ import Foundation public final class SystemUIBridge: @unchecked Sendable { - public init() {} + private let networkAccessPolicy: NetworkAccessPolicy + + public init(networkAccessPolicy: NetworkAccessPolicy = .standard) { + self.networkAccessPolicy = networkAccessPolicy + } public func pickCalendar(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { try validateCalendarPickerArguments(arguments) @@ -188,12 +192,11 @@ public final class SystemUIBridge: @unchecked Sendable { } private func validateCalendarEventArguments(_ arguments: [String: JSONValue]) throws { - let formatter = ISO8601DateFormatter() for key in ["start", "end"] { guard let value = arguments.string(key) else { continue } - guard formatter.date(from: value) != nil else { + guard CodeModeDate.parse(value) != nil else { throw BridgeError.invalidArguments("calendar.ui.presentNewEvent requires \(key) to be an ISO8601 timestamp when provided") } } @@ -447,6 +450,13 @@ public final class SystemUIBridge: @unchecked Sendable { else { throw BridgeError.invalidArguments("\(capability) \(key) must be an absolute HTTP(S) URL") } + + // Destinations opened in a browser were exempt from the policy that + // governs fetch, so a script blocked from reaching an origin could still + // put it in front of the user — or start an OAuth flow against it. + if let reason = networkAccessPolicy.violationReason(for: url) { + throw BridgeError.networkPolicyViolation("\(capability) \(key): \(reason)") + } } private func validateNonemptyString(_ arguments: [String: JSONValue], key: String, capability: String) throws { diff --git a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift index b4d47d7..398af2d 100644 --- a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift @@ -327,7 +327,7 @@ struct WebUIPresentTool: BuiltInCodeModeTool { @BuiltInCodeMode(.authUIWebAuthenticate, path: "apple.auth.webAuthenticate") struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { static let codeModeTitle = "Authenticate with system web UI" - static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication." + static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy." static let codeModeTags = ["auth", "oauth", "web", "browser", "system-ui"] static let codeModeExample = "await apple.auth.webAuthenticate({ url: 'https://example.com/oauth', callbackURLScheme: 'myapp' })" static let codeModeResultSummary = "Object with action callback/cancelled and callbackURL when available." @@ -337,7 +337,7 @@ struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { var url: String @ToolParam("Optional custom URL scheme that completes the session.") var callbackURLScheme: String? - @ToolParam("Whether to prefer a private browser session; default false.") + @ToolParam("Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.") var prefersEphemeralSession: Bool? @ToolParam("Optional timeout for waiting on callback/cancellation.") var timeoutMs: Double? diff --git a/Sources/CodeMode/Catalog/BridgeCatalog.swift b/Sources/CodeMode/Catalog/BridgeCatalog.swift index 9e55914..8ec69f1 100644 --- a/Sources/CodeMode/Catalog/BridgeCatalog.swift +++ b/Sources/CodeMode/Catalog/BridgeCatalog.swift @@ -1,25 +1,54 @@ import Foundation -struct BridgeCatalog: Sendable { +/// The model-facing view of the registry. +/// +/// Tracks the registry rather than snapshotting it once at init. The snapshot +/// used to be frozen in `CodeModeAgentTools.init`, which made +/// `CapabilityRegistry`'s public post-init `register(...)` methods either +/// unreachable in the supported flow or — if reached — a way to desync what +/// search advertises from what execution can actually invoke. +final class BridgeCatalog: @unchecked Sendable { private struct SearchCatalogPayload: Sendable, Codable { var references: [JavaScriptAPIReference] var byCapability: [String: JavaScriptAPIReference] var byJSName: [String: JavaScriptAPIReference] } - private let references: [JavaScriptAPIReference] - private let referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] - private let searchCatalog: JSONValue - private let allJavaScriptNames: [String] + private struct Snapshot { + var generation: Int + var references: [JavaScriptAPIReference] + var referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] + var searchCatalog: JSONValue + var allJavaScriptNames: [String] + } + + private let registry: CapabilityRegistry + private let lock = NSLock() + private var snapshot: Snapshot init(registry: CapabilityRegistry) { + self.registry = registry + self.snapshot = Self.makeSnapshot(registry: registry) + } + + /// Rebuilds if the registry has changed since the last read. Rebuilding is + /// the whole catalog, but it only happens when a provider is actually + /// registered — not on every search. + private func current() -> Snapshot { + lock.lock() + defer { lock.unlock() } + let generation = registry.generation + if snapshot.generation != generation { + snapshot = Self.makeSnapshot(registry: registry) + } + return snapshot + } + + private static func makeSnapshot(registry: CapabilityRegistry) -> Snapshot { + let generation = registry.generation let functions = registry.allRegisteredFunctions().sorted { $0.catalogCapability < $1.catalogCapability } let references = functions.map(Self.reference(from:)) - self.references = references - self.referencesByCapability = Dictionary(uniqueKeysWithValues: references.map { ($0.capabilityKey, $0) }) - self.allJavaScriptNames = Array(Set(references.flatMap(\.jsNames))).sorted() - var byJSName: [String: JavaScriptAPIReference] = [:] for reference in references { for jsName in reference.jsNames { @@ -27,29 +56,41 @@ struct BridgeCatalog: Sendable { } } - self.searchCatalog = Self.jsonValue( - from: SearchCatalogPayload( - references: references, - byCapability: Dictionary(uniqueKeysWithValues: references.map { ($0.capability, $0) }), - byJSName: byJSName - ) + return Snapshot( + generation: generation, + references: references, + referencesByCapability: Dictionary(references.map { ($0.capabilityKey, $0) }, uniquingKeysWith: { first, _ in first }), + searchCatalog: Self.jsonValue( + from: SearchCatalogPayload( + references: references, + byCapability: Dictionary(references.map { ($0.capability, $0) }, uniquingKeysWith: { first, _ in first }), + byJSName: byJSName + ) + ), + allJavaScriptNames: Array(Set(references.flatMap(\.jsNames))).sorted() ) } func reference(for capability: CapabilityID) -> JavaScriptAPIReference? { - referencesByCapability[capability.codeModeKey] + current().referencesByCapability[capability.codeModeKey] } func reference(for capabilityKey: CodeModeCapabilityKey) -> JavaScriptAPIReference? { - referencesByCapability[capabilityKey] + current().referencesByCapability[capabilityKey] } func allReferences() -> [JavaScriptAPIReference] { - references + current().references } func searchCatalogValue() -> JSONValue { - searchCatalog + current().searchCatalog + } + + /// TypeScript declarations for the platform-filtered references that pass + /// `isIncluded`, for a host to drop into its system prompt. + func typeDeclarations(where isIncluded: (JavaScriptAPIReference) -> Bool) -> String { + TypeScriptDeclarations.surface(for: current().references.filter(isIncluded)) } func closestFunctionNames(to candidate: String, limit: Int = 3) -> [String] { @@ -62,7 +103,7 @@ struct BridgeCatalog: Sendable { let distanceThreshold = max(2, min(8, normalizedCandidate.count / 3 + 1)) - let ranked = allJavaScriptNames.map { name in + let ranked = current().allJavaScriptNames.map { name in let normalizedName = name.lowercased() let distance = Self.levenshtein(normalizedCandidate, normalizedName) let prefixBonus = normalizedName.hasPrefix(normalizedCandidate) || normalizedCandidate.hasPrefix(normalizedName) ? -2 : 0 @@ -99,7 +140,7 @@ struct BridgeCatalog: Sendable { } private static func reference(from function: RegisteredCodeModeFunction) -> JavaScriptAPIReference { - return JavaScriptAPIReference( + let reference = JavaScriptAPIReference( capability: function.catalogCapability, capabilityKey: function.capabilityKey, builtInCapability: function.builtInCapability, @@ -114,6 +155,9 @@ struct BridgeCatalog: Sendable { argumentConstraints: function.argumentConstraints, resultSummary: function.resultSummary ) + var withTypes = reference + withTypes.dts = TypeScriptDeclarations.declaration(for: reference) + return withTypes } private static func levenshtein(_ lhs: String, _ rhs: String) -> Int { diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift new file mode 100644 index 0000000..29cfe77 --- /dev/null +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -0,0 +1,368 @@ +import Foundation + +/// Generates TypeScript declarations for the CodeMode JavaScript surface. +/// +/// The registry already holds names, required/optional arguments, types, enum +/// constraints, and hints; before this, the model saw that as a flat +/// `argumentTypes` map of six coarse cases and a prose `resultSummary`. Models +/// write markedly better code against real declarations, so the same metadata is +/// emitted as `.d.ts` — per capability for search results, and whole-surface for +/// a system prompt. +/// +/// Result types are `CodeModeValue` for now: built-in tools return untyped +/// `JSONValue` dictionaries, so a typed result would be a fiction. The prose +/// `resultSummary` is carried in the doc comment instead, and a per-capability +/// result schema can tighten this later without changing the shape here. +public enum TypeScriptDeclarations { + /// Shared types and the runtime globals that are not catalog capabilities. + /// + /// Hand-authored deliberately: these are the Node-compatibility shims in + /// `RuntimeJavaScript.bootstrap`, and unlike the namespaced helpers they take + /// *positional* arguments, which the catalog has no way to express. Keeping + /// them here means the declarations tell the truth about both conventions. + public static let preamble = """ + /** Any JSON-representable value. Bridge results are JSON, not class instances. */ + type CodeModeValue = + | null + | boolean + | number + | string + | CodeModeValue[] + | { [key: string]: CodeModeValue }; + + declare const console: { + log(...values: unknown[]): void; + info(...values: unknown[]): void; + warn(...values: unknown[]): void; + error(...values: unknown[]): void; + }; + + interface CodeModeResponse { + ok: boolean; + status: number; + statusText: string; + headers: { get(name: string): string | null }; + text(): Promise; + json(): Promise; + } + + /** + * HTTP(S) fetch. Runs on an isolated session: no app cookies, no stored + * credentials. Cookie/Authorization/Proxy-* request headers are refused + * unless the host app permits them. + */ + declare function fetch( + url: string, + options?: { + method?: string; + headers?: { [name: string]: string }; + body?: string; + bodyBase64?: string; + timeoutMs?: number; + responseEncoding?: "text" | "base64"; + } + ): Promise; + + /** + * Node-style filesystem aliases. Unlike the apple.* helpers these take + * POSITIONAL arguments and return Node-like values. + */ + declare const fs: { + promises: { + readFile(path: string, encoding?: "utf8" | "base64"): Promise; + writeFile(path: string, data: string, encoding?: "utf8" | "base64"): Promise; + readdir(path: string): Promise; + stat(path: string): Promise; + access(path: string): Promise; + mkdir(path: string, options?: { recursive?: boolean }): Promise; + rm(path: string, options?: { recursive?: boolean }): Promise; + rename(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + copyFile(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + }; + }; + + declare const path: { + join(...parts: string[]): string; + }; + """ + + /// The declaration for one capability, suitable for a search result payload. + /// + /// Declared at its real call path — `declare const apple: { fs: { read(…) } }` + /// — so what the model reads is exactly what it can call. + /// + /// A fragment type-checks alongside `preamble`, which defines `CodeModeValue`; + /// on its own that name is unresolved. Fragments also do not merge with each + /// other — two for the same root each declare it — which is harmless when a + /// model reads joined search results, but for one compilation unit covering + /// several capabilities use `surface(for:)` (or + /// `CodeModeAgentTools.typeDeclarations(for:)`), which merges them under a + /// single declaration per root. Verified with `tsc --strict`: every fragment + /// plus the preamble, and every platform's whole surface, compile cleanly. + public static func declaration(for reference: JavaScriptAPIReference) -> String { + guard let canonical = canonicalName(for: reference) else { + return "" + } + // A bare global such as `fetch` has no namespace to sit in. The whole + // surface covers it from the hand-authored preamble; a per-reference + // declaration has to say it directly. + guard canonical.contains(".") else { + return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(canonical)\(functionSignature(for: reference));" + } + return renderNamespaces(for: [reference]) + } + + /// The preamble plus the given capabilities, grouped by canonical dotted + /// JavaScript name under one declaration per root — valid as a single + /// compilation unit. + public static func surface(for references: [JavaScriptAPIReference]) -> String { + """ + // CodeMode JavaScript API — generated from the capability registry. + // Only helpers supported on the current host platform appear here. + + \(preamble) + + \(renderNamespaces(for: references)) + """ + } + + private static func renderNamespaces(for references: [JavaScriptAPIReference]) -> String { + let root = Namespace(name: "") + for reference in references.sorted(by: { $0.capability < $1.capability }) { + guard let canonical = canonicalName(for: reference) else { + continue + } + var components = canonical.split(separator: ".").map(String.init) + guard components.count >= 2 else { + // A bare global such as `fetch` is covered by the preamble. + continue + } + let functionName = components.removeLast() + root.insert( + Member( + name: functionName, + documentation: documentation(for: reference, canonicalName: canonical), + signature: functionSignature(for: reference) + ), + at: components + ) + } + + return root.children + .sorted { $0.key < $1.key } + .map { render(namespace: $0.value, indent: "", isTopLevel: true) } + .joined(separator: "\n\n") + } + + // MARK: - Declaration pieces + + /// The namespaced helper name, preferred over Node-style aliases: aliases use + /// positional arguments the catalog cannot describe, and are documented in + /// the preamble instead. + private static func canonicalName(for reference: JavaScriptAPIReference) -> String? { + reference.jsNames.first { $0.contains(".") && $0.hasPrefix("fs.promises.") == false } + ?? reference.jsNames.first + } + + private static func documentation(for reference: JavaScriptAPIReference, canonicalName: String) -> String { + var lines = ["/**"] + lines.append(" * \(reference.summary)") + lines.append(" *") + lines.append(" * Capability: \(reference.capability)") + lines.append(" * Returns: \(reference.resultSummary)") + + let aliases = reference.jsNames.filter { $0 != canonicalName } + if aliases.isEmpty == false { + // Only the fs.promises.* shims are positional; other aliases are + // additional namespaced spellings of the same object-argument call. + let positional = aliases.filter { $0.hasPrefix("fs.promises.") || $0 == "fetch" } + let note = positional.isEmpty ? "" : " — \(positional.joined(separator: ", ")) take positional arguments" + lines.append(" * Aliases: \(aliases.joined(separator: ", "))\(note)") + } + if reference.example.isEmpty == false { + lines.append(" *") + lines.append(" * @example \(reference.example)") + } + lines.append(" */") + return lines.joined(separator: "\n") + } + + private static func functionSignature(for reference: JavaScriptAPIReference) -> String { + let tree = ArgumentTree(reference: reference) + guard tree.isEmpty == false else { + return "(): Promise" + } + // `args` is optional only when nothing in it is required. + let optionalMarker = reference.requiredArguments.isEmpty ? "?" : "" + return "(args\(optionalMarker): \(tree.render(indent: ""))): Promise" + } + + // MARK: - Argument tree + // + // Arguments are declared as dotted paths (`options.timeoutMs`), so the object + // literal has to be rebuilt from them. + + private struct ArgumentTree { + private var children: [String: ArgumentTree] = [:] + private var leaves: [Leaf] = [] + + private struct Leaf { + var name: String + var type: String + var optional: Bool + var hint: String? + } + + init(reference: JavaScriptAPIReference) { + let allPaths = Array(Set( + reference.requiredArguments + reference.optionalArguments + Array(reference.argumentTypes.keys) + )).sorted() + let required = Set(reference.requiredArguments) + + for path in allPaths { + let components = path.split(separator: ".").map(String.init) + insert( + components: components, + leaf: Leaf( + name: components.last ?? path, + type: TypeScriptDeclarations.tsType( + for: reference.argumentTypes[path] ?? .any, + allowedValues: reference.argumentConstraints.allowedStringValues[path] + ), + // A nested path is optional unless the exact path is + // listed as required. + optional: required.contains(path) == false, + hint: reference.argumentHints[path] + ) + ) + } + } + + private init() {} + + var isEmpty: Bool { + children.isEmpty && leaves.isEmpty + } + + private mutating func insert(components: [String], leaf: Leaf) { + guard components.count > 1 else { + // A declared parent (`options: object`) whose children also appear + // is represented by the nested literal, not by a duplicate leaf. + if children[leaf.name] == nil, leaves.contains(where: { $0.name == leaf.name }) == false { + leaves.append(leaf) + } + return + } + let head = components[0] + leaves.removeAll { $0.name == head } + var child = children[head] ?? ArgumentTree() + child.insert(components: Array(components.dropFirst()), leaf: leaf) + children[head] = child + } + + func render(indent: String) -> String { + let inner = indent + " " + var lines: [String] = ["{"] + + for leaf in leaves.sorted(by: { $0.name < $1.name }) { + if let hint = leaf.hint, hint.isEmpty == false { + lines.append("\(inner)/** \(hint) */") + } + lines.append("\(inner)\(leaf.name)\(leaf.optional ? "?" : ""): \(leaf.type);") + } + + for (name, child) in children.sorted(by: { $0.key < $1.key }) { + lines.append("\(inner)\(name)?: \(child.render(indent: inner));") + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + } + + private static func tsType(for type: CapabilityArgumentType, allowedValues: [String]?) -> String { + if let allowedValues, allowedValues.isEmpty == false { + // The constraint is the type: a union beats `string` plus prose the + // model has to notice. + return allowedValues.map { "\"\($0)\"" }.joined(separator: " | ") + } + + switch type { + case .string: + return "string" + case .number: + return "number" + case .bool: + return "boolean" + case .object: + return "{ [key: string]: CodeModeValue }" + case .array: + return "CodeModeValue[]" + case .any: + return "CodeModeValue" + } + } + + // MARK: - Namespace rendering + + private struct Member { + var name: String + var documentation: String + var signature: String + } + + private final class Namespace { + let name: String + var children: [String: Namespace] = [:] + var members: [Member] = [] + + init(name: String) { + self.name = name + } + + func insert(_ member: Member, at path: [String]) { + guard let head = path.first else { + members.append(member) + return + } + let child = children[head] ?? Namespace(name: head) + children[head] = child + child.insert(member, at: Array(path.dropFirst())) + } + } + + /// Renders as an object type — `declare const apple: { keychain: { + /// delete(args): …; }; };` — not as `declare namespace`. + /// + /// Namespace members are declarations and need identifiers, so helpers named + /// with reserved words (`apple.keychain.delete`, `apple.photos.export`) + /// rendered as `function delete(…)`: not valid TypeScript, which broke any + /// host that type-checks the output. Object type members may be any property + /// name. + private static func render(namespace: Namespace, indent: String, isTopLevel: Bool) -> String { + let inner = indent + " " + var lines = [isTopLevel ? "\(indent)declare const \(namespace.name): {" : "\(indent)\(namespace.name): {"] + + for member in namespace.members.sorted(by: { $0.name < $1.name }) { + lines.append(indented(member.documentation, by: inner)) + lines.append("\(inner)\(member.name)\(indented(member.signature, by: inner, skipFirstLine: true));") + } + + for (_, child) in namespace.children.sorted(by: { $0.key < $1.key }) { + lines.append(render(namespace: child, indent: inner, isTopLevel: false)) + } + + lines.append("\(indent)};") + return lines.joined(separator: "\n") + } + + private static func indented(_ text: String, by indent: String, skipFirstLine: Bool = false) -> String { + text + .split(separator: "\n", omittingEmptySubsequences: false) + .enumerated() + .map { index, line in + index == 0 && skipFirstLine ? String(line) : indent + line + } + .joined(separator: "\n") + } +} diff --git a/Sources/CodeMode/Host/CodeModeAgentGuidance.swift b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift new file mode 100644 index 0000000..f80b5ec --- /dev/null +++ b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift @@ -0,0 +1,188 @@ +import Foundation + +/// System-prompt guidance that teaches an agent *why* it is writing a program +/// rather than calling tools one at a time. +/// +/// The tool descriptions and the generated TypeScript answer "what is this helper +/// called and what does it take". Neither answers "what shape should my work +/// take", and an agent that never learns the second question will use +/// `executeJavaScript` as a thin RPC — one call per operation — which is the +/// pattern this package exists to replace. +/// +/// Pair it with `CodeModeAgentTools.typeDeclarations()`, which supplies the +/// vocabulary this supplies the strategy for. Those declarations follow the +/// host's `CapabilityGrant`, so a host with a ceiling pays only for what it +/// grants — the unrestricted iOS surface is roughly 21,000 tokens: +/// +/// ```swift +/// let prompt = CodeModeAgentGuidance.systemPrompt(.standard) +/// + "\n\n" +/// + tools.typeDeclarations() +/// ``` +public enum CodeModeAgentGuidance { + /// How much prompt budget to spend on the guidance. + /// + /// Every tier is self-contained and states the core idea; the larger ones add + /// worked examples and failure modes rather than qualifying what came before, + /// so a host can drop down a tier without losing a rule. + public enum Length: String, Sendable, CaseIterable { + // Declared smallest to largest; `length(forApproximateTokenBudget:)` + // relies on that order. + /// One paragraph: the core idea and nothing else. + case brief + /// The idea, the workflow, and one worked example. The default. + case standard + /// Adds fan-out and error-handling examples plus the anti-pattern list. + /// Worth it when the agent will do open-ended multi-step work. + case full + + /// Rough size of this tier's text, for budgeting against a prompt. + /// + /// Estimated at four characters per token — the usual English-prose rule + /// of thumb, not a real tokenization. Treat it as a bound to plan with, + /// not a number to assert on. + public var approximateTokenCount: Int { + CodeModeAgentGuidance.systemPrompt(self).count / 4 + } + } + + /// The guidance text at the requested length. + public static func systemPrompt(_ length: Length = .standard) -> String { + switch length { + case .brief: + return core + case .standard: + return [core, workflow, workedExample].joined(separator: "\n\n") + case .full: + return [core, workflow, workedExample, fanOutExample, antiPatterns].joined(separator: "\n\n") + } + } + + /// The largest tier that fits the budget, or `.brief` when nothing does. + /// + /// `.brief` is returned even when it exceeds the budget: some guidance is the + /// difference between an agent that batches and one that does not, so the + /// floor is deliberate rather than an empty string. + public static func systemPrompt(approximateTokenBudget: Int) -> String { + systemPrompt(length(forApproximateTokenBudget: approximateTokenBudget)) + } + + /// The tier `systemPrompt(approximateTokenBudget:)` would choose. + public static func length(forApproximateTokenBudget budget: Int) -> Length { + Length.allCases + .reversed() + .first { $0.approximateTokenCount <= budget } + ?? .brief + } + + // MARK: - Sections + + private static let core = """ + ## Writing code instead of calling tools + + You have a JavaScript runtime with direct access to this device's APIs. Use it \ + to run whole tasks, not single operations. One `executeJavaScript` call should \ + do the entire job — read, loop, branch, filter, transform, write — and return \ + only the finished answer. Ten helper calls inside one script is normal and \ + fast; ten `executeJavaScript` calls that each make one helper call is the \ + mistake this runtime exists to avoid, because every round trip costs a model \ + turn and pushes intermediate data you do not need through your context. \ + Request only the capabilities your script actually calls. + """ + + private static let workflow = """ + ## How to work + + 1. Call `searchJavaScriptAPI` first when you are unsure of helper names, \ + arguments, or result shapes. Return `ref.dts` — the TypeScript declaration — \ + rather than assembling the metadata fields yourself. + 2. Write one script that completes the task. Reach for ordinary control flow: \ + loops over collections, `if` for branching, `try`/`catch` around anything that \ + may fail per item. Before writing a loop of single calls, check the declaration \ + for an argument that takes a collection — `apple.contacts.list({ identifiers })` \ + is one call, not one per contact. + 3. Do the filtering and aggregation *in the script*. Return the graded answer — \ + a count, a summary, the three matching records — not the raw collection you \ + read. A large return value is truncated, and the data you did not need cost \ + you context on the way through. + 4. Repair from the structured error. It names the capability, the line, and \ + what to change; the suggestions tell you when a retry cannot help. + """ + + private static let workedExample = """ + ## The shape to aim for + + Task: "how much did I spend on the trip, and file the receipts?" + + ```javascript + // ONE call. Reads, filters, sums, writes, and returns just the total. + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + ``` + + `allowedCapabilities: ["fs.list", "fs.read", "fs.move"]` — exactly the three \ + helpers the script calls, and nothing else. + """ + + private static let fanOutExample = """ + ## Handling many items and partial failure + + Keep going when one item fails, and report what happened rather than throwing \ + the whole run away: + + ```javascript + const results = []; + for (const city of ['lisbon', 'porto', 'faro']) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + if (!response.ok) { results.push({ city, error: response.status }); continue; } + const data = await response.json(); + results.push({ city, tempC: data.current.temp_c }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + ``` + + Native calls run one at a time, so `Promise.all` does not make these overlap — \ + a loop is just as fast here and reads better. Use \ + `await new Promise(r => setTimeout(r, 1000))` to back off between retries; \ + delays are real. + """ + + private static let antiPatterns = """ + ## Anti-patterns + + - **One call per operation.** If your second `executeJavaScript` starts from \ + data the first one returned, both should have been one script. + - **Returning the raw collection.** Reading 500 records to answer "how many \ + are overdue" should return a number, not 500 records. + - **Asking for capabilities you do not call.** Request the exact set your \ + script uses. A wider list grants nothing useful, and it is the wrong thing to \ + ask a user to approve. + - **Forgetting `await`.** An un-awaited helper's failure is silently \ + discarded and you will be told the run succeeded. A \ + `BRIDGE_FAILURES_NOT_SURFACED` diagnostic means exactly this happened. + - **Re-running a call the error told you not to retry.** \ + `NETWORK_POLICY_VIOLATION`, `UI_PRESENTER_UNAVAILABLE`, and a host-withheld \ + `CAPABILITY_DENIED` will not succeed on a second attempt. + """ +} diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 62cace0..84e53b3 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -42,7 +42,7 @@ public enum CodeModeAgentToolDescriptions { "items": .object([ "type": .string("string"), ]), - "description": .string("Custom provider capability keys required by the JavaScript. Built-in capability IDs may also be accepted here by hosts that expose one allowlist field."), + "description": .string("Custom provider capability keys required by the JavaScript, for example myapp.tasks.complete. Built-in capability IDs do not belong here and are ignored — list those in allowedCapabilities."), ]), "timeoutMs": .object([ "type": .string("integer"), @@ -75,6 +75,7 @@ public enum CodeModeAgentToolDescriptions { argumentHints: Record; argumentConstraints: { allowedStringValues: Record }; resultSummary: string; + dts: string; } declare const api: { @@ -86,6 +87,8 @@ public enum CodeModeAgentToolDescriptions { Your code must evaluate to an async function and return JSON-serializable output. Search has a 2s budget, so slice broad result sets before returning them. + Prefer returning ref.dts: it is the TypeScript declaration for the helper, with argument names, types, string-literal unions for constrained values, optionality, and the result summary. It is more precise and usually shorter than assembling argumentTypes, argumentHints, and argumentConstraints yourself. + Examples: async () => { return api.references @@ -106,7 +109,14 @@ public enum CodeModeAgentToolDescriptions { } async () => { - return api.byJSName["apple.fs.read"]; + return api.byJSName["apple.fs.read"].dts; + } + + async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); } """ ) @@ -114,21 +124,29 @@ public enum CodeModeAgentToolDescriptions { public static let executeJavaScript = CodeModeAgentToolDescription( name: "executeJavaScript", description: """ - Execute JavaScript against the CodeMode runtime. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + Execute JavaScript against the CodeMode runtime. Use one call to run a whole task — read, loop, branch, filter, transform, write — and return only the finished answer. Many helper calls inside one script is the intended shape; many executeJavaScript calls that each make one helper call is not, because every round trip costs a turn and pushes intermediate data through your context. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + + Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. - Calling conventions: apple.* helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. + Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. - Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. + Concurrency: setTimeout and clearTimeout work normally — callbacks are deferred and delays are honoured, so `await new Promise(r => setTimeout(r, 1000))` really waits. There is no I/O event loop, though: native calls run one at a time, so Promise.all over several helpers completes them sequentially rather than concurrently. A promise with no resolve path and no pending timer can never settle and fails fast with JS_RUNTIME_ERROR rather than running out the clock. - Allowlisting: include only the required built-in capabilities in allowedCapabilities and custom provider keys in allowedCapabilityKeys. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. + Allowlisting: request the exact set of capabilities your script calls and nothing more — a capability you list but never call widens the grant for nothing. Built-in capability IDs go in allowedCapabilities; custom provider keys go in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. Error repair guide: JS_API_NOT_FOUND: use the suggested JS helper names or searchJavaScriptAPI. - CAPABILITY_DENIED: add the exact capability to allowedCapabilities or allowedCapabilityKeys and retry. + CAPABILITY_DENIED: add the exact capability to allowedCapabilities (built-ins) or allowedCapabilityKeys (custom provider keys) and retry — unless the error suggestions say the host withheld it, in which case retrying will not help. PERMISSION_DENIED: the capability is allowlisted but the OS, host, or custom provider denied permission; request permission if a helper exists, otherwise tell the user or host. UI_PRESENTER_UNAVAILABLE: host configuration issue; do not retry the same call. INVALID_ARGUMENTS: use the catalog requiredArguments, optionalArguments, argumentHints, and example. NETWORK_POLICY_VIOLATION: the host's network access policy refused the destination; do not retry the same URL and do not add capabilities to work around it. + JS_RUNTIME_ERROR "can never settle": a promise in your script has no resolve path; await every promise and make sure each one resolves. + + Warning diagnostics worth acting on: + BRIDGE_FAILURES_NOT_SURFACED: a helper failed but your script still returned successfully — usually a missing await, which discards the rejection. + RESULT_TRUNCATED: the return value exceeded the size limit; return a summary or write the payload to the sandbox and return the path. + TIMER_CALLBACK_ERROR: a setTimeout callback threw; its error cannot reach the caller, so handle it inside the callback. """ ) diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index 1959dfb..4c6356f 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -4,10 +4,18 @@ public final class CodeModeAgentTools: @unchecked Sendable { private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let runtime: BridgeRuntime + private let capabilityGrant: CapabilityGrant - public init(config: CodeModeConfiguration = .init()) { + public convenience init(config: CodeModeConfiguration = .init()) { + self.init(config: config, clock: RealClock()) + } + + /// Test seam: a virtual clock makes timer-driven executions instant and + /// exact. Not public — production time is not a host decision. + init(config: CodeModeConfiguration, clock: any RuntimeClock) { let allDefaultRegistrations = DefaultCapabilityLoader.loadAllRegistrations( fileSystem: config.fileSystem, + fileSystemLimits: config.fileSystemLimits, networkAccessPolicy: config.networkAccessPolicy, eventInbox: config.eventInbox, cloudKitClient: config.cloudKitClient, @@ -32,12 +40,14 @@ public final class CodeModeAgentTools: @unchecked Sendable { let providerRegistrations = config.codeModeProviders.flatMap { $0.codeModeRegistrations() } let registry = CapabilityRegistry(registrations: registrations, codeModeRegistrations: providerRegistrations) self.registry = registry + self.capabilityGrant = config.capabilityGrant self.catalog = BridgeCatalog(registry: registry) self.runtime = BridgeRuntime( registry: registry, catalog: self.catalog, config: config, - unsupportedBuiltInJavaScriptNames: unsupportedBuiltInJavaScriptNames + unsupportedBuiltInJavaScriptNames: unsupportedBuiltInJavaScriptNames, + clock: clock ) } @@ -45,7 +55,65 @@ public final class CodeModeAgentTools: @unchecked Sendable { try await runtime.searchAsync(request) } + /// Declared `async throws` for a body that is currently synchronous and + /// non-throwing. That is deliberate: making the bridge ABI asynchronous is + /// planned, and this signature is the one that will not have to change. public func executeJavaScript(_ request: JavaScriptExecutionRequest) async throws -> JavaScriptExecutionCall { runtime.makeExecutionCall(request) } + + /// TypeScript declarations for what this host actually permits: the + /// platform-filtered surface, narrowed to the configured `CapabilityGrant`. + /// + /// Models write markedly better code against real types than against prose, + /// but the unrestricted iOS surface is roughly 21,000 tokens. A host with a + /// grant pays only for what it grants — and never advertises a helper the + /// model would be denied. Searches return a per-capability `dts` for the + /// cases in between. + public func typeDeclarations() -> String { + catalog.typeDeclarations { reference in + if let builtIn = reference.builtInCapability { + return capabilityGrant.permits(builtIn) + } + return capabilityGrant.permits(key: reference.capabilityKey) + } + } + + /// Declarations for an explicit subset — for example, the capabilities a + /// particular agent task needs, when even the grant is too broad to inline. + /// Identifiers not available on this host are skipped. + public func typeDeclarations( + for capabilities: Set, + capabilityKeys: Set = [] + ) -> String { + catalog.typeDeclarations { reference in + if let builtIn = reference.builtInCapability { + return capabilities.contains(builtIn) + } + return capabilityKeys.contains(reference.capabilityKey) + } + } + + /// Every capability available on this host, for consent UI and for hosts + /// building their own `CapabilityGrant`. + public func capabilities() -> [JavaScriptAPIReference] { + catalog.allReferences() + } + + /// Adds host providers after construction. + /// + /// Providers no longer have to be known at init: the catalog tracks the + /// registry, so search starts advertising these immediately and the JS + /// bindings are installed for the next execution. A host whose domain APIs + /// depend on runtime state — a signed-in account, a loaded document — can + /// register them when that state arrives. + /// + /// Registering a key that already exists replaces it. + public func register(providers: [any CodeModeProvider]) { + registry.register(providers.flatMap { $0.codeModeRegistrations() }) + } + + public func register(provider: any CodeModeProvider) { + register(providers: [provider]) + } } diff --git a/Sources/CodeMode/Host/HostConfigurationValidator.swift b/Sources/CodeMode/Host/HostConfigurationValidator.swift index d9eb7f6..5796cb0 100644 --- a/Sources/CodeMode/Host/HostConfigurationValidator.swift +++ b/Sources/CodeMode/Host/HostConfigurationValidator.swift @@ -119,6 +119,26 @@ public enum HostConfigurationValidator { validate(requiredCapabilities: requiredCapabilities, infoPlist: bundle.infoDictionary ?? [:]) } + /// Checks the runtime configuration itself, independent of Info.plist. + /// + /// The one thing worth flagging today is an unrestricted `capabilityGrant`. + /// It is the default for source compatibility, but it means the *model* + /// decides what a script may reach — the allowlist in the tool call is + /// model-authored. A host that pipes tool JSON straight into + /// `executeJavaScript` with this default has no capability boundary at all. + public static func validate(configuration: CodeModeConfiguration) -> [HostConfigurationIssue] { + guard configuration.capabilityGrant == .unrestricted else { + return [] + } + return [ + HostConfigurationIssue( + severity: .warning, + key: "capabilityGrant", + message: "CodeModeConfiguration.capabilityGrant is .unrestricted: the model-authored allowedCapabilities is the only allowlist in force. Set CapabilityGrant.only(...) to the capabilities this host actually permits." + ) + ] + } + public static func validate(requiredCapabilities: Set, infoPlist: [String: Any]) -> [HostConfigurationIssue] { var issues: [HostConfigurationIssue] = [] let keys = requiredInfoPlistKeys(for: requiredCapabilities) diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index b50f76d..e5bb049 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -16,7 +16,10 @@ public enum CapabilityArgumentType: String, Sendable, Codable, Equatable { if case .string = value { return true } return false case .number: - if case .number = value { return true } + // Non-finite is not a usable number for any bridge: it cannot be + // converted to an Int, cannot round-trip through JSON, and reaches + // here from `"inf"`/`"nan"` coercion or a JS `Infinity`. + if case let .number(number) = value { return number.isFinite } return false case .bool: if case .bool = value { return true } @@ -432,6 +435,11 @@ public final class CapabilityRegistry: @unchecked Sendable { private let lock = NSLock() private var registrations: [CapabilityID: CapabilityRegistration] = [:] private var codeModeRegistrations: [CodeModeCapabilityKey: CodeModeRegistration] = [:] + /// Bumped on every mutation so `BridgeCatalog` can tell whether its snapshot + /// is still current. Without it the catalog was frozen at init, which made + /// the public `register(...)` methods below either unreachable or a way to + /// desync what search advertises from what execution can actually invoke. + private var generationValue = 0 public init(registrations: [CapabilityRegistration] = [], codeModeRegistrations: [CodeModeRegistration] = []) { for registration in registrations { @@ -442,9 +450,17 @@ public final class CapabilityRegistry: @unchecked Sendable { } } + /// Monotonic version of the registry's contents. + public var generation: Int { + lock.lock() + defer { lock.unlock() } + return generationValue + } + public func register(_ registration: CapabilityRegistration) { lock.lock() registrations[registration.descriptor.id] = registration + generationValue += 1 lock.unlock() } @@ -453,12 +469,14 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.registrations[registration.descriptor.id] = registration } + generationValue += 1 lock.unlock() } public func register(_ registration: CodeModeRegistration) { lock.lock() codeModeRegistrations[registration.capabilityKey] = registration + generationValue += 1 lock.unlock() } @@ -467,6 +485,7 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.codeModeRegistrations[registration.capabilityKey] = registration } + generationValue += 1 lock.unlock() } @@ -506,14 +525,13 @@ public final class CapabilityRegistry: @unchecked Sendable { return registrations[capability].map(RegisteredCodeModeFunction.init) } + /// Resolves a *custom provider* key only. Built-ins are deliberately not + /// reachable here: they are dispatched by `CapabilityID` and gated by + /// `allowedCapabilities`, so resolving them through the loosely-typed key + /// namespace would let `allowedCapabilityKeys` alias a built-in. func registeredFunction(for capabilityKey: CodeModeCapabilityKey) -> RegisteredCodeModeFunction? { lock.lock() defer { lock.unlock() } - if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue), - let registration = registrations[builtIn] - { - return RegisteredCodeModeFunction(registration) - } return codeModeRegistrations[capabilityKey].map(RegisteredCodeModeFunction.init) } @@ -535,7 +553,11 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeBuiltIn(_ capability: CapabilityID, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { - guard context.allowedCapabilities.contains(capability) || context.allowedCapabilityKeys.contains(capability.codeModeKey) else { + // Built-ins are gated by `allowedCapabilities` alone. `allowedCapabilityKeys` + // accepts arbitrary strings and is not validated against `CapabilityID` at + // decode time, so honouring it here would bypass any host that vets only + // the strictly-typed field. + guard context.allowedCapabilities.contains(capability) else { throw BridgeError.capabilityDenied(capability) } @@ -547,6 +569,13 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeCodeMode(_ capabilityKey: CodeModeCapabilityKey, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { + // Defense in depth: `invoke` already routes anything that parses as a + // built-in to `invokeBuiltIn`, but a key that spells a built-in must never + // reach a handler through the provider path. + if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue) { + throw BridgeError.capabilityDenied(builtIn) + } + guard context.allowedCapabilityKeys.contains(capabilityKey) else { throw BridgeError.capabilityKeyDenied(capabilityKey) } @@ -594,7 +623,10 @@ public final class CapabilityRegistry: @unchecked Sendable { let trimmed = raw.trimmingCharacters(in: .whitespaces) switch type { case .number: - guard let number = Double(trimmed) else { return nil } + // `Double.init(String)` accepts "inf", "-infinity", and "nan", so a + // declared-number argument sent as a string is a way to smuggle a + // non-finite value past the type check. + guard let number = Double(trimmed), number.isFinite else { return nil } return .number(number) case .bool: switch trimmed.lowercased() { diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 4a64acf..2148721 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -2,8 +2,17 @@ import Foundation public final class BridgeInvocationContext: @unchecked Sendable { public let executionContext: ExecutionContext + /// Built-in capabilities this execution may reach: the model's declaration + /// intersected with the host's `CapabilityGrant`. public let allowedCapabilities: Set + /// Custom provider keys this execution may reach. Deliberately disjoint from + /// `allowedCapabilities` — a built-in capability is never granted by a key, or + /// `allowedCapabilityKeys` would be a way around a host that vets only the + /// strictly-typed `allowedCapabilities`. public let allowedCapabilityKeys: Set + /// Identifiers the request asked for and the host's grant withheld. Used to + /// tell the model a denial is not repairable by widening `allowedCapabilities`. + let hostWithheldCapabilityIdentifiers: Set public let pathPolicy: any PathPolicy public let artifactStore: any ArtifactStore public let permissionBroker: any PermissionBroker @@ -12,6 +21,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { private let lock = NSLock() private var validatedPermissions: Set = [] + private var failedCapabilityInvocations: [(capability: String, code: String)] = [] private let transcript: ExecutionTranscript private let cancellationController: ExecutionCancellationController @@ -19,6 +29,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { executionContext: ExecutionContext, allowedCapabilities: Set, allowedCapabilityKeys: Set = [], + hostWithheldCapabilityIdentifiers: Set = [], pathPolicy: any PathPolicy, artifactStore: any ArtifactStore, permissionBroker: any PermissionBroker, @@ -29,7 +40,8 @@ public final class BridgeInvocationContext: @unchecked Sendable { ) { self.executionContext = executionContext self.allowedCapabilities = allowedCapabilities - self.allowedCapabilityKeys = allowedCapabilityKeys.union(allowedCapabilities.map(\.codeModeKey)) + self.allowedCapabilityKeys = allowedCapabilityKeys + self.hostWithheldCapabilityIdentifiers = hostWithheldCapabilityIdentifiers self.pathPolicy = pathPolicy self.artifactStore = artifactStore self.permissionBroker = permissionBroker @@ -97,12 +109,64 @@ public final class BridgeInvocationContext: @unchecked Sendable { return resolved } + /// True when the request declared this identifier but the host's + /// `CapabilityGrant` withheld it — a denial the model cannot repair. + func isWithheldByHost(_ identifier: String) -> Bool { + hostWithheldCapabilityIdentifiers.contains(identifier) + } + + /// Builds a `CodeModeToolError` carrying this execution's transcript. + /// + /// Lives here rather than on the runtime because the transcript fields are + /// this type's; three separate call sites were otherwise assembling the same + /// three accessors by hand. + func toolError( + code: String, + message: String, + functionName: String? = nil, + capability: CapabilityID? = nil, + capabilityKey: CodeModeCapabilityKey? = nil, + suggestions: [String] = [] + ) -> CodeModeToolError { + CodeModeToolError( + code: code, + message: message, + functionName: functionName, + capability: capability, + capabilityKey: capabilityKey, + suggestions: suggestions, + diagnostics: allDiagnostics(), + logs: allLogs(), + permissionEvents: allPermissionEvents() + ) + } + func recordDiagnostic(_ diagnostic: ToolDiagnostic) { transcript.record(diagnostic: diagnostic) } + /// Notes a capability call that failed, so a script that finishes + /// *successfully* despite failed bridge calls can be flagged. Nothing installs + /// an unhandled-rejection hook — a forgotten `await` is the most common LLM + /// JavaScript mistake, and it silently discards a `CAPABILITY_DENIED` while + /// the agent is told the run succeeded. + func recordCapabilityFailure(capability: String, code: String) { + lock.lock() + failedCapabilityInvocations.append((capability, code)) + lock.unlock() + } + + func capabilityFailures() -> [(capability: String, code: String)] { + lock.lock() + defer { lock.unlock() } + return failedCapabilityInvocations + } + func checkCancellation() throws { - if cancellationController.isCancelled || Task.isCancelled { + // Bridge handlers run on a GCD worker, not inside a Task, so + // `Task.isCancelled` was always false here. The controller is the signal + // `JavaScriptExecutionCall.cancel()` actually sets. + if cancellationController.isCancelled { throw BridgeError.cancelled } } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 4eec3c2..2c8f9cc 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -18,6 +18,11 @@ final class BridgeRuntime: @unchecked Sendable { var suggestions: [String] } + /// Ceiling on the re-armed watchdog budget for result serialization. Bounded + /// independently of the execution timeout so a long-running script cannot buy + /// an equally long serialization window. + private static let serializationBudgetMs = 1_000 + private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let config: CodeModeConfiguration @@ -27,17 +32,36 @@ final class BridgeRuntime: @unchecked Sendable { qos: .userInitiated, attributes: .concurrent ) + /// Caps how many executions hold a thread and a live `JSContext` at once. + /// + /// The queue is concurrent and each execution blocks its thread for the whole + /// run, so thirty parallel `executeJavaScript` calls meant thirty blocked GCD + /// threads and thirty JavaScriptCore VMs. Excess executions park as suspended + /// tasks — not blocked threads — until a slot frees; `timeoutMs` still + /// measures the run itself, not the wait. + private let executionSlots: ExecutionSlots + /// Search has its own slots. It is cheap, capability-free, and bounded to 2s, + /// and sharing the execution slots meant API discovery stalled behind long + /// scripts — exactly when an agent is trying to work out its next step. + private let searchSlots: ExecutionSlots + /// Real in production; virtual under test so timer-driven tests are exact + /// and instant. See RuntimeClock. + private let clock: any RuntimeClock init( registry: CapabilityRegistry, catalog: BridgeCatalog, config: CodeModeConfiguration, - unsupportedBuiltInJavaScriptNames: [String] = [] + unsupportedBuiltInJavaScriptNames: [String] = [], + clock: any RuntimeClock = RealClock() ) { self.registry = registry self.catalog = catalog self.config = config self.unsupportedBuiltInJavaScriptNames = unsupportedBuiltInJavaScriptNames + self.clock = clock + self.executionSlots = ExecutionSlots(count: config.executionLimits.maxConcurrentExecutions) + self.searchSlots = ExecutionSlots(count: config.executionLimits.maxConcurrentExecutions) } func search(_ request: JavaScriptAPISearchRequest) throws -> JavaScriptAPISearchResponse { @@ -49,7 +73,7 @@ final class BridgeRuntime: @unchecked Sendable { ) } - let transcript = ExecutionTranscript() + let transcript = ExecutionTranscript(limits: config.executionLimits) let cancellationController = ExecutionCancellationController() let invocationContext = BridgeInvocationContext( executionContext: .init(), @@ -96,7 +120,7 @@ final class BridgeRuntime: @unchecked Sendable { } func searchAsync(_ request: JavaScriptAPISearchRequest) async throws -> JavaScriptAPISearchResponse { - try await runOnExecutionQueue { + try await runOnExecutionQueue(slots: searchSlots) { try self.search(request) } } @@ -104,10 +128,28 @@ final class BridgeRuntime: @unchecked Sendable { func makeExecutionCall(_ request: JavaScriptExecutionRequest) -> JavaScriptExecutionCall { let cancellationController = ExecutionCancellationController() let continuationBox = LockedBox.Continuation?>(nil) - let events = AsyncStream { continuation in + let resultTaskBox = LockedBox?>(nil) + + // Shared by the call and the stream; the execution is cancelled when + // the last of the two goes away. See ExecutionObservationToken. + let observation = ExecutionObservationToken { + cancellationController.cancel() + resultTaskBox.get()?.cancel() + } + + // Bounded: the default policy buffers without limit, so a chatty script + // whose events the host is not draining grows the buffer until the app + // dies. Dropping the oldest keeps the terminal events — which is what + // the host actually needs — and the full transcript is on the result. + let events = AsyncStream( + bufferingPolicy: .bufferingNewest(config.executionLimits.maxBufferedEvents) + ) { continuation in continuationBox.set(continuation) + // The stream's storage retains this closure until termination, which + // is what keeps the token alive for as long as someone is iterating. + continuation.onTermination = { _ in withExtendedLifetime(observation) {} } } - let transcript = ExecutionTranscript { event in + let transcript = ExecutionTranscript(limits: config.executionLimits) { event in continuationBox.get()?.yield(event) } @@ -154,9 +196,12 @@ final class BridgeRuntime: @unchecked Sendable { } } + resultTaskBox.set(resultTask) + return JavaScriptExecutionCall( events: events, resultTask: resultTask, + observation: observation, cancelImpl: { cancellationController.cancel() } @@ -164,9 +209,17 @@ final class BridgeRuntime: @unchecked Sendable { } private func runOnExecutionQueue( + slots: ExecutionSlots? = nil, _ operation: @escaping @Sendable () throws -> Output ) async throws -> Output { - try await withCheckedThrowingContinuation { continuation in + // The slot is taken *here*, in the async context, so an execution beyond + // the limit suspends without touching the thread pool. A cancel while + // parked throws before anything is dispatched. + let slots = slots ?? executionSlots + try await slots.acquire() + defer { Task { await slots.release() } } + + return try await withCheckedThrowingContinuation { continuation in executionQueue.async { do { continuation.resume(returning: try operation()) @@ -182,10 +235,35 @@ final class BridgeRuntime: @unchecked Sendable { transcript: ExecutionTranscript, cancellationController: ExecutionCancellationController ) throws -> JavaScriptExecutionResult { + // `allowedCapabilities` / `allowedCapabilityKeys` are model-authored, so they + // are a declaration of intent, not a boundary. The host's grant is the + // boundary: the effective set is always requested ∩ granted. + let grant = config.capabilityGrant.resolve( + requestedCapabilities: Set(request.allowedCapabilities), + requestedCapabilityKeys: Set(request.allowedCapabilityKeys) + ) + + let withheld = grant.withheld + if withheld.isEmpty == false { + transcript.record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "CAPABILITY_WITHHELD_BY_HOST", + message: "The host's capability grant withheld: \(withheld.joined(separator: ", ")). Calls needing these fail with CAPABILITY_DENIED and cannot be repaired from the script.", + category: "security", + suggestions: [ + "Do not retry with a wider allowedCapabilities; the host decides this set.", + "Complete what you can with the granted capabilities, or report the missing access to the user.", + ] + ) + ) + } + let invocationContext = BridgeInvocationContext( executionContext: request.context, - allowedCapabilities: Set(request.allowedCapabilities), - allowedCapabilityKeys: Set(request.allowedCapabilityKeys), + allowedCapabilities: grant.capabilities, + allowedCapabilityKeys: grant.capabilityKeys, + hostWithheldCapabilityIdentifiers: Set(withheld), pathPolicy: config.pathPolicy, artifactStore: config.artifactStore, permissionBroker: config.permissionBroker, @@ -256,8 +334,10 @@ final class BridgeRuntime: @unchecked Sendable { let errorPayload = self.bridgeFailurePayload( for: bridgeError, capability: capabilityID, - capabilityKey: capabilityKey + capabilityKey: capabilityKey, + invocationContext: invocationContext ) + invocationContext.recordCapabilityFailure(capability: capability, code: errorPayload.code) invocationContext.log(.error, message: "Capability failed \(capability): \(errorPayload.message)") invocationContext.auditLogger.log(AuditEvent(capability: capability, message: "failed: \(errorPayload.message)")) @@ -413,13 +493,12 @@ final class BridgeRuntime: @unchecked Sendable { lastException: LockedBox ) throws -> JSONValue? { let executionCode = Self.returningBareAwaitCode(from: code) ?? code + context.setObject(executionCode, forKeyedSubscript: Self.sourceGlobal as NSString) let script = """ globalThis.__codemode.state = 'pending'; globalThis.__codemode.result = undefined; globalThis.__codemode.error = null; - (async function(){ - \(indented(executionCode, prefix: " ")) - })() + \(Self.compileSource("Object.getPrototypeOf(async function(){}).constructor", body: "source"))() .then(function(value){ globalThis.__codemode.state = 'fulfilled'; globalThis.__codemode.result = value; @@ -436,7 +515,7 @@ final class BridgeRuntime: @unchecked Sendable { }); """ - let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController) + let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController, clock: clock) watchdog.install(on: context) defer { watchdog.uninstall(from: context) } @@ -445,16 +524,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "EXECUTION_TIMEOUT", - message: "Execution timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Execution timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Execution cancelled", - transcript: invocationContext + message: "Execution cancelled" ) case nil: break @@ -462,27 +539,36 @@ final class BridgeRuntime: @unchecked Sendable { let snapshot = lastException.get() ?? Self.snapshot(from: context.exception) if snapshot != nil || evaluation == nil { - throw syntaxError(from: snapshot, lineOffset: 4) + throw syntaxError(from: snapshot) } let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "EXECUTION_TIMEOUT", - timeoutMessage: "Execution timed out after \(timeoutMs)ms", - cancelMessage: "Execution cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "EXECUTION_TIMEOUT", + timeoutMessage: "Execution timed out after \(timeoutMs)ms", + cancelMessage: "Execution cancelled" + ) ) // Give result serialization its own bounded budget so a script that // settled near the deadline still serializes, while a runaway getter or - // toJSON is terminated instead of hanging the execution thread. - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + // toJSON is terminated instead of hanging the execution thread. `min`, + // not `max`: the point is a *bounded* budget, and `max` handed a 60s + // execution another 60s just to run JSON.stringify. + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: - let output = try decodeOutput(from: context) + recordSwallowedFailures(invocationContext: invocationContext) + let output = try decodeOutput( + from: context, + watchdog: watchdog, + invocationContext: invocationContext + ) if output == nil { invocationContext.recordDiagnostic(Self.noReturnValueDiagnostic(for: code)) } @@ -498,25 +584,36 @@ final class BridgeRuntime: @unchecked Sendable { case rejected } - /// Polls the wrapped promise's settled state until it resolves, is - /// cancelled, or the watchdog deadline passes. The settled state is checked - /// before the deadline on every iteration, so a script that has already - /// fulfilled — the common case under the synchronous bridge model — returns - /// its result even if evaluation finished a hair past the deadline, instead - /// of being discarded with a spurious timeout. + /// Runs the runtime's event loop until the wrapped promise settles, the + /// deadline passes, or the program is provably stuck. + /// + /// The settled state is checked before the deadline on every iteration, so a + /// script that has already fulfilled returns its result even if evaluation + /// finished a hair past the deadline, instead of being discarded with a + /// spurious timeout. + /// + /// When the promise is still pending, the *only* thing that can advance it + /// under the synchronous bridge model is a queued timer — microtasks have + /// already drained inside `evaluateScript`, and no native call is + /// outstanding. So this either sleeps until the next timer is due and fires + /// it, or, when no timer is queued, reports immediately that the promise can + /// never settle instead of sleeping a thread to the deadline for nothing. private func waitForSettlement( context: JSContext, - watchdog: ExecutionWatchdog, - cancellationController: ExecutionCancellationController, - invocationContext: BridgeInvocationContext, - timeoutCode: String, - timeoutMessage: String, - cancelMessage: String + settlement: SettlementParameters ) throws -> SettlementState { while true { - if cancellationController.isCancelled || Task.isCancelled { - cancellationController.cancel() - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) + // Cancellation only, deliberately: the deadline is checked *after* + // the settled state below, so a script that fulfilled a hair past it + // still returns its result instead of a spurious timeout. + // + // `Task.isCancelled` is not checked at all — this runs on a plain GCD + // worker with no surrounding Task, so it was always false, dead code + // that read as a second layer of cancellation support. The + // controller, which `JavaScriptExecutionCall.cancel()` sets, is the + // real signal. + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) } let state = context.evaluateScript("globalThis.__codemode.state")?.toString() ?? "unknown" @@ -526,22 +623,143 @@ final class BridgeRuntime: @unchecked Sendable { case "rejected": return .rejected default: - if let termination = watchdog.termination { - switch termination { - case .cancelled: - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) - case .timedOut: - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } - } - if Date() >= watchdog.deadline { - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) + try checkInterrupted(settlement) + + let nextTimerDelay = context + .evaluateScript("globalThis.__codemode.nextTimerDelay()")? + .toNumber()? + .doubleValue ?? -1 + + guard nextTimerDelay >= 0 else { + throw settlement.error( + code: "JS_RUNTIME_ERROR", + message: "Script finished with a promise that can never settle: nothing is left to resolve it. This runtime has no I/O event loop, so only a pending setTimeout can advance a program after the synchronous work is done.", + suggestions: [ + "Await every promise the script creates, and make sure each one has a resolve path.", + "A `new Promise(...)` whose executor never calls resolve or reject will hang forever here.", + ] + ) } - Thread.sleep(forTimeInterval: 0.01) + + try runDueTimers(delayMs: nextTimerDelay, context: context, settlement: settlement) } } } + /// Everything the settlement loop needs to report a cancellation or timeout. + /// The execution and search paths differ only in these message strings. + private struct SettlementParameters { + var watchdog: ExecutionWatchdog + var cancellationController: ExecutionCancellationController + var invocationContext: BridgeInvocationContext + var timeoutCode: String + var timeoutMessage: String + var cancelMessage: String + + func error(code: String, message: String, suggestions: [String] = []) -> CodeModeToolError { + invocationContext.toolError(code: code, message: message, suggestions: suggestions) + } + } + + /// Throws if the run was cancelled or has run out of time — the check every + /// waiting loop in this file has to make before sleeping again. + private func checkInterrupted(_ settlement: SettlementParameters) throws { + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + } + switch settlement.watchdog.termination { + case .cancelled: + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + case .timedOut: + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + case nil: + break + } + if settlement.watchdog.hasPassedDeadline { + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + } + } + + /// Sleeps out a timer's delay — bounded by the deadline and responsive to + /// cancellation — then fires everything that has come due. + private func runDueTimers( + delayMs: Double, + context: JSContext, + settlement: SettlementParameters + ) throws { + let start = clock.now + // Bounded by the remaining budget *before* conversion. `delayMs` comes + // out of JavaScript, and `Int(_: Double)` traps on anything non-finite or + // out of range — `setTimeout(r, 1e300)` killed the host process here, + // with no capability required. The script normalizes delays too, but + // this conversion must not depend on that: whatever arrives, the wait is + // at most the time left before the deadline, which is always small. + let remainingMs = max(0, (settlement.watchdog.deadline - start).milliseconds) + let waitMs = delayMs.isFinite ? min(max(0, delayMs), remainingMs) : remainingMs + let target = start.advanced(by: .milliseconds(Int(waitMs.rounded(.up)))) + + while clock.now < target { + try checkInterrupted(settlement) + // One interruptible wait until the timer is due or the deadline + // passes — a cancel wakes it immediately, and the loop re-checks + // both against the clock on waking. + clock.sleep(until: min(target, settlement.watchdog.deadline), cancellation: settlement.cancellationController) + } + + // Advance by the time that actually passed, so timers that came due while + // we slept fire together. The call returns the callbacks' uncaught errors, + // which cannot propagate to whoever called `setTimeout` and so surface as + // diagnostics the way an uncaught task error does in a real event loop. + let elapsed = (clock.now - start).milliseconds + let errors = Self.stringArray( + from: context, + evaluating: "globalThis.__codemode.advanceTimers(\(elapsed))" + ) + + for message in errors { + settlement.invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "TIMER_CALLBACK_ERROR", + message: "A setTimeout callback threw and the error was discarded: \(message)", + category: "execution", + suggestions: ["Handle errors inside timer callbacks; they cannot propagate to the caller of setTimeout."] + ) + ) + } + } + + /// Flags bridge calls that failed while the script still reported success. + /// + /// The runtime installs no unhandled-rejection hook, so a forgotten `await` — + /// the most common LLM JavaScript mistake — discards a `CAPABILITY_DENIED` + /// and the agent is told the run completed. This does not prove the failure + /// was unhandled (a deliberate try/catch looks the same), so it is a warning + /// rather than an error, but it makes the silent case visible. + private func recordSwallowedFailures(invocationContext: BridgeInvocationContext) { + let failures = invocationContext.capabilityFailures() + guard failures.isEmpty == false else { + return + } + + let summary = failures + .map { "\($0.capability) (\($0.code))" } + .joined(separator: ", ") + + invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "BRIDGE_FAILURES_NOT_SURFACED", + message: "The script completed successfully, but \(failures.count) bridge call(s) failed and the failure did not reach the result: \(summary).", + category: "execution", + suggestions: [ + "If that was not deliberate, check for a missing await — an un-awaited helper's rejection is silently discarded here.", + "If it was deliberate, ignore this; the result is what the script returned.", + ] + ) + ) + } + private static func noReturnValueDiagnostic(for code: String) -> ToolDiagnostic { let mentionsAsyncIIFE = code.contains("(async") && (code.contains("})()") || code.contains("})();")) let message: String @@ -584,13 +802,18 @@ final class BridgeRuntime: @unchecked Sendable { cancellationController: ExecutionCancellationController, lastException: LockedBox ) throws -> JSONValue? { + context.setObject(code, forKeyedSubscript: Self.sourceGlobal as NSString) + // The expression is parenthesized inside the compiled body, so the most + // it can escape is the parentheses — into a function body it could + // write anyway as an async arrow. The newline keeps a trailing `//` + // comment from swallowing the close. let script = """ globalThis.__codemode.state = 'pending'; globalThis.__codemode.result = undefined; globalThis.__codemode.error = null; - Promise.resolve() - .then(function(){ - const __search = (\(code)); + Promise.resolve(\(Self.compileSource("Function", body: "'return (' + source + '\\n);'"))) + .then(function(makeSearch){ return makeSearch(); }) + .then(function(__search){ if (typeof __search !== 'function') { const error = new Error('searchJavaScriptAPI code must evaluate to a function'); error.code = 'INVALID_REQUEST'; @@ -611,7 +834,7 @@ final class BridgeRuntime: @unchecked Sendable { }); """ - let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController) + let watchdog = ExecutionWatchdog(timeoutMs: timeoutMs, cancellationController: cancellationController, clock: clock) watchdog.install(on: context) defer { watchdog.uninstall(from: context) } @@ -620,16 +843,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "SEARCH_TIMEOUT", - message: "Search timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Search timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Search cancelled", - transcript: invocationContext + message: "Search cancelled" ) case nil: break @@ -637,25 +858,29 @@ final class BridgeRuntime: @unchecked Sendable { let snapshot = lastException.get() ?? Self.snapshot(from: context.exception) if snapshot != nil || evaluation == nil { - throw syntaxError(from: snapshot, lineOffset: 5) + throw syntaxError(from: snapshot) } let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "SEARCH_TIMEOUT", - timeoutMessage: "Search timed out after \(timeoutMs)ms", - cancelMessage: "Search cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "SEARCH_TIMEOUT", + timeoutMessage: "Search timed out after \(timeoutMs)ms", + cancelMessage: "Search cancelled" + ) ) - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: return try decodeOutput( from: context, + watchdog: watchdog, + invocationContext: invocationContext, errorCode: "INVALID_SEARCH_RESULT", errorMessagePrefix: "Search result must be JSON-serializable" ) @@ -667,6 +892,8 @@ final class BridgeRuntime: @unchecked Sendable { private func decodeOutput( from context: JSContext, + watchdog: ExecutionWatchdog? = nil, + invocationContext: BridgeInvocationContext? = nil, errorCode: String = "INVALID_RESULT", errorMessagePrefix: String = "Execution result must be JSON-serializable" ) throws -> JSONValue? { @@ -674,17 +901,46 @@ final class BridgeRuntime: @unchecked Sendable { return nil } - guard let serialization = context.evaluateScript( + let limits = config.executionLimits + // Size is measured and the result is replaced *inside* JS: bringing a + // gigabyte of JSON across into a Swift String and then rejecting it has + // already spent the memory the cap exists to protect. The replacement is + // still valid JSON, so the model gets a parseable value plus a + // diagnostic rather than a truncated fragment it cannot read. + let serialization = context.evaluateScript( """ (function(){ try { - return { ok: true, json: JSON.stringify(globalThis.__codemode.result) }; + var json = JSON.stringify(globalThis.__codemode.result); + if (typeof json === 'string' && json.length > \(limits.maxResultCharacters)) { + return { + ok: true, + truncated: true, + size: json.length, + json: JSON.stringify({ + __codeModeTruncated: true, + characterCount: json.length, + limit: \(limits.maxResultCharacters), + preview: json.slice(0, \(limits.truncatedResultPreviewCharacters)) + }) + }; + } + return { ok: true, json: json }; } catch (error) { return { ok: false, message: String(error) }; } })() """ - ) else { + ) + + // Checked before anything else: a watchdog termination during + // serialization is a timeout, not a serialization problem, and reporting + // `INVALID_RESULT: must be JSON-serializable` for a runaway getter + // steered the model to fix the wrong thing. + if let termination = watchdog?.termination { + throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") + } + guard let serialization else { throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) } @@ -693,6 +949,22 @@ final class BridgeRuntime: @unchecked Sendable { throw CodeModeToolError(code: errorCode, message: "\(errorMessagePrefix): \(message)") } + if serialization.forProperty("truncated")?.toBool() == true { + let size = serialization.forProperty("size")?.toNumber()?.intValue ?? limits.maxResultCharacters + invocationContext?.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "RESULT_TRUNCATED", + message: "The result serialized to \(size) characters, over the \(limits.maxResultCharacters)-character limit. It was replaced with a descriptor object carrying a \(limits.truncatedResultPreviewCharacters)-character preview.", + category: "execution", + suggestions: [ + "Return a summary, a count, or a slice instead of the whole collection.", + "Write large payloads to the sandbox with apple.fs.write and return the path.", + ] + ) + ) + } + guard let jsonValue = serialization.forProperty("json"), jsonValue.isUndefined == false, jsonValue.isNull == false, let serialized = jsonValue.toString() else { @@ -706,12 +978,37 @@ final class BridgeRuntime: @unchecked Sendable { return try JSONDecoder.codeModeBridge.decode(JSONValue.self, from: data) } + /// Reports a watchdog termination that happened outside the script body — so + /// far, during result serialization — as the timeout or cancellation it + /// actually is. + private func terminationError( + _ termination: ExecutionWatchdog.Termination, + invocationContext: BridgeInvocationContext?, + during phase: String + ) -> CodeModeToolError { + let (code, message): (String, String) = switch termination { + case .timedOut: + ("EXECUTION_TIMEOUT", "Execution timed out during \(phase). A getter or toJSON on the returned value did not finish within the serialization budget.") + case .cancelled: + ("CANCELLED", "Execution cancelled during \(phase)") + } + + return invocationContext?.toolError(code: code, message: message) + ?? CodeModeToolError(code: code, message: message) + } + private func bridgeFailurePayload( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> CodeModeToolError { - let (message, suggestions) = enrichedBridgeFailure(for: error, capability: capability, capabilityKey: capabilityKey) + let (message, suggestions) = enrichedBridgeFailure( + for: error, + capability: capability, + capabilityKey: capabilityKey, + invocationContext: invocationContext + ) return CodeModeToolError( code: error.diagnosticCode, message: message, @@ -798,14 +1095,41 @@ final class BridgeRuntime: @unchecked Sendable { ) } - private func syntaxError(from snapshot: JavaScriptExceptionSnapshot?, lineOffset: Int) -> CodeModeToolError { + /// The global a script's source is handed over in. Read and deleted by + /// `compileSource` before the script runs. + private static let sourceGlobal = "__codemodeSource" + + /// A JS expression compiling the script source with `constructor`. + /// + /// The source is never spliced into the harness. It used to be pasted into + /// `(async function(){ … })()`, so `return 1; })(); throw …; (async function(){` + /// closed the wrapper and ran at top level — outside the harness's + /// `.catch`, and with full control of the exception the host then read. The + /// `Function` constructors parse their body on its own, so it cannot close + /// the function it is compiled into; a body that tries is a syntax error. + private static func compileSource(_ constructor: String, body: String) -> String { + """ + (function(){ + const source = globalThis.\(sourceGlobal); + delete globalThis.\(sourceGlobal); + return new (\(constructor))(\(body)); + })() + """ + } + + /// A compile error from `compileSource`. JavaScriptCore numbers lines in the + /// generated `function anonymous(\n) {\n…` source, so the script's first line + /// is line 3. It reports the column of the constructor *call*, not of the + /// error, so no column is given rather than a wrong one. + private func syntaxError(from snapshot: JavaScriptExceptionSnapshot?) -> CodeModeToolError { + let generatedPreambleLines = 2 let message = snapshot?.message ?? "JavaScript syntax error" - let adjustedLine = snapshot?.line.map { max(1, $0 - lineOffset) } + // Non-overflowing even if a line ever arrives unvalidated. + let adjustedLine = snapshot?.line.map { $0 > generatedPreambleLines ? $0 - generatedPreambleLines : 1 } return CodeModeToolError( code: "JS_SYNTAX_ERROR", message: message, - line: adjustedLine, - column: snapshot?.column + line: adjustedLine ) } @@ -833,13 +1157,19 @@ final class BridgeRuntime: @unchecked Sendable { } private func rejectedSuggestions(from context: JSContext) -> [String] { - guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.error?.suggestions ?? [])")?.toString(), + Self.stringArray(from: context, evaluating: "globalThis.__codemode.error?.suggestions ?? []") + } + + /// Reads a JavaScript array of strings across the bridge. Returns an empty + /// array if the expression is missing or not string-shaped. + private static func stringArray(from context: JSContext, evaluating expression: String) -> [String] { + guard let json = context.evaluateScript("JSON.stringify(\(expression))")?.toString(), let data = json.data(using: .utf8), - let suggestions = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) + let values = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) else { return [] } - return suggestions + return values } private func event(for error: CodeModeToolError) -> JavaScriptExecutionEvent { @@ -855,28 +1185,6 @@ final class BridgeRuntime: @unchecked Sendable { } } - private func toolError( - code: String, - message: String, - transcript: BridgeInvocationContext, - functionName: String? = nil, - capability: CapabilityID? = nil, - capabilityKey: CodeModeCapabilityKey? = nil, - suggestions: [String] = [] - ) -> CodeModeToolError { - CodeModeToolError( - code: code, - message: message, - functionName: functionName, - capability: capability, - capabilityKey: capabilityKey, - suggestions: suggestions, - diagnostics: transcript.allDiagnostics(), - logs: transcript.allLogs(), - permissionEvents: transcript.allPermissionEvents() - ) - } - private func bridgeSuggestions(for capability: CapabilityID?) -> [String] { bridgeSuggestions(for: capability, capabilityKey: capability?.codeModeKey) } @@ -908,14 +1216,34 @@ final class BridgeRuntime: @unchecked Sendable { private func enrichedBridgeFailure( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> (String, [String]) { + // A host-withheld denial reads the same whichever allowlist it came from, + // and is the one denial the model cannot repair — so it short-circuits + // both cases below. + let deniedIdentifier: String? = switch error { + case let .capabilityDenied(capability): capability.rawValue + case let .capabilityKeyDenied(capabilityKey): capabilityKey.rawValue + default: nil + } + if let deniedIdentifier, invocationContext?.isWithheldByHost(deniedIdentifier) == true { + return ( + error.localizedDescription, + [ + "The host app's capability grant does not include \"\(deniedIdentifier)\".", + "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", + "Continue with the capabilities you do have, or report the missing access to the user.", + ] + ) + } + let suggestions: [String] switch error { case let .capabilityDenied(capability): suggestions = [ "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", - "If your host uses a unified capability-key allowlist, add \"\(capability.rawValue)\" to allowedCapabilityKeys.", + "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) case let .capabilityKeyDenied(capabilityKey): suggestions = [ @@ -1075,22 +1403,28 @@ final class BridgeRuntime: @unchecked Sendable { return JavaScriptExceptionSnapshot(name: name, message: message, line: line, column: column) } + /// A line or column from an exception object, or nil if it is not one. + /// + /// The exception is not necessarily JavaScriptCore's own — until scripts + /// were compiled on their own (see `compileSource`), one could throw an + /// arbitrary object straight to the host. `NSNumber.intValue` saturates + /// `-Infinity` to `Int.min`, and the line adjustment then overflowed: + /// `throw { line: -Infinity }` crashed the host. Only a positive whole + /// number in a plausible range is accepted. private static func numericProperty(in exception: JSValue, names: [String]) -> Int? { for name in names { - if let value = exception.forProperty(name)?.toNumber() { - return value.intValue + guard let number = exception.forProperty(name)?.toNumber() else { + continue } + let value = number.doubleValue + guard value.isFinite, value == value.rounded(), value >= 1, value <= Double(Int32.max) else { + return nil + } + return Int(value) } return nil } - private func indented(_ code: String, prefix: String) -> String { - code - .split(separator: "\n", omittingEmptySubsequences: false) - .map { prefix + $0 } - .joined(separator: "\n") - } - private func normalizedOptionalString(_ value: String?) -> String? { guard let value else { return nil diff --git a/Sources/CodeMode/Runtime/ExecutionLimits.swift b/Sources/CodeMode/Runtime/ExecutionLimits.swift new file mode 100644 index 0000000..f5529fb --- /dev/null +++ b/Sources/CodeMode/Runtime/ExecutionLimits.swift @@ -0,0 +1,66 @@ +import Foundation + +/// Bounds on how much an execution may accumulate, plus how many may run at once. +/// +/// Nothing about a script's *output* was previously capped: the whole result was +/// stringified, copied into a Swift `String`, and re-decoded into a `JSONValue`; +/// every `console.log` was retained and then copied again into any +/// `CodeModeToolError`; and the events `AsyncStream` used unbounded buffering, so +/// a chatty script that the host was not draining buffered without limit. A +/// single script could jetsam an iOS host. +/// +/// For an LLM consumer, a truncated result carrying a diagnostic is strictly +/// better than an out-of-memory kill, so every limit here degrades rather than +/// fails. +public struct ExecutionLimits: Sendable, Equatable { + /// Maximum size of the serialized result, in JSON characters. A larger + /// result is replaced by a descriptor object carrying a prefix of the JSON. + public var maxResultCharacters: Int + + /// How much of the oversized JSON to include in that descriptor. + public var truncatedResultPreviewCharacters: Int + + /// Maximum retained `console.log` entries. + public var maxLogEntries: Int + + /// Maximum characters retained per log message; longer messages are elided + /// in the middle, which keeps both the prefix and the tail readable. + public var maxLogMessageCharacters: Int + + /// Maximum retained diagnostics. + public var maxDiagnostics: Int + + /// Maximum retained permission events. + public var maxPermissionEvents: Int + + /// How many executions may hold a thread and a live JavaScriptCore context + /// at once. Excess executions queue rather than exhausting the GCD thread + /// pool; the per-execution `timeoutMs` measures the run, not the wait. + public var maxConcurrentExecutions: Int + + /// Depth of the events `AsyncStream` buffer. A host that stops draining + /// drops the oldest events rather than growing without bound. + public var maxBufferedEvents: Int + + public init( + maxResultCharacters: Int = 1_000_000, + truncatedResultPreviewCharacters: Int = 4_000, + maxLogEntries: Int = 1_000, + maxLogMessageCharacters: Int = 8_000, + maxDiagnostics: Int = 200, + maxPermissionEvents: Int = 200, + maxBufferedEvents: Int = 1_000, + maxConcurrentExecutions: Int = 8 + ) { + self.maxResultCharacters = maxResultCharacters + self.truncatedResultPreviewCharacters = truncatedResultPreviewCharacters + self.maxLogEntries = maxLogEntries + self.maxLogMessageCharacters = maxLogMessageCharacters + self.maxDiagnostics = maxDiagnostics + self.maxPermissionEvents = maxPermissionEvents + self.maxBufferedEvents = maxBufferedEvents + self.maxConcurrentExecutions = maxConcurrentExecutions + } + + public static let standard = ExecutionLimits() +} diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 1b20c1f..ac6f1d2 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -1,56 +1,201 @@ import Foundation final class ExecutionCancellationController: @unchecked Sendable { - private let lock = NSLock() + private let condition = NSCondition() private var cancelled = false func cancel() { - lock.lock() + condition.lock() cancelled = true - lock.unlock() + // Wakes anything parked in `sleep(until:)`, so a cancel lands + // immediately instead of at the next poll. + condition.broadcast() + condition.unlock() } var isCancelled: Bool { - lock.lock() - defer { lock.unlock() } + condition.lock() + defer { condition.unlock() } return cancelled } + + /// Blocks until `date` or until cancelled, whichever is first. The timer + /// loop used to poll `Thread.sleep` in 5ms slices — 200 wakeups a second per + /// sleeping execution, and up to 5ms of cancellation latency — where one + /// interruptible wait does both jobs. + func sleep(until date: Date) { + condition.lock() + defer { condition.unlock() } + while cancelled == false, Date() < date { + condition.wait(until: date) + } + } +} + +/// Bounds how many executions run at once, parking the excess as suspended +/// tasks rather than blocked threads. +/// +/// A `DispatchSemaphore` acquired on the GCD worker got this exactly backwards: +/// every execution beyond the limit held a thread while waiting for a slot, which +/// is the resource the limit exists to protect. Waiters here cost nothing until +/// a slot frees, and a cancelled waiter leaves the queue without ever running. +actor ExecutionSlots { + private var available: Int + private var waiters: [(id: UUID, continuation: CheckedContinuation)] = [] + + init(count: Int) { + available = max(1, count) + } + + func acquire() async throws { + try Task.checkCancellation() + if available > 0 { + available -= 1 + return + } + + let id = UUID() + try await withTaskCancellationHandler { + // Runs synchronously on the actor, so the waiter is always parked + // before a `withdraw` from the cancellation handler can be scheduled + // here — even when the task was cancelled before this point. + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + waiters.append((id, continuation)) + } + } onCancel: { + Task { await self.withdraw(id) } + } + } + + func release() { + // Hand the slot straight to the next waiter; `available` only grows when + // nobody is waiting for it. + if waiters.isEmpty == false { + waiters.removeFirst().continuation.resume() + } else { + available += 1 + } + } + + /// Removes a cancelled waiter. Not finding it means `release()` already + /// handed it a slot — the cancel arrived too late to matter, and the + /// execution observes it through its cancellation controller instead. An + /// earlier version recorded such ids "for later" and never removed them. + private func withdraw(_ id: UUID) { + if let index = waiters.firstIndex(where: { $0.id == id }) { + waiters.remove(at: index).continuation.resume(throwing: CancellationError()) + } + } +} + +/// Cancels an execution once nothing can observe its outcome. +/// +/// Held by both the `JavaScriptExecutionCall` and its events stream, so the +/// execution keeps running while *either* is reachable — a host iterating +/// `call.events` after its last use of `call` is still observing. Cancelling in +/// the call's own `deinit` cut that host off mid-stream: Swift does not keep a +/// local alive to the end of its scope, and an optimized build released `call` as +/// soon as `.events` had been read. +final class ExecutionObservationToken: @unchecked Sendable { + private let onDrop: @Sendable () -> Void + + init(onDrop: @escaping @Sendable () -> Void) { + self.onDrop = onDrop + } + + deinit { + onDrop() + } } final class ExecutionTranscript: @unchecked Sendable { private let lock = NSLock() + private let limits: ExecutionLimits private var logs: [ExecutionLog] = [] private var diagnostics: [ToolDiagnostic] = [] private var permissionEvents: [PermissionEvent] = [] + private var noticedLogOverflow = false private let emitEvent: @Sendable (JavaScriptExecutionEvent) -> Void - init(emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in }) { + init( + limits: ExecutionLimits = .standard, + emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in } + ) { + self.limits = limits self.emitEvent = emitEvent } func record(log: ExecutionLog) { + var entry = log + entry.message = Self.elide(entry.message, to: limits.maxLogMessageCharacters) + + // Keep the first N: the beginning of a runaway log is where the cause is, + // and the tail is the same line a million times. lock.lock() - logs.append(log) + let accepted = logs.count < limits.maxLogEntries + let isFirstOverflow = accepted == false && noticedLogOverflow == false + if accepted { + logs.append(entry) + } else { + noticedLogOverflow = true + } lock.unlock() - emitEvent(.log(log)) + + if accepted { + // Streaming stays live even after the retained transcript is full; + // it is the retained copy — re-copied into every CodeModeToolError — + // that is the memory risk. + emitEvent(.log(entry)) + } else if isFirstOverflow { + record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "LOG_LIMIT_REACHED", + message: "Kept the first \(limits.maxLogEntries) console.log entries; later entries are dropped.", + category: "execution", + suggestions: ["Log summaries rather than per-item lines, or return the data instead of logging it."] + ) + ) + } } func record(diagnostic: ToolDiagnostic) { lock.lock() - diagnostics.append(diagnostic) + let accepted = diagnostics.count < limits.maxDiagnostics + if accepted { + diagnostics.append(diagnostic) + } lock.unlock() - if diagnostic.severity != .error { + // Error-severity diagnostics used to be recorded but never emitted, so a + // host watching the stream saw every diagnostic *except* the ones that + // mattered most. They are on the result either way; the stream is the + // live view and should not be the incomplete one. + if accepted { emitEvent(.diagnostic(diagnostic)) } } func record(permissionEvent: PermissionEvent) { lock.lock() - permissionEvents.append(permissionEvent) + if permissionEvents.count < limits.maxPermissionEvents { + permissionEvents.append(permissionEvent) + } lock.unlock() } + /// Elides the middle of an over-long message, keeping both ends: a stack + /// trace's origin and its outcome are usually at opposite ends. + private static func elide(_ message: String, to limit: Int) -> String { + guard limit > 0, message.count > limit else { + return message + } + let keep = max(1, (limit - 1) / 2) + let head = message.prefix(keep) + let tail = message.suffix(keep) + return "\(head)… [\(message.count - 2 * keep) characters elided] …\(tail)" + } + func snapshot(output: JSONValue? = nil) -> JavaScriptExecutionResult { lock.lock() defer { lock.unlock() } diff --git a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift index a9b8ea4..c70c1b4 100644 --- a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift +++ b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift @@ -40,21 +40,30 @@ final class ExecutionWatchdog: @unchecked Sendable { fileprivate static let checkInterval: TimeInterval = 0.05 private let lock = NSLock() - private var deadlineValue: Date + /// `ContinuousClock`, not `Date`: a wall-clock deadline moves when NTP + /// adjusts the clock, which can push a timeout arbitrarily far out (or fire it + /// immediately) while a script is running. + private var deadlineValue: ContinuousClock.Instant private var terminationValue: Termination? private let cancellationController: ExecutionCancellationController + let clock: any RuntimeClock - init(timeoutMs: Int, cancellationController: ExecutionCancellationController) { - self.deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + init(timeoutMs: Int, cancellationController: ExecutionCancellationController, clock: any RuntimeClock = RealClock()) { + self.clock = clock + self.deadlineValue = clock.now.advanced(by: .milliseconds(timeoutMs)) self.cancellationController = cancellationController } - var deadline: Date { + var deadline: ContinuousClock.Instant { lock.lock() defer { lock.unlock() } return deadlineValue } + var hasPassedDeadline: Bool { + clock.now >= deadline + } + var termination: Termination? { lock.lock() defer { lock.unlock() } @@ -67,7 +76,7 @@ final class ExecutionWatchdog: @unchecked Sendable { /// user-defined getters/`toJSON`), while a runaway getter is still terminated. func rearm(timeoutMs: Int) { lock.lock() - deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + deadlineValue = clock.now.advanced(by: .milliseconds(timeoutMs)) lock.unlock() } @@ -100,7 +109,7 @@ final class ExecutionWatchdog: @unchecked Sendable { terminationValue = .cancelled return true } - if Date() >= deadlineValue { + if clock.now >= deadlineValue { terminationValue = .timedOut return true } diff --git a/Sources/CodeMode/Runtime/RuntimeClock.swift b/Sources/CodeMode/Runtime/RuntimeClock.swift new file mode 100644 index 0000000..1ff11fe --- /dev/null +++ b/Sources/CodeMode/Runtime/RuntimeClock.swift @@ -0,0 +1,83 @@ +import Foundation + +/// The runtime's source of time, so the parts of execution that wait — the +/// timer loop and the watchdog deadline — can be driven by a virtual clock under +/// test instead of by real elapsed time. +/// +/// Production always uses `RealClock`. The seam exists because timer tests that +/// really slept were the least reliable tests in the suite: their assertions were +/// wall-clock bounds, and a loaded CI runner turned a 150ms wait into seconds. +/// Under a `VirtualClock` a sleep is a jump, so those tests are instant and +/// exact, and they can assert on how long the runtime *asked* to wait rather +/// than on how long the machine took. +/// +/// Watchdog termination of a CPU-bound script stays real-time regardless: it is +/// driven by JavaScriptCore's own execution-time callback, which virtual time +/// cannot advance. Tests of that path use `RealClock`. +protocol RuntimeClock: Sendable { + var now: ContinuousClock.Instant { get } + + /// Blocks until `deadline` or until `cancellation` fires, whichever is + /// first. A virtual clock jumps instead of blocking. + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) +} + +struct RealClock: RuntimeClock { + var now: ContinuousClock.Instant { ContinuousClock.now } + + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) { + // The wall-clock `Date` is only the wake-up hint; callers re-check + // against `now`, so an NTP adjustment cannot move a deadline. + let seconds = max(0, (deadline - ContinuousClock.now).milliseconds / 1_000) + cancellation.sleep(until: Date(timeIntervalSinceNow: seconds)) + } +} + +/// A clock that only moves when the runtime sleeps. +/// +/// `sleep(until:)` advances `now` to the deadline — plus `overshoot`, which +/// models a slow machine waking late. That is the one condition a real clock +/// cannot reproduce on demand, and it is exactly the condition that exposed the +/// timer-ordering bug: two timers with different delays both coming due in one +/// tick. +final class VirtualClock: RuntimeClock, @unchecked Sendable { + private let base = ContinuousClock.now + private let lock = NSLock() + private var offset: Duration = .zero + let overshoot: Duration + + init(overshoot: Duration = .zero) { + self.overshoot = overshoot + } + + var now: ContinuousClock.Instant { + base.advanced(by: elapsed) + } + + /// Total virtual time the runtime has slept through. + var elapsed: Duration { + lock.lock() + defer { lock.unlock() } + return offset + } + + func sleep(until deadline: ContinuousClock.Instant, cancellation: ExecutionCancellationController) { + // A cancelled sleep returns without advancing, as a real one would wake + // early; the caller's interrupt check then throws. + guard cancellation.isCancelled == false else { + return + } + lock.lock() + offset = max(offset, (deadline - base) + overshoot) + lock.unlock() + } +} + +extension Duration { + /// Whole and fractional milliseconds. `components` splits into seconds plus + /// attoseconds, which is otherwise an awkward two-term conversion. + var milliseconds: Double { + let parts = components + return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15 + } +} diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index f10ef84..ddce142 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -1,6 +1,97 @@ import Foundation enum RuntimeJavaScript { + /// Shared by the execution and search runtimes. + private static let timerRuntime = """ + // A real timer queue, drained by the host between JavaScript turns. + // + // `setTimeout` used to invoke its callback synchronously and ignore the + // delay, which made three things wrong at once: `clearTimeout` could never + // cancel, an error thrown by the callback propagated to setTimeout's *caller* + // rather than being an uncaught task error, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was + // a hot loop that hammered remote APIs through `fetch`. + globalThis.__codemode.timers = { nextID: 1, entries: [] }; + + globalThis.setTimeout = function(fn, delay) { + if (typeof fn !== 'function') { + return 0; + } + const timers = globalThis.__codemode.timers; + const id = timers.nextID++; + timers.entries.push({ + id: id, + fn: fn, + args: Array.prototype.slice.call(arguments, 2), + dueIn: __normalizedTimerDelay(delay) + }); + return id; + }; + + // NaN, negative, and missing delays mean "as soon as possible", as on the + // web. Anything above 2^31-1 ms — including Infinity and 1e300 — is clamped + // to it: ~24.8 days is far beyond any execution deadline, so such a timer + // can only end in a timeout, and the host never sees an unrepresentable + // number. (Browsers instead wrap large delays to fire immediately, which + // would make `setTimeout(r, Infinity)` a hot loop.) + function __normalizedTimerDelay(delay) { + const ms = Number(delay); + if (!(ms > 0)) { + return 0; + } + return Math.min(ms, 2147483647); + } + + globalThis.clearTimeout = function(id) { + const timers = globalThis.__codemode.timers; + timers.entries = timers.entries.filter(function(entry){ return entry.id !== id; }); + }; + + // Milliseconds until the earliest pending timer, or -1 when none are queued. + // -1 means nothing can advance the program: under this runtime's synchronous + // bridge there is no other source of asynchrony. + globalThis.__codemode.nextTimerDelay = function() { + const entries = globalThis.__codemode.timers.entries; + if (entries.length === 0) { + return -1; + } + return entries.reduce(function(min, entry){ return Math.min(min, entry.dueIn); }, Infinity); + }; + + globalThis.__codemode.advanceTimers = function(elapsedMs) { + const timers = globalThis.__codemode.timers; + const due = []; + const remaining = []; + timers.entries.forEach(function(entry){ + entry.dueIn -= elapsedMs; + if (entry.dueIn <= 0) { due.push(entry); } else { remaining.push(entry); } + }); + timers.entries = remaining; + // Due time first, registration order only to break ties — the order a + // real event loop fires in. Sorting by id alone was wrong whenever the + // host's sleep overshot far enough for two timers with *different* + // delays to come due in the same tick: the later one fired first purely + // because it was registered first. `dueIn` is now negative for anything + // overdue, so ascending puts the most overdue first. + due.sort(function(a, b){ + return a.dueIn === b.dueIn ? a.id - b.id : a.dueIn - b.dueIn; + }); + const errors = []; + due.forEach(function(entry){ + try { + entry.fn.apply(null, entry.args); + } catch (error) { + // An uncaught error in a task cannot propagate to whoever called + // setTimeout; the host reports it as a diagnostic instead. + errors.push(String(error)); + } + }); + // Returned rather than buffered, so the host needs one round trip per + // tick instead of two. + return errors; + }; + """ + static func pruningScript(removingJavaScriptNames names: some Sequence) -> String { let bindingsToRemove = Array(Set(names)).sorted() @@ -123,13 +214,7 @@ enum RuntimeJavaScript { error: function(){ __searchConsole('error', Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) """ static let bootstrap = """ @@ -200,13 +285,7 @@ enum RuntimeJavaScript { error: function(){ __nativeConsoleLog(Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) if (typeof URLSearchParams === 'undefined') { globalThis.URLSearchParams = function(initial){ @@ -258,16 +337,32 @@ enum RuntimeJavaScript { }; globalThis.apple = globalThis.apple || {}; + // These accept either the object form the catalog advertises — + // apple.keychain.get({ key }) — or the positional form the examples have + // always shown. The catalog described object arguments while the wrapper took + // a positional string, so code written from the metadata sent "[object + // Object]" as the key. + function __keychainArgs(first, value) { + if (first && typeof first === 'object') { + const args = { key: String(first.key) }; + if (first.value !== undefined) args.value = String(first.value); + return args; + } + const args = { key: String(first) }; + if (value !== undefined) args.value = String(value); + return args; + } + globalThis.apple.keychain = { - get: function(key) { return __invokeAsync('keychain.read', { key: String(key) }); }, - set: function(key, value) { return __invokeAsync('keychain.write', { key: String(key), value: String(value) }); }, - delete: function(key) { return __invokeAsync('keychain.delete', { key: String(key) }); } + get: function(key) { return __invokeAsync('keychain.read', __keychainArgs(key)); }, + set: function(key, value) { return __invokeAsync('keychain.write', __keychainArgs(key, value)); }, + delete: function(key) { return __invokeAsync('keychain.delete', __keychainArgs(key)); } }; globalThis.apple.location = { - getPermissionStatus: function() { return __invokeAsync('location.read', { mode: 'permissionStatus' }); }, + getPermissionStatus: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'permissionStatus' }, args || {})); }, requestPermission: function() { return __invokeAsync('location.permission.request', {}); }, - getCurrentPosition: function() { return __invokeAsync('location.read', { mode: 'current' }); } + getCurrentPosition: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'current' }, args || {})); } }; globalThis.apple.calendar = { @@ -353,6 +448,17 @@ enum RuntimeJavaScript { open: function(args) { return __invokeAsync('settings.ui.open', args || {}); } }; + // fs.move / fs.copy refuse an existing destination without an explicit + // overwrite, so the Node-style aliases need a way to pass one through. + function __fsDestinationArgs(from, to, options) { + const args = { from: String(from), to: String(to) }; + if (options && typeof options === 'object') { + if (options.overwrite !== undefined) args.overwrite = !!options.overwrite; + if (options.recursive !== undefined) args.recursive = !!options.recursive; + } + return args; + } + globalThis.fs = { promises: { readFile: function(path, options) { @@ -378,8 +484,12 @@ enum RuntimeJavaScript { rm: function(path, options) { return __invokeAsync('fs.delete', { path: String(path), recursive: !!(options && options.recursive) }); }, - rename: function(from, to) { return __invokeAsync('fs.move', { from: String(from), to: String(to) }); }, - copyFile: function(from, to) { return __invokeAsync('fs.copy', { from: String(from), to: String(to) }); } + rename: function(from, to, options) { + return __invokeAsync('fs.move', __fsDestinationArgs(from, to, options)); + }, + copyFile: function(from, to, options) { + return __invokeAsync('fs.copy', __fsDestinationArgs(from, to, options)); + } } }; diff --git a/Sources/CodeMode/Security/CapabilityGrant.swift b/Sources/CodeMode/Security/CapabilityGrant.swift new file mode 100644 index 0000000..d9315e9 --- /dev/null +++ b/Sources/CodeMode/Security/CapabilityGrant.swift @@ -0,0 +1,85 @@ +import Foundation + +/// The host-owned ceiling on what an execution may reach. +/// +/// `JavaScriptExecutionRequest.allowedCapabilities` / `allowedCapabilityKeys` are +/// *model-authored* — they travel in the advertised tool schema, so a script +/// declares them for itself. They are a least-privilege declaration of intent and +/// a useful audit signal, but they are not a sandbox: on their own, a script that +/// asks for `keychain.read` gets it. +/// +/// `CapabilityGrant` is the half the host owns. It is configured on +/// `CodeModeConfiguration` and intersected with the request before the invocation +/// context is built, so the effective set is always +/// `requested ∩ granted`. Anything the model asks for that the grant withholds +/// fails with `CAPABILITY_DENIED` and is reported to the model as *not* +/// repairable by retrying with a wider `allowedCapabilities`. +public struct CapabilityGrant: Sendable, Equatable { + /// Built-in capabilities the host permits, or `nil` for "no host ceiling". + public var capabilities: Set? + /// Custom provider capability keys the host permits, or `nil` for "no host ceiling". + public var capabilityKeys: Set? + + public init(capabilities: Set?, capabilityKeys: Set?) { + self.capabilities = capabilities + self.capabilityKeys = capabilityKeys + } + + /// No host ceiling: whatever the request declares is what the script gets. + /// + /// This is the default for source compatibility, and it is the right choice + /// only when the host has already vetted the request itself. Hosts that pipe + /// model-authored tool JSON straight into `executeJavaScript` should set an + /// explicit grant. + public static let unrestricted = CapabilityGrant(capabilities: nil, capabilityKeys: nil) + + /// Grants nothing; every capability request is withheld. + public static let none = CapabilityGrant(capabilities: [], capabilityKeys: []) + + /// Grants exactly the listed built-in capabilities and provider keys. + public static func only( + _ capabilities: Set, + capabilityKeys: Set = [] + ) -> CapabilityGrant { + CapabilityGrant(capabilities: capabilities, capabilityKeys: capabilityKeys) + } + + /// Whether the grant admits this capability. Used to keep what the host + /// advertises consistent with what it will actually allow. + public func permits(_ capability: CapabilityID) -> Bool { + capabilities?.contains(capability) ?? true + } + + public func permits(key: CodeModeCapabilityKey) -> Bool { + capabilityKeys?.contains(key) ?? true + } + + /// Applies the ceiling to a model-authored request. + /// + /// Returns the effective sets alongside everything that was asked for and + /// withheld, so the caller can tell the model why a retry will not help. + public func resolve( + requestedCapabilities: Set, + requestedCapabilityKeys: Set + ) -> Resolution { + let effectiveCapabilities = capabilities.map { requestedCapabilities.intersection($0) } ?? requestedCapabilities + let effectiveKeys = capabilityKeys.map { requestedCapabilityKeys.intersection($0) } ?? requestedCapabilityKeys + + return Resolution( + capabilities: effectiveCapabilities, + capabilityKeys: effectiveKeys, + withheld: ( + requestedCapabilities.subtracting(effectiveCapabilities).map(\.rawValue) + + requestedCapabilityKeys.subtracting(effectiveKeys).map(\.rawValue) + ).sorted() + ) + } + + public struct Resolution: Sendable, Equatable { + public var capabilities: Set + public var capabilityKeys: Set + /// Identifiers the request declared and the grant refused, sorted. Both + /// namespaces together, because every consumer reports them as one list. + public var withheld: [String] + } +} diff --git a/Sources/CodeMode/Security/NetworkAccessPolicy.swift b/Sources/CodeMode/Security/NetworkAccessPolicy.swift index 3dd10f8..6bb8e80 100644 --- a/Sources/CodeMode/Security/NetworkAccessPolicy.swift +++ b/Sources/CodeMode/Security/NetworkAccessPolicy.swift @@ -5,8 +5,11 @@ import Foundation /// /// The default (`.standard`) blocks loopback, private-range, link-local, and /// other non-public destinations so agent-authored JavaScript cannot reach -/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or hosts on -/// the local network, and caps buffered response bodies at 10 MB. +/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or names that +/// resolve onto the local network, and caps buffered response bodies at 10 MB. +/// "Names that resolve onto the local network" covers `localhost`, +/// `*.local`/`*.localhost`/`*.internal`, and bare single-label hosts such as +/// `router` or `intranet`, which reach the LAN through the DHCP search domain. /// /// Host matching is by URL host only; DNS resolution is not performed, so a /// public hostname that resolves to a private address is not detected. Hosts @@ -31,28 +34,64 @@ public struct NetworkAccessPolicy: Sendable, Equatable { /// that exceed this are cancelled mid-transfer. public var maxResponseBytes: Int + /// Whether script-supplied credential headers reach the wire. + /// + /// `Cookie`, `Authorization`, and the `Proxy-*` pair are how a request + /// carries authority. Script-authored code should not be minting them + /// against arbitrary origins, so they are refused by default. Hosts whose + /// scripts legitimately call an authenticated API — a bearer token the + /// script fetched from the keychain, say — set this to true. + public var allowsCredentialHeaders: Bool + + /// Request headers refused when `allowsCredentialHeaders` is false. + /// Matched case-insensitively; `proxy-` is matched as a prefix. + static let credentialHeaderNames: Set = [ + "cookie", + "cookie2", + "authorization", + ] + public init( allowedHosts: [String]? = nil, blockedHosts: [String] = [], blocksPrivateNetworks: Bool = true, - maxResponseBytes: Int = 10_485_760 + maxResponseBytes: Int = 10_485_760, + allowsCredentialHeaders: Bool = false ) { self.allowedHosts = allowedHosts self.blockedHosts = blockedHosts self.blocksPrivateNetworks = blocksPrivateNetworks self.maxResponseBytes = maxResponseBytes + self.allowsCredentialHeaders = allowsCredentialHeaders } - /// Secure default: private networks blocked, 10 MB response cap. + /// Secure default: private networks blocked, 10 MB response cap, no + /// script-supplied credential headers. public static let standard = NetworkAccessPolicy() - /// No destination restrictions and no response size cap. Matches the - /// pre-policy behavior of `network.fetch`; opt in deliberately. + /// No destination restrictions, no response size cap, and script-supplied + /// credential headers permitted. Matches the pre-policy behavior of + /// `network.fetch`; opt in deliberately. public static let permissive = NetworkAccessPolicy( blocksPrivateNetworks: false, - maxResponseBytes: .max + maxResponseBytes: .max, + allowsCredentialHeaders: true ) + /// Returns a reason the header is refused, or nil when it may be sent. + func headerViolationReason(for name: String) -> String? { + guard allowsCredentialHeaders == false else { + return nil + } + + let normalized = name.lowercased() + guard Self.credentialHeaderNames.contains(normalized) || normalized.hasPrefix("proxy-") else { + return nil + } + + return "the \"\(name)\" request header carries authority and is refused by the host app's network access policy" + } + /// Returns a human-readable reason the URL is refused, or nil when allowed. func violationReason(for url: URL) -> String? { guard let rawHost = url.host, rawHost.isEmpty == false else { @@ -119,6 +158,14 @@ public struct NetworkAccessPolicy: Sendable, Equatable { return isPrivateOrLocal(ipv6: ipv6) } + // A bare single-label name — `router`, `intranet`, `nas`, `wpad` — is not + // a public destination. It resolves through the DHCP-supplied search + // domain, so it lands on the local network exactly like `*.local` does, + // and it previously fell straight through to "allowed". + if host.contains(".") == false { + return true + } + return false } diff --git a/Sources/CodeMode/Security/PathPolicy.swift b/Sources/CodeMode/Security/PathPolicy.swift index 6ba31f8..35d0154 100644 --- a/Sources/CodeMode/Security/PathPolicy.swift +++ b/Sources/CodeMode/Security/PathPolicy.swift @@ -23,6 +23,26 @@ public struct PathPolicyConfig: Sendable { public protocol PathPolicy: Sendable { func resolve(path: String) throws -> URL + + /// The roots this policy admits. + /// + /// `resolve` accepts a root itself — `documents:` resolves to the documents + /// root — so destructive filesystem operations need to recognize one and + /// refuse it. Deliberately has no default: an earlier default of `[]` meant a + /// host with its own policy silently got none of the root protection on + /// `fs.move`, `fs.copy`, and `fs.delete`. A policy that genuinely has no + /// roots to protect can return `[]` explicitly. + var allowedRoots: [URL] { get } +} + +public extension PathPolicy { + /// True when `url` *is* one of the allowed roots rather than something inside one. + func isAllowedRoot(_ url: URL) -> Bool { + let candidate = url.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path + return allowedRoots.contains { root in + root.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path == candidate + } + } } public struct DefaultPathPolicy: PathPolicy { @@ -32,6 +52,10 @@ public struct DefaultPathPolicy: PathPolicy { self.config = config } + public var allowedRoots: [URL] { + [config.tmpRoot, config.cachesRoot, config.documentsRoot, config.appGroupRoot].compactMap { $0 } + } + public func resolve(path: String) throws -> URL { let cleaned = path.trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.isEmpty == false else { @@ -98,15 +122,54 @@ public struct DefaultPathPolicy: PathPolicy { var missingComponents: [String] = [] let fileManager = FileManager.default - while fileManager.fileExists(atPath: existingAncestor.path) == false, + // `fileExists` follows symlinks, so a symlink pointing at a *nonexistent* + // out-of-root target reported "missing" and was treated as a component to + // re-append rather than a link to resolve — admitting it. Checking for the + // link itself keeps a dangling symlink in the resolution path, where + // `resolvingSymlinksInPath` sends containment to the real target. + while Self.isMissingComponent(existingAncestor, fileManager: fileManager), existingAncestor.path != existingAncestor.deletingLastPathComponent().path { missingComponents.insert(existingAncestor.lastPathComponent, at: 0) existingAncestor.deleteLastPathComponent() } - let resolvedAncestor = existingAncestor.resolvingSymlinksInPath().standardizedFileURL + let resolvedAncestor = Self.followingDanglingSymlink( + existingAncestor.resolvingSymlinksInPath().standardizedFileURL, + fileManager: fileManager + ) return missingComponents.reduce(resolvedAncestor) { partial, component in partial.appendingPathComponent(component) }.standardizedFileURL } + + /// True when nothing exists at this path — not even a broken symlink. + /// `FileManager.fileExists` follows links and so answers false for one. + private static func isMissingComponent(_ url: URL, fileManager: FileManager) -> Bool { + if fileManager.fileExists(atPath: url.path) { + return false + } + // `attributesOfItem` does not traverse the final symlink, so a dangling + // link still has attributes here. + return (try? fileManager.attributesOfItem(atPath: url.path)) == nil + } + + /// Resolves a symlink whose target does not exist. + /// + /// `URL.resolvingSymlinksInPath()` only rewrites links it can follow, so a + /// link to a nonexistent out-of-root target came back unchanged and passed + /// containment. Containment must be judged on where the link *points*. + private static func followingDanglingSymlink(_ url: URL, fileManager: FileManager) -> URL { + var current = url + // Bounded: a symlink cycle would otherwise spin here. + for _ in 0..<16 { + guard let destination = try? fileManager.destinationOfSymbolicLink(atPath: current.path) else { + return current + } + let next = destination.hasPrefix("/") + ? URL(fileURLWithPath: destination) + : current.deletingLastPathComponent().appendingPathComponent(destination) + current = next.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL + } + return current + } } diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index 27f4451..27fd3f1 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -45,6 +45,18 @@ import HomeKit #endif public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable { + /// How long a `request(for:)` waits on the system's TCC dialog before giving + /// up and re-reading the status. This is a human-latency budget, and it is + /// short: a user who takes longer than this leaves the caller reading the + /// pre-prompt status. + static let permissionPromptTimeoutSeconds: TimeInterval = 10 + + /// Every TCC request in this type waits that same budget, so the timeout is + /// not repeated at six call sites where it could drift apart. + private static func awaitingPrompt(_ request: (@escaping @Sendable () -> Void) -> Void) { + CompletionWait.completion(timeout: permissionPromptTimeoutSeconds, request) + } + public init() {} public func status(for permission: PermissionKind) -> PermissionStatus { @@ -111,8 +123,15 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func locationStatus() -> PermissionStatus { #if canImport(CoreLocation) - let manager = CLLocationManager() - switch manager.authorizationStatus { + return Self.mapLocationAuthorization(CLLocationManager().authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(CoreLocation) + static func mapLocationAuthorization(_ status: CLAuthorizationStatus) -> PermissionStatus { + switch status { case .authorizedWhenInUse, .authorizedAlways: return .granted case .denied: @@ -124,10 +143,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable @unknown default: return .unavailable } - #else - return .unavailable - #endif } + #endif private func contactsStatus() -> PermissionStatus { #if canImport(Contacts) @@ -320,9 +337,47 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable #endif } + /// Reads HomeKit authorization *without* constructing a manager. + /// + /// Constructing `HMHomeManager` is itself what triggers the HomeKit TCC + /// prompt, so the previous implementation put a user-facing dialog inside a + /// status check — and `requestHomeKitPermission` called it twice, then built a + /// third manager. A status query now answers from the shared manager if one + /// already exists and otherwise fails closed with `.notDetermined`, which + /// routes the caller through the request path where a prompt belongs. private func homeKitStatus() -> PermissionStatus { #if canImport(HomeKit) - let status = HMHomeManager().authorizationStatus + guard let manager = Self.existingHomeManager() else { + return .notDetermined + } + return Self.mapHomeKitAuthorization(manager.authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(HomeKit) + /// Process-wide so repeated permission work reuses one manager instead of + /// re-triggering the prompt. + private static let homeManagerBox = LockedBox(nil) + + private static func existingHomeManager() -> HMHomeManager? { + homeManagerBox.get() + } + + /// Creates the shared manager if needed. This is the call that prompts. + private static func makeHomeManager(delegate: HMHomeManagerDelegate) -> HMHomeManager { + if let existing = homeManagerBox.get() { + existing.delegate = delegate + return existing + } + let manager = HMHomeManager() + manager.delegate = delegate + homeManagerBox.set(manager) + return manager + } + + private static func mapHomeKitAuthorization(_ status: HMHomeManagerAuthorizationStatus) -> PermissionStatus { if status.contains(.authorized) { return .granted } @@ -333,10 +388,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable return .notDetermined } return .unavailable - #else - return .unavailable - #endif } + #endif private func speechRecognitionStatus() -> PermissionStatus { #if canImport(Speech) @@ -419,24 +472,41 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestLocationPermission() -> PermissionStatus { #if canImport(CoreLocation) && os(iOS) - let manager = CLLocationManager() let delegate = LocationPermissionDelegate() - manager.delegate = delegate if Thread.isMainThread { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() + // Cannot block main for the callback; the caller sees the pre-prompt + // status and a later call observes the resolved one. return locationStatus() } - // async, not sync: execution runs on a background queue, and the host may be - // blocking the main thread waiting on the result — main.sync would deadlock. - // If main never runs the request, the delegate wait below times out instead. + // `CLLocationManager` delivers delegate callbacks on the run loop of the + // thread that *created* it, and the bridge's GCD workers have none — so a + // manager constructed here never called back, the wait below always burned + // its full 10s, and the result was reported from a status re-read that + // races the TCC write. Creating the manager, assigning the delegate, and + // requesting all happen on main. + // + // async, not sync: the host may be blocking main waiting on our result, so + // main.sync would deadlock. If main never runs this, the wait times out. DispatchQueue.main.async { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() } - _ = delegate.wait(timeout: 10) - return locationStatus() + guard delegate.wait(timeout: 10) == .success else { + return locationStatus() + } + + // Prefer the status the delegate callback carried over a fresh read, + // which can still observe the pre-prompt value. + return delegate.observedStatus().map(Self.mapLocationAuthorization) ?? locationStatus() #else return .unavailable #endif @@ -444,12 +514,10 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestContactsPermission() -> PermissionStatus { #if canImport(Contacts) - let semaphore = DispatchSemaphore(value: 0) let store = CNContactStore() - store.requestAccess(for: .contacts) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + store.requestAccess(for: .contacts) { _, _ in complete() } } - _ = semaphore.wait(timeout: .now() + 10) return contactsStatus() #else return .unavailable @@ -459,19 +527,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarReadStatus() #else return .unavailable @@ -481,19 +543,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarWritePermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestWriteOnlyAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestWriteOnlyAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarWriteStatus() #else return .unavailable @@ -503,19 +559,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestRemindersPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToReminders { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .reminder) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToReminders { _, _ in complete() } + } else { + store.requestAccess(to: .reminder) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return remindersStatus() #else return .unavailable @@ -524,19 +574,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestPhotoLibraryPermission() -> PermissionStatus { #if canImport(Photos) - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 14.0, macOS 11.0, *) { - PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in - semaphore.signal() - } - } else { - PHPhotoLibrary.requestAuthorization { _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 14.0, macOS 11.0, *) { + PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in complete() } + } else { + PHPhotoLibrary.requestAuthorization { _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return photoLibraryStatus() #else return .unavailable @@ -545,13 +589,11 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestNotificationsPermission() -> PermissionStatus { #if canImport(UserNotifications) - let semaphore = DispatchSemaphore(value: 0) - - UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in + complete() + } } - - _ = semaphore.wait(timeout: .now() + 10) return notificationsStatus() #else return .unavailable @@ -560,15 +602,17 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestHomeKitPermission() -> PermissionStatus { #if canImport(HomeKit) - if homeKitStatus() != .notDetermined { - return homeKitStatus() + // One status read, not two, and one manager, not three: each construction + // is a potential prompt. + let current = homeKitStatus() + if current != .notDetermined, Self.existingHomeManager() != nil { + return current } let delegate = HomeKitPermissionDelegate() - let manager = HMHomeManager() - manager.delegate = delegate - _ = delegate.wait(timeout: 10) - return homeKitStatus() + let manager = Self.makeHomeManager(delegate: delegate) + _ = delegate.wait(timeout: Self.permissionPromptTimeoutSeconds) + return Self.mapHomeKitAuthorization(manager.authorizationStatus) #else return .unavailable #endif @@ -667,13 +711,40 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable } #if canImport(CoreLocation) && os(iOS) -private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate { +/// `@unchecked Sendable` because it is genuinely thread-safe and has to cross an +/// isolation boundary: the manager is created inside a `DispatchQueue.main.async` +/// block — main-actor-isolated on iOS — while `wait`/`observedStatus` are called +/// from the requesting thread. All mutable state is lock-guarded and the handoff +/// is a semaphore. +private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate, @unchecked Sendable { private let semaphore = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var status: CLAuthorizationStatus? + /// The manager is created inside a `DispatchQueue.main.async` block and would + /// otherwise be released as soon as that block returns — before the callback + /// it exists to receive. `CLLocationManager` does not retain its delegate, so + /// the ownership runs the other way here. + private var manager: CLLocationManager? + + func hold(_ manager: CLLocationManager) { + lock.lock() + self.manager = manager + lock.unlock() + } func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + lock.lock() + status = manager.authorizationStatus + lock.unlock() semaphore.signal() } + func observedStatus() -> CLAuthorizationStatus? { + lock.lock() + defer { lock.unlock() } + return status + } + func wait(timeout: TimeInterval) -> DispatchTimeoutResult { semaphore.wait(timeout: .now() + timeout) } diff --git a/Sources/CodeMode/Support/CodeModeDate.swift b/Sources/CodeMode/Support/CodeModeDate.swift new file mode 100644 index 0000000..e3c89d7 --- /dev/null +++ b/Sources/CodeMode/Support/CodeModeDate.swift @@ -0,0 +1,58 @@ +import Foundation + +/// The single ISO 8601 parser for every bridge. +/// +/// Two spellings were previously in circulation: Health and Alarm tried +/// `.withFractionalSeconds` as a fallback, while EventKit, Notifications, and the +/// system-UI validators used a bare `ISO8601DateFormatter`. So +/// `2026-07-24T10:00:00.000Z` — a spelling models emit constantly, and what +/// `Date.toISOString()` produces in JavaScript — parsed in some bridges and not +/// others. +enum CodeModeDate { + /// Parses an ISO 8601 timestamp with or without fractional seconds. + static func parse(_ text: String?) -> Date? { + guard let text else { + return nil + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.isEmpty == false else { + return nil + } + + let formatter = ISO8601DateFormatter() + if let date = formatter.date(from: trimmed) { + return date + } + + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.date(from: trimmed) + } + + /// Parses a timestamp that the caller requires, throwing a repairable error + /// rather than substituting a default. + /// + /// Silently falling back to "now" (or "now + 14 days") gave the script a + /// plausible but wrong window with no diagnostic — the worst failure mode + /// available, because the model has no way to notice it happened. + static func require(_ text: String?, argument: String, capability: String) throws -> Date { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + throw BridgeError.invalidArguments("\(capability) requires '\(argument)'") + } + guard let date = parse(text) else { + throw BridgeError.invalidArguments( + "\(capability) could not parse '\(argument)' as an ISO 8601 timestamp: \"\(text)\". Use a form like 2026-07-24T10:00:00Z." + ) + } + return date + } + + /// Parses an optional timestamp, throwing when a value is present but + /// unparseable. A missing value returns nil and the caller's default applies. + static func optional(_ text: String?, argument: String, capability: String) throws -> Date? { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + return nil + } + return try require(text, argument: argument, capability: capability) + } +} diff --git a/Sources/CodeMode/Support/CompletionWait.swift b/Sources/CodeMode/Support/CompletionWait.swift new file mode 100644 index 0000000..38f54f4 --- /dev/null +++ b/Sources/CodeMode/Support/CompletionWait.swift @@ -0,0 +1,55 @@ +import Foundation + +/// Bridges a callback-style system API into the bridges' synchronous world. +/// +/// The pattern this replaces — a `var` written from the framework's completion +/// queue, read after `_ = semaphore.wait(...)` with the timeout result +/// discarded — is a data race on timeout: the late callback writes while the +/// caller reads, and the caller cannot tell "timed out" from "no results". Every +/// wait here either produces the value the callback delivered or throws +/// `BridgeError.timeout`; the box stays alive for a late callback to write into +/// harmlessly. +enum CompletionWait { + /// Runs `operation`, which must call its `complete` argument exactly once, + /// and returns the delivered value. + /// + /// - Throws: `BridgeError.timeout` if `complete` is not called in time. + static func value( + timeout: TimeInterval, + operationName: String, + _ operation: (@escaping @Sendable (Value) -> Void) -> Void + ) throws -> Value { + let box = LockedBox(nil) + let semaphore = DispatchSemaphore(value: 0) + + operation { value in + box.set(value) + semaphore.signal() + } + + guard semaphore.wait(timeout: .now() + timeout) == .success else { + throw BridgeError.timeout(milliseconds: Int(timeout * 1_000)) + } + + guard let value = box.get() else { + throw BridgeError.nativeFailure("\(operationName) completed without a result") + } + + return value + } + + /// Waits for a completion that carries no value, for APIs whose result is + /// read back from the store afterwards. + /// + /// - Returns: true when the callback arrived, false on timeout. Callers that + /// must distinguish the two should use `value(timeout:operationName:_:)`. + @discardableResult + static func completion( + timeout: TimeInterval, + _ operation: (@escaping @Sendable () -> Void) -> Void + ) -> Bool { + let semaphore = DispatchSemaphore(value: 0) + operation { semaphore.signal() } + return semaphore.wait(timeout: .now() + timeout) == .success + } +} diff --git a/Sources/CodeMode/Support/JSONValue.swift b/Sources/CodeMode/Support/JSONValue.swift index d96eec3..0442e13 100644 --- a/Sources/CodeMode/Support/JSONValue.swift +++ b/Sources/CodeMode/Support/JSONValue.swift @@ -131,13 +131,34 @@ public extension JSONValue { return nil } + /// Integer view of a JSON number, or `nil` when the value cannot be + /// represented. + /// + /// `Int.init(_: Double)` *traps* on NaN, infinity, and anything outside + /// `Int`'s range, and every value here came from adversarial JSON — a model + /// tool call or a script's `fetch(url, { timeoutMs: 1e300 })`. The conversion + /// has to be total or a single number kills the host process. var intValue: Int? { if case let .number(value) = self { - return Int(value) + return JSONValue.exactInt(from: value) } return nil } + /// Total `Double` → `Int` conversion. Rejects non-finite values and anything + /// the conversion cannot represent; truncates toward zero otherwise, matching + /// the previous behavior for in-range input. + static func exactInt(from value: Double) -> Int? { + guard value.isFinite else { return nil } + let truncated = value.rounded(.towardZero) + guard truncated >= -9_223_372_036_854_775_808.0, + truncated < 9_223_372_036_854_775_808.0 + else { + return nil + } + return Int(truncated) + } + var doubleValue: Double? { if case let .number(value) = self { return value diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift index c3e95b3..f4d8633 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift @@ -96,7 +96,6 @@ extension UIKitSystemUIPresenter { public func presentNewCalendarEvent(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let formatter = ISO8601DateFormatter() let token = UUID() return try runUIOperation(timeoutMs: timeoutMs, onTimeout: { self.releaseCoordinator(token) }) { complete in @@ -108,9 +107,9 @@ extension UIKitSystemUIPresenter { event.notes = arguments.string("notes") event.location = arguments.string("location") - let startDate = arguments.string("start").flatMap { formatter.date(from: $0) } ?? Date() + let startDate = CodeModeDate.parse(arguments.string("start")) ?? Date() event.startDate = startDate - event.endDate = arguments.string("end").flatMap { formatter.date(from: $0) } + event.endDate = CodeModeDate.parse(arguments.string("end")) ?? Calendar.current.date(byAdding: .hour, value: 1, to: startDate) ?? startDate event.calendar = store.defaultCalendarForNewEvents diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift index 8fd4a7c..e3a5f8b 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift @@ -365,7 +365,12 @@ extension UIKitSystemUIPresenter { public func presentWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + // The bridge parses the URL first, but UIKitSystemUIPresenter is public + // API and a host may call it directly — a force unwrap here crashes the + // app on a malformed string. + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("web.ui.present requires an absolute 'url'") + } let token = UUID() #if os(iOS) @@ -402,7 +407,9 @@ extension UIKitSystemUIPresenter { public func authenticateWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("auth.ui.webAuthenticate requires an absolute 'url'") + } let callbackURLScheme = arguments.string("callbackURLScheme") let token = UUID() @@ -446,7 +453,11 @@ extension UIKitSystemUIPresenter { } } session.presentationContextProvider = coordinator - session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? false + // Ephemeral by default: a script-initiated OAuth flow inside the + // user's live Safari session starts out already signed in as the + // user and hands the callback URL — with its code or token — to + // the script. Opting out is a deliberate host/script choice. + session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? true coordinator.session = session self.retainCoordinator(coordinator, token: token) diff --git a/Sources/CodeModeAuthoring/README.md b/Sources/CodeModeAuthoring/README.md index 57069fa..7a992e9 100644 --- a/Sources/CodeModeAuthoring/README.md +++ b/Sources/CodeModeAuthoring/README.md @@ -69,6 +69,33 @@ let call = try await tools.executeJavaScript( ) ``` +The two allowlists are strictly disjoint: a built-in capability ID listed in +`allowedCapabilityKeys` is ignored, and a provider key listed in +`allowedCapabilities` will not decode. Both fields are model-authored, so treat +them as a declaration of intent — set `CodeModeConfiguration.capabilityGrant` for +the host-owned ceiling that actually enforces access. + Macro v1 maps one type to one JavaScript function. `Arguments` must be a nested struct, and no-arg tools use an empty `Arguments` struct. `call(arguments:)` must be `throws` or `async throws`, and it can return `Void` or a nested `Result` struct. -Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, and optional forms. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. +Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, optional forms, and `CodeModeStringEnum` types. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. + +An enum-typed argument advertises its accepted values, so the catalog, the +generated TypeScript declaration, and argument validation all agree: + +```swift +enum Priority: String, CodeModeStringEnum { + case low, normal, high +} + +struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: Priority? +} +``` + +Providers do not have to be known at construction. `CodeModeAgentTools.register(provider:)` +adds them later — search advertises them immediately and the JavaScript bindings +are installed for the next execution — which is what a host needs when its domain +APIs depend on runtime state such as a signed-in account. diff --git a/Sources/CodeModeEvaluation/EvalModels.swift b/Sources/CodeModeEvaluation/EvalModels.swift index b8ae289..5511a4b 100644 --- a/Sources/CodeModeEvaluation/EvalModels.swift +++ b/Sources/CodeModeEvaluation/EvalModels.swift @@ -54,15 +54,39 @@ public struct CodeModeEvalExecuteStep: Codable, Sendable, Equatable { public var code: String public var allowedCapabilities: [CapabilityID] public var timeoutMs: Int? + /// Whether this step is *supposed* to fail. + /// + /// Repair scenarios deliberately call a helper wrong and then fix it from the + /// structured error, so their first step failing is the point. Every other + /// step failing should stop the run — otherwise a later step masks it and the + /// scenario passes on a broken transcript. + public var expectsFailure: Bool public init( code: String, allowedCapabilities: [CapabilityID] = [], - timeoutMs: Int? = nil + timeoutMs: Int? = nil, + expectsFailure: Bool = false ) { self.code = code self.allowedCapabilities = allowedCapabilities self.timeoutMs = timeoutMs + self.expectsFailure = expectsFailure + } + + private enum CodingKeys: String, CodingKey { + case code + case allowedCapabilities + case timeoutMs + case expectsFailure + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.code = try container.decode(String.self, forKey: .code) + self.allowedCapabilities = try container.decodeIfPresent([CapabilityID].self, forKey: .allowedCapabilities) ?? [] + self.timeoutMs = try container.decodeIfPresent(Int.self, forKey: .timeoutMs) + self.expectsFailure = try container.decodeIfPresent(Bool.self, forKey: .expectsFailure) ?? false } } diff --git a/Sources/CodeModeEvaluation/EvalRunner.swift b/Sources/CodeModeEvaluation/EvalRunner.swift index 30fba8a..2204a7f 100644 --- a/Sources/CodeModeEvaluation/EvalRunner.swift +++ b/Sources/CodeModeEvaluation/EvalRunner.swift @@ -58,6 +58,11 @@ public final class CodeModeEvalRunner: Sendable { searchDiagnostics = response.diagnostics } + // Logs and diagnostics accumulate across steps and an intermediate + // error stops the run. Overwriting each field per step meant + // validation only ever saw the last step: a scenario could pass while + // step 1 failed in a way step 2 masked, and step 1's logs — the + // evidence for why — were discarded. for executeStep in executeSteps(for: scenario) { toolCalls.append( CodeModeEvalToolCall( @@ -74,15 +79,25 @@ public final class CodeModeEvalRunner: Sendable { ) ) let observed = await observe(call) + // The graded value is the last step's output, and the graded error + // is the last step's error — so a successful repair step clears the + // deliberate failure that preceded it. executionOutput = observed.output - executionLogs = observed.logs - executionDiagnostics = observed.diagnostics observedError = observed.error + executionLogs.append(contentsOf: observed.logs) + executionDiagnostics.append(contentsOf: observed.diagnostics) + + // A repair scenario's first step is meant to fail and the next step + // fixes it. Anything else stops here rather than letting a later + // step mask the failure. + if observed.error != nil, executeStep.expectsFailure == false { + break + } } } catch let error as CodeModeToolError { observedError = error - executionLogs = error.logs - executionDiagnostics = error.diagnostics + executionLogs.append(contentsOf: error.logs) + executionDiagnostics.append(contentsOf: error.diagnostics) } catch { failures.append("Unexpected runner failure: \(error.localizedDescription)") } diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index 091e965..44359cb 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -16,6 +16,9 @@ public enum CodeModeEvalScenarios { filesystemCapabilityDenied, executionConsoleLogs, executionTimeout, + executionUnsettleablePromise, + executionTimerBackoff, + filesystemWholeJobInOneScript, reminderCatalogDiscovery, catalogFileSystemReadShape, catalogConsoleDiagnostics, @@ -319,10 +322,16 @@ public enum CodeModeEvalScenarios { return api.byJSName["apple.fs.read"]; } """, - executeCode: """ - return await apple.fs.read({ encoding: "utf8" }); - """, executeSteps: [ + // Deliberately wrong, so the next step can repair from the structured + // error. Marked so the runner does not treat it as the run failing. + CodeModeEvalExecuteStep( + code: """ + return await apple.fs.read({ encoding: "utf8" }); + """, + allowedCapabilities: [.fsRead], + expectsFailure: true + ), CodeModeEvalExecuteStep( code: """ const result = await apple.fs.read({ path: "tmp:repair.txt", encoding: "utf8" }); @@ -485,10 +494,10 @@ public enum CodeModeEvalScenarios { ) ) - public static let executionTimeout = CodeModeEvalScenario( - id: "execution.timeout", - title: "Unresolved promises time out", - task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + public static let executionUnsettleablePromise = CodeModeEvalScenario( + id: "execution.unsettleable-promise", + title: "Unresolvable promises are reported, not waited out", + task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured error.", executeCode: """ await new Promise(() => {}); return { never: true }; @@ -499,10 +508,93 @@ public enum CodeModeEvalScenarios { toolOrder: [.executeJavaScript], exactAllowedCapabilities: [], requiredExecuteCodeFragments: ["new Promise"], + expectedErrorCode: "JS_RUNTIME_ERROR" + ) + ) + + public static let executionTimeout = CodeModeEvalScenario( + id: "execution.timeout", + title: "CPU-bound scripts hit the execution timeout", + task: "Run a CPU-bound infinite loop. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + executeCode: """ + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 200, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["while"], expectedErrorCode: "EXECUTION_TIMEOUT" ) ) + public static let filesystemWholeJobInOneScript = CodeModeEvalScenario( + id: "fs.whole-job-one-script", + title: "A multi-step job runs as one script", + task: "Read every .json receipt in documents:receipts, total the amounts for trip 'lisbon', and return { total, count }. Do the whole job in a single executeJavaScript call — list, read, filter, and sum inside the script — and return only the totals, not the receipts.", + searchCode: """ + async () => { + return api.references + .filter(ref => ["fs.list", "fs.read"].includes(ref.capability)) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """, + executeCode: """ + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + let count = 0; + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + count += 1; + } + return { total, count }; + """, + allowedCapabilities: [.fsList, .fsRead], + seedFiles: [ + CodeModeEvalSeedFile(path: "documents:receipts/a.json", text: #"{"trip":"lisbon","amount":12}"#), + CodeModeEvalSeedFile(path: "documents:receipts/b.json", text: #"{"trip":"porto","amount":99}"#), + CodeModeEvalSeedFile(path: "documents:receipts/c.json", text: #"{"trip":"lisbon","amount":30}"#), + ], + expectation: CodeModeEvalExpectation( + // One search, one execute. A transcript that splits the list/read/sum + // across several executions fails here — that is the whole point of + // the scenario, and it only has teeth against real LLM transcripts. + toolOrder: [.searchJavaScriptAPI, .executeJavaScript], + exactAllowedCapabilities: [.fsList, .fsRead], + forbiddenCapabilities: [.fsWrite, .fsDelete, .fsMove], + requiredExecuteCodeFragments: ["apple.fs.list", "apple.fs.read"], + expectedOutput: .object(["total": .number(42), "count": .number(2)]) + ) + ) + + public static let executionTimerBackoff = CodeModeEvalScenario( + id: "execution.timer-backoff", + title: "setTimeout honours its delay", + task: "Use setTimeout to wait before returning, and cancel a second timer with clearTimeout so its callback never runs.", + executeCode: """ + const marks = []; + const cancelled = setTimeout(() => { marks.push('cancelled'); }, 5); + clearTimeout(cancelled); + await new Promise(resolve => setTimeout(resolve, 20)); + marks.push('resumed'); + return { marks }; + """, + allowedCapabilities: [], + timeoutMs: 2_000, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["setTimeout"], + expectedOutput: .object(["marks": .array([.string("resumed")])]) + ) + ) + public static let reminderCatalogDiscovery = CodeModeEvalScenario( id: "catalog.reminder-create", title: "Reminder helper discovery", diff --git a/Sources/CodeModeMacros/CodeModeMacro.swift b/Sources/CodeModeMacros/CodeModeMacro.swift index 72f6086..6d19b3f 100644 --- a/Sources/CodeModeMacros/CodeModeMacro.swift +++ b/Sources/CodeModeMacros/CodeModeMacro.swift @@ -116,12 +116,13 @@ public struct CodeModeMacro: ExtensionMacro { jsPath: \(literal(path)), title: \(literal(title(for: path))), summary: \(literal(summary)), - tags: [\(literal(parentPath(for: path)))], - example: "await \(path)({})", + tags: \(tagsSource(for: path)), + example: \(literal(example(for: path, fields: argumentFields.fields))), requiredArguments: [\(argumentFields.fields.filter { !$0.optional }.map { literal($0.name) }.joined(separator: ", "))], optionalArguments: [\(argumentFields.fields.filter(\.optional).map { literal($0.name) }.joined(separator: ", "))], argumentTypes: \(argumentTypesSource(for: argumentFields.fields)), argumentHints: \(argumentHintsSource(for: argumentFields.fields)), + argumentConstraints: \(argumentConstraintsSource(for: argumentFields.fields)), resultSummary: \(literal(returnsVoid ? "null" : resultInfo?.summary ?? "JSON value")), handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -275,7 +276,8 @@ public struct CodeModeMacro: ExtensionMacro { swiftType: typeInfo.swiftType, argumentType: typeInfo.argumentType, optional: typeInfo.optional, - description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first } + description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first }, + isStringEnum: typeInfo.isStringEnum ) ) } @@ -471,6 +473,60 @@ public struct CodeModeMacro: ExtensionMacro { return segments.dropLast().joined(separator: ".") } + /// Every path segment, so a helper is findable by its namespace *and* its + /// domain — the old single parent-path tag made `myapp.tasks.complete` + /// discoverable only by searching for "myapp.tasks". + private static func tagsSource(for path: String) -> String { + let segments = path.split(separator: ".").map(String.init) + var tags: [String] = [] + for segment in segments where tags.contains(segment) == false { + tags.append(segment) + } + if segments.count > 1 { + tags.append(segments.dropLast().joined(separator: ".")) + } + return "[" + tags.map(literal).joined(separator: ", ") + "]" + } + + /// A call the model can pattern-match, rather than the vacuous + /// `await path({})` that was emitted regardless of the arguments. + private static func example(for path: String, fields: [ToolField]) -> String { + let required = fields.filter { $0.optional == false } + guard required.isEmpty == false else { + return "await \(path)({})" + } + let pairs = required.map { field in + "\(field.name): \(placeholder(for: field))" + } + return "await \(path)({ " + pairs.joined(separator: ", ") + " })" + } + + private static func placeholder(for field: ToolField) -> String { + switch field.argumentType { + case "number": + return "0" + case "bool": + return "true" + case "array": + return "[]" + case "object": + return "{}" + default: + return "\"\(field.name)\"" + } + } + + private static func argumentConstraintsSource(for fields: [ToolField]) -> String { + let enumFields = fields.filter(\.isStringEnum) + guard enumFields.isEmpty == false else { + return "CapabilityArgumentConstraints.none" + } + let entries = enumFields.map { field in + "\(literal(field.name)): \(field.swiftType).codeModeAllowedValues" + }.joined(separator: ", ") + return "CapabilityArgumentConstraints(allowedStringValues: [\(entries)])" + } + private static func literal(_ value: String) -> String { let escaped = value .replacingOccurrences(of: "\\", with: "\\\\") @@ -511,6 +567,7 @@ private struct ToolField { var argumentType: String var optional: Bool var description: String? + var isStringEnum = false } private struct TypeInfo { @@ -518,6 +575,7 @@ private struct TypeInfo { var argumentType: String var optional: Bool var isSupported: Bool + var isStringEnum = false init(typeSyntax rawType: String) { let trimmed = rawType.replacingOccurrences(of: " ", with: "") @@ -554,6 +612,15 @@ private struct TypeInfo { if unwrapped.hasPrefix("["), unwrapped.hasSuffix("]") { self.argumentType = unwrapped.contains(":") ? "object" : "array" self.isSupported = Self.isSupportedCollection(unwrapped) + } else if Self.isSimpleIdentifier(unwrapped), Self.knownNonEnumTypes.contains(unwrapped) == false { + // An unrecognized simple type is taken to be a + // `CodeModeStringEnum`, the same assumption @BuiltInCodeMode + // makes. Host providers could not express constrained values at + // all before this, so their `allowedStringValues` was always + // empty while built-ins advertised theirs. + self.argumentType = "string" + self.isStringEnum = true + self.isSupported = true } else { self.argumentType = "any" self.isSupported = false @@ -561,6 +628,23 @@ private struct TypeInfo { } } + /// Types that are plainly not `CodeModeStringEnum`s. Without this list a + /// natural mistake such as `var date: Date` would be taken for an enum and + /// surface as a confusing error inside generated code instead of a macro + /// diagnostic pointing at the property. + private static let knownNonEnumTypes: Set = [ + "Date", "URL", "Data", "UUID", "Decimal", "Any", "AnyObject", "AnyHashable", + "Never", "Void", "Character", "Substring", "Int8", "Int16", "Int32", "Int64", + "UInt", "UInt8", "UInt16", "UInt32", "UInt64", "CGFloat", "NSNumber", "NSString", + ] + + private static func isSimpleIdentifier(_ type: String) -> Bool { + guard let first = type.first, first.isLetter || first == "_" else { + return false + } + return type.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "." } + } + private static func isSupportedCollection(_ type: String) -> Bool { let scalarTypes: Set = ["String", "Bool", "Int", "Double", "Float", "JSONValue"] let inner = String(type.dropFirst().dropLast()) diff --git a/TODO.md b/TODO.md index 282f76a..c5e8cf8 100644 --- a/TODO.md +++ b/TODO.md @@ -37,11 +37,18 @@ just the watchdog: serialization, cancellation, catch-proof termination, context recovery). - [ ] **JS heap / memory cap** — nothing bounds `JSContext`/`JSContextGroup` heap; `new Array(1e9)` can still exhaust host memory. NOT addressed. -- [ ] **Bound on concurrent executions** — `executionQueue` is `.concurrent` - with spin-waiting workers (`BridgeRuntime.swift:25-29`), so N long-running - scripts occupy N threads. No concurrency cap. NOT addressed. -- [ ] `runOnExecutionQueue` still has no task-cancellation handler wired into the - dispatched block (best-effort only). NOT addressed. +- [x] **Bound on concurrent executions** — `ExecutionLimits.maxConcurrentExecutions` + (default 8) is enforced by the `ExecutionSlots` actor, acquired in the async + context *before* dispatch, so excess executions park as suspended tasks rather + than blocked GCD threads. Search has its own slots so discovery never waits + behind long scripts. Covered by `concurrentExecutionsAllCompleteAndStayIsolated`, + `executionsBeyondTheSlotLimitQueueRatherThanFail`, and + `searchDoesNotWaitBehindALongExecution`. +- [x] Cancellation reaches `runOnExecutionQueue` at every stage: a cancel while + waiting for a slot throws before anything is dispatched + (`cancellingWhileWaitingForASlotNeverRunsTheScript`), and once dispatched, + `cancel()`, a cancelled `call.result` await, and dropping both the call and its + events stream all set the cancellation controller the runtime polls. ### 2. `fetch` has no destination restrictions or size limits [x] - [x] `NetworkAccessPolicy` on `CodeModeConfiguration`: by default requests are @@ -221,15 +228,22 @@ tool model). --- ## MUST-DO BEFORE MERGE -- [/] **Verify on macOS** — done 2026-07-11 (macOS 26.5, Swift 6.3.2 / Xcode 26.5): +- [x] **Verify on macOS** — done 2026-07-11 (macOS 26.5, Swift 6.3.2 / Xcode 26.5): - [x] `CCodeModeJSC` shim links and the private symbols resolve at load time - on macOS. (iOS/visionOS remain compile-verified only, via CI.) + on macOS, and on the iOS simulator (below). visionOS is compile-verified + only, via CI. - [x] `swift test` passes — 206 tests, 0 failures, ~5s. Note: the *first* verification run hung indefinitely and exposed the watchdog re-arm bug (see CRITICAL ISSUES §1); after the fix the suite is green. - - [ ] Watchdog terminates `while(true){}` on an iOS device/simulator — - verified on macOS only; JSC ships per-OS, so worth one manual check on a - simulator before tagging. + - [x] Watchdog terminates `while(true){}` on an iOS simulator — verified + 2026-10-01 on an iPhone 16 Pro simulator, iOS 18.6, by running the full + `CodeModeTests` target there (`xcodebuild test -scheme CodeMode-Package + -destination 'platform=iOS Simulator,…' -only-testing:CodeModeTests`). All + seven watchdog tests pass: CPU loop, loop inside a promise chain, + catch-proof termination, cancel mid-loop, runaway getter, serialization + budget, and search. Keychain storage tests skip there — an unsigned test + bundle has no keychain entitlement — and everything else passes. Still + simulator, not device. ## SUGGESTED ORDER OF ATTACK 1. Quick wins: tag `0.1.0`, fix calendar-span drift, add core-policy-layer tests, @@ -238,6 +252,84 @@ tool model). → "resource- and network-bounded runtime." 3. Metadata-consolidation refactor + macro decision, protected by new drift tests. +## REVIEW FOLLOW-UP — STILL OPEN +From `REVIEW.md`. Everything in P0 and P2 is done; these are the parts of the P1 +items that were not completed, kept explicit so they do not read as finished. + +### #9 — async bridge ABI (partially done) +Done: real timer queue (`setTimeout`/`clearTimeout` honour delays and cancel), +immediate diagnosis of unsettleable promises, `BRIDGE_FAILURES_NOT_SURFACED` for +swallowed rejections, bounded concurrent executions (actor slots, separate for +search), `ContinuousClock` deadlines behind an injectable `RuntimeClock`, and +cancellation once neither the call nor its events stream is reachable. +- [ ] **`CapabilityHandler` is still synchronous**, so `Promise.all([fetch(a), fetch(b)])` + is still serial. This is the change that requires resolving JS promises from + Swift via retained resolve/reject `JSValue`s on a per-execution serial executor. +- [ ] **Timeout/cancel still cannot interrupt an in-flight native call.** The + watchdog only traps while JS runs; `MediaBridge.transcode` can still overrun a + short `timeoutMs` by minutes. +- [ ] **`CodeModeAsyncBridge.run` still parks a semaphore** for up to 30s (longer + than the 10s default execution timeout) for async host tools. +- [ ] Optional: pool `JSVirtualMachine`s. Constraint to respect — the JSC time + limit is per context *group*, so a pooled VM must never host two concurrent + executions. + +### #11 — generate the JS bindings from the catalog +Done: the concrete drift found while generating declarations is fixed (keychain +object/positional, `fs.promises.rename`/`copyFile` options, location argument +pass-through, tool-description calling conventions). +- [ ] `RuntimeJavaScript.bootstrap` is still ~250 hand-written lines that can + drift from the catalog again. Wrappers that inject hidden arguments + (`createEvent` forces `operation: 'create'`) still cannot be expressed in + catalog metadata, so the generated `.d.ts` advertises an argument the wrapper + overrides. + +### #13 — provider ergonomics + MCP +Done: catalog unfrozen (post-init `register(provider:)` works and stays in sync), +enum constraints, real examples, per-segment tags. +- [ ] `@CodeModeProvider(namespace:)` on a type with `@CodeModeTool` on methods — + today it is still one struct per JavaScript function. +- [ ] `requiredPermissions` and typed results for provider registrations. +- [ ] MCP adapter materializing an MCP server's tools as `CodeModeRegistration`s + under `mcp..`. Blocked on async handlers (#9). + +### Test seams +Done: EventKit event/reminder serialization is now covered directly (the two +implicit-unwrap crashes), concurrent execution is covered, CI runners are pinned +to `macos-15` rather than `macos-latest`. `FetchTaskHandler`'s redirect +re-validation and size-cap cancellation turned out to be covered already +(`NetworkAccessPolicyTests`), contrary to the review. +- [ ] **Store protocols for EventKit/Health/Photos/Home/Contacts**, mirroring + `CodeModeFileSystem`. Their create/update/filter logic still executes nowhere: + the bridges hard-instantiate `EKEventStore()`, `HKHealthStore()`, `PHAsset`, so + tests can only reach permission denial and argument validation. +- [ ] A simulator test job, so UIKit presenters, AlarmKit, and the TCC-gated happy + paths are executed somewhere rather than only compiled. + +### Smaller P2 items not taken +- [ ] **No "prompt pending" permission status.** `request*` waits 10s and ignores + the timeout, so a user who takes 11s gets `PERMISSION_DENIED` while the dialog + is still on screen. Needs a new `PermissionStatus` case, which is source-breaking + for host code switching exhaustively. + +## DECLINED REVIEW FINDINGS +- **"Stop imposing swift-syntax on every consumer" (REVIEW.md P1 #12)** — + considered 2026-07-25 and declined. The proposal is to check in the expanded + `@BuiltInCodeMode` members so the `CodeMode` target can drop its + `CodeModeMacros` dependency. Declined because: + - The cost it cites is already measured and mitigated: prebuilt swift-syntax is + default-on for macros on the supported toolchains (`PLAN-registration-macros.md` + §"The swift-syntax cost, measured today"), so consumers pay ~0 incremental + build time, and the version range is already widened to `602.0.0..<604.0.0` + for `mlx-swift-lm` co-resolution — the one concrete conflict named. + - The migration is effectively one-way across 117 tool definitions and strips + the `@ToolParam("hint")` annotations off the property declarations, leaving + the hints readable only inside the generated argument arrays. + - It works against REVIEW #11 and #13, which both ask for *more* generation + from this metadata, not less. + Revisit if a consumer hits a swift-syntax version conflict the range cannot + absorb, or if prebuilts stop being default-on. + ## OPEN QUESTIONS FOR THE USER - [ ] Decision on the private-but-exported JSC symbol (App Store risk)? - [ ] After CI is green: tag `0.1.0` and restore the versioned install snippet? diff --git a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift index a6bed25..513097c 100644 --- a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift +++ b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift @@ -69,7 +69,7 @@ private struct FailingTaskTool: CodeModeProvider { #expect(registration.jsPath == "myapp.tasks.complete") #expect(registration.title == "complete") #expect(registration.summary == "Mark a task complete.") - #expect(registration.tags == ["myapp.tasks"]) + #expect(registration.tags == ["myapp", "tasks", "complete", "myapp.tasks"]) #expect(registration.requiredArguments == ["id"]) #expect(registration.optionalArguments == ["note"]) #expect(registration.argumentTypes["id"] == .string) @@ -233,12 +233,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.greet", title: "greet", summary: "Greet a person.", - tags: ["macro.api"], - example: "await macro.api.greet({})", + tags: ["macro", "api", "greet", "macro.api"], + example: "await macro.api.greet({ name: \\"name\\" })", requiredArguments: ["name"], optionalArguments: ["excited"], argumentTypes: ["name": CapabilityArgumentType.string, "excited": CapabilityArgumentType.bool], argumentHints: ["name": "Person name", "excited": "Whether to add emphasis"], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "Greeting payload", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -292,12 +293,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.ping", title: "ping", summary: "Ping.", - tags: ["macro.api"], + tags: ["macro", "api", "ping", "macro.api"], example: "await macro.api.ping({})", requiredArguments: [], optionalArguments: [], argumentTypes: [:], argumentHints: [:], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "null", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -511,3 +513,76 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution macros: codeModeTestMacros() ) } + +// MARK: - Constrained argument values for host providers + +/// Host providers could not express constrained values at all: the macro +/// rejected any non-primitive property type, so their `allowedStringValues` was +/// always empty while built-ins advertised theirs. This is the parity fix. +enum DemoPriority: String, CodeModeStringEnum { + case low + case normal + case high +} + +@CodeMode(path: "macro.api.prioritize", description: "Set a priority.") +struct PrioritizeTool: Sendable { + struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: DemoPriority? + } + + struct Result { + var id: String + var priority: String + } + + func call(arguments: Arguments) throws -> Result { + Result(id: arguments.id, priority: (arguments.priority ?? .normal).rawValue) + } +} + +@Test func codeModeMacroAdvertisesEnumConstraintsForHostProviders() throws { + let registration = try #require(PrioritizeTool().codeModeRegistrations().first) + + #expect(registration.argumentTypes["priority"] == .string) + #expect(registration.argumentConstraints.allowedStringValues["priority"] == ["low", "normal", "high"]) + // The generated example is a call the model can pattern-match, not the + // vacuous `await path({})` that used to be emitted regardless of arguments. + #expect(registration.example == #"await macro.api.prioritize({ id: "id" })"#) + // Tags cover every path segment, so the helper is findable by namespace and + // by name, not only by the full parent path. + #expect(registration.tags.contains("prioritize")) + #expect(registration.tags.contains("macro.api")) +} + +@Test func codeModeMacroRejectsValuesOutsideTheConstraint() async throws { + let tools = CodeModeAgentTools(config: CodeModeConfiguration(codeModeProviders: [PrioritizeTool()])) + + let denied = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "urgent" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + var deniedError: CodeModeToolError? + do { + _ = try await denied.result + } catch let error as CodeModeToolError { + deniedError = error + } + #expect(deniedError?.code == "INVALID_ARGUMENTS") + + let accepted = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "high" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + let result = try await accepted.result + #expect(result.output?.objectValue?["priority"] == .string("high")) +} diff --git a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift index 340f01b..5580d09 100644 --- a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift +++ b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift @@ -142,3 +142,147 @@ private func failureSummary(_ results: [CodeModeEvalResult]) -> String { } .joined(separator: "\n") } + +// MARK: - Multi-step scenarios + +// The runner overwrote executionOutput/Logs/Diagnostics/observedError on every +// step and did not stop on an intermediate error, so validation saw only the last +// step: a scenario could pass while step 1 failed in a way step 2 masked, and +// step 1's logs — the evidence — were gone. + +@Test func multiStepScenariosAccumulateLogsAcrossSteps() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-log-accumulation", + title: "Logs from every step survive", + task: "internal test", + executeSteps: [ + CodeModeEvalExecuteStep(code: "console.log('step-one-ran'); return 1;", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "console.log('step-two-ran'); return 2;", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript, .executeJavaScript], + exactAllowedCapabilities: [], + requiredExecutionLogFragments: ["step-one-ran", "step-two-ran"], + expectedOutput: .number(2) + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.passed, "\(result.failures)") + #expect(result.executionLogs.contains { $0.message.contains("step-one-ran") }) + #expect(result.executionLogs.contains { $0.message.contains("step-two-ran") }) + #expect(result.executionOutput == .number(2)) +} + +@Test func anIntermediateStepFailureStopsTheRunAndIsReported() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-intermediate-failure", + title: "A failing first step is not masked by a passing second", + task: "internal test", + executeSteps: [ + // Denied: the step declares no capabilities. + CodeModeEvalExecuteStep(code: "return await apple.fs.read({ path: 'tmp:nope.txt' });", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "return 'second step succeeded';", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + expectedErrorCode: "CAPABILITY_DENIED" + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.error?.code == "CAPABILITY_DENIED") + // The second step must not have run and must not have supplied the output. + #expect(result.toolCalls.filter { $0.tool == .executeJavaScript }.count == 1) + #expect(result.executionOutput != .string("second step succeeded")) +} + +// MARK: - Grading the one-script-per-job standard + +// The docs now tell agents to do a whole job in one executeJavaScript call. That +// is only a real standard if a transcript that ignores it fails, so this grades +// the shape rather than just asserting the reference solution. + +@Test func aTranscriptSplitAcrossExecutionsFailsTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // The anti-pattern: list in one call, read in another, sum in a third — + // every intermediate result round-tripped through the model's context. + let split = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.list({ path: 'documents:receipts' });", + allowedCapabilities: [.fsList] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/a.json' });", + allowedCapabilities: [.fsRead] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/c.json' });", + allowedCapabilities: [.fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: split, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Tool order") }) +} + +@Test func theSingleScriptTranscriptPassesTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + let single = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: single, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.isEmpty, "\(failures)") +} + +@Test func anOverBroadCapabilityRequestFailsEvenWithTheRightAnswer() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // Minimization is now stated plainly in the tool description, so grading it + // is no longer an unstated rule. + let overBroad = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead, .fsWrite] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: overBroad, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Allowed capabilities") || $0.contains("Forbidden capabilities") }) +} diff --git a/Tests/CodeModeTests/AgentGuidanceTests.swift b/Tests/CodeModeTests/AgentGuidanceTests.swift new file mode 100644 index 0000000..9dbbd94 --- /dev/null +++ b/Tests/CodeModeTests/AgentGuidanceTests.swift @@ -0,0 +1,171 @@ +import Foundation +import Testing +@testable import CodeMode + +// The tool descriptions and the generated TypeScript teach an agent what the +// helpers are called. Nothing taught it what shape its work should take, so an +// agent could reasonably use executeJavaScript as a thin RPC — one call per +// operation — which is the pattern this package exists to replace. + +@Test func everyTierStatesTheCoreIdea() { + // A host dropping to a smaller tier must not lose a rule, only the examples. + for length in CodeModeAgentGuidance.Length.allCases { + let text = CodeModeAgentGuidance.systemPrompt(length) + #expect(text.contains("One `executeJavaScript` call should do the entire job")) + #expect(text.contains("Request only the capabilities your script actually calls")) + } +} + +@Test func tiersGrowStrictlyAndAreAdditive() { + let brief = CodeModeAgentGuidance.systemPrompt(.brief) + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(brief.count < standard.count) + #expect(standard.count < full.count) + // Additive, not rewritten: the larger tiers contain the smaller ones verbatim. + #expect(standard.contains(brief)) + #expect(full.contains(standard)) +} + +@Test func largerTiersAddExamplesAndFailureModes() { + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(standard.contains("```javascript")) + #expect(standard.contains("searchJavaScriptAPI")) + #expect(full.contains("Anti-patterns")) + #expect(full.contains("BRIDGE_FAILURES_NOT_SURFACED")) + // The concurrency reality must not be over-promised in the fan-out example. + #expect(full.contains("`Promise.all` does not make these overlap")) +} + +@Test func aBudgetSelectsTheLargestTierThatFits() { + let brief = CodeModeAgentGuidance.Length.brief.approximateTokenCount + let standard = CodeModeAgentGuidance.Length.standard.approximateTokenCount + let full = CodeModeAgentGuidance.Length.full.approximateTokenCount + + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full + 5_000) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full - 1) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard - 1) == .brief) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: brief) == .brief) +} + +@Test func aBudgetTooSmallForAnythingStillReturnsTheCoreIdea() { + // Deliberately a floor rather than an empty string: some guidance is the + // difference between an agent that batches and one that does not. + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 0) == CodeModeAgentGuidance.systemPrompt(.brief)) + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: -100) == CodeModeAgentGuidance.systemPrompt(.brief)) +} + +@Test func guidanceExamplesUseHelpersAndCapabilitiesThatActuallyExist() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let full = CodeModeAgentGuidance.systemPrompt(.full) + let jsNames = Set(tools.capabilities().flatMap(\.jsNames)) + + // Guidance naming a helper the runtime does not install would teach the exact + // failure it is trying to prevent. + for helper in ["apple.fs.list", "apple.fs.read", "apple.fs.move"] { + #expect(full.contains(helper)) + #expect(jsNames.contains(helper), "guidance references missing helper \(helper)") + } + + for capability in ["fs.list", "fs.read", "fs.move"] { + #expect(CapabilityID(rawValue: capability) != nil, "guidance references missing capability \(capability)") + } +} + +@Test func theWorkedExampleActuallyRuns() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The example is the shape we ask agents to copy, so it has to be a script + // this runtime accepts — arguments, calling conventions, and all. The setup + // lines are the only addition. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.mkdir({ path: 'documents:receipts/lisbon' }); + await apple.fs.write({ + path: 'documents:receipts/a.json', + data: JSON.stringify({ trip: 'lisbon', amount: 12 }) + }); + await apple.fs.write({ + path: 'documents:receipts/b.json', + data: JSON.stringify({ trip: 'porto', amount: 99 }) + }); + + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + """, + allowedCapabilities: [.fsMkdir, .fsWrite, .fsList, .fsRead, .fsMove] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.int("total") == 12) + #expect(output.int("filedCount") == 1) +} + +@Test func everyHelperCallInTheDocsNamesARealHelperAndRealArguments() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + let references = tools.capabilities() + let byJSName = Dictionary(references.flatMap { ref in ref.jsNames.map { ($0, ref) } }, uniquingKeysWith: { first, _ in first }) + + // An example naming a helper or argument the runtime does not have teaches + // the exact failure the docs exist to prevent — and the wrong argument name + // is the easy mistake (`identifier` vs `identifiers` got through once). + // Parsed rather than listed, so new examples are checked automatically. + let texts = [CodeModeAgentToolDescriptions.executeJavaScript.description] + + CodeModeAgentGuidance.Length.allCases.map { CodeModeAgentGuidance.systemPrompt($0) } + let call = try Regex(#"((?:apple|ios)\.[A-Za-z.]+)\(\{([^}]*)\}"#) + let key = try Regex(#"(?:^|[,{\s])([A-Za-z_][A-Za-z0-9_]*)\s*(?=[:,}]|$)"#) + + var checked = 0 + for text in texts { + for match in text.matches(of: call) { + let helper = String(match.output[1].substring!) + let reference = try #require(byJSName[helper], "docs reference missing helper \(helper)") + let known = Set(reference.requiredArguments + reference.optionalArguments) + let body = String(match.output[2].substring!) + for argument in body.matches(of: key).map({ String($0.output[1].substring!) }) { + #expect(known.contains(argument), "\(helper) has no argument '\(argument)'") + } + checked += 1 + } + } + #expect(checked >= 5, "the parser found too few helper calls to be checking anything") +} + +@Test func theToolDescriptionStaysWithinItsBudget() { + // Sent on every turn and not trimmable by the host — unlike the guidance + // tiers, which exist precisely so worked examples can be budgeted. Examples + // belong there; this guard stops the description regrowing them. + #expect(CodeModeAgentToolDescriptions.executeJavaScript.description.count < 4_800) + #expect(CodeModeAgentToolDescriptions.executeJavaScript.description.contains("const results = []") == false) +} diff --git a/Tests/CodeModeTests/CodeModeProviderTests.swift b/Tests/CodeModeTests/CodeModeProviderTests.swift index 67387c9..1574388 100644 --- a/Tests/CodeModeTests/CodeModeProviderTests.swift +++ b/Tests/CodeModeTests/CodeModeProviderTests.swift @@ -578,7 +578,11 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.bool("done") == true) } -@Test func builtInCapabilitiesCanBeAllowedByCapabilityKey() async throws { +// `allowedCapabilityKeys` accepts arbitrary strings and is not validated against +// `CapabilityID` at decode time, so it must never reach a built-in bridge — a host +// that vets only the strictly-typed `allowedCapabilities` would otherwise be +// bypassed by spelling the same ID in the loose field. +@Test func builtInCapabilitiesAreNotGrantedByCapabilityKey() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -590,7 +594,83 @@ private func jsonLiteral(_ value: JSONValue) -> String { ) ) let observed = await observe(call) - #expect(observed.result?.output == .null) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.capabilityKey == CapabilityID.keychainRead.codeModeKey) +} + +@Test func builtInCapabilitiesRemainGrantedByCapabilityID() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("missing-provider-key-test");"#, + allowedCapabilities: [.keychainRead] + ) + ) + let observed = await observe(call) + // The subject is capability gating, not keychain storage: reaching the + // bridge at all is the proof. Where the test host has no keychain + // entitlement the bridge then fails for that reason, which is still not a + // denial. + #expect(observed.error?.code != "CAPABILITY_DENIED") + if keychainIsAvailable { + #expect(observed.result?.output == .null) + } +} + +@Test func hostCapabilityGrantOverridesModelAuthoredAllowlist() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([.fsRead], capabilityKeys: []), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + // The script asks for keychain.read and a provider key; the host grants neither. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("anything");"#, + allowedCapabilities: [.keychainRead], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.suggestions.contains { $0.contains("host app's capability grant") }) + #expect( + error.diagnostics.contains { $0.code == "CAPABILITY_WITHHELD_BY_HOST" } + ) + + let providerCall = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let providerObserved = await observe(providerCall) + #expect(providerObserved.error?.code == "CAPABILITY_DENIED") +} + +@Test func hostCapabilityGrantPermitsWhatItIncludes() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([], capabilityKeys: ["myapp.api.doTheThing"]), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let result = try #require(observed.result?.output?.objectValue) + #expect(result.string("id") == "123") } @Test func customProviderFunctionErrorsRemainStructured() async throws { @@ -632,3 +712,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.string("custom") == "myapp.api.doTheThing") #expect(result.string("builtIn") == CapabilityID.weatherRead.rawValue) } + +// MARK: - Post-init provider registration + +private struct LateProvider: CodeModeProvider { + let codeModePath = "myapp.late" + + func codeModeRegistrations() -> [CodeModeRegistration] { + [ + CodeModeRegistration( + capabilityKey: "myapp.late.echo", + jsPath: "myapp.late.echo", + title: "echo", + summary: "Echo a value registered after init.", + tags: ["myapp", "late"], + example: #"await myapp.late.echo({ value: "hi" })"#, + requiredArguments: ["value"], + argumentTypes: ["value": .string], + argumentHints: ["value": "Value to echo"], + resultSummary: "Echoed value" + ) { arguments, _ in + .object(["value": .string(try CodeModeArgumentDecoder.requireString("value", in: arguments))]) + }, + ] + } +} + +// The catalog used to be snapshotted in CodeModeAgentTools.init, so the +// registry's public post-init register(...) methods were unreachable in the +// supported flow — and would have desynced search from execution if reached. +@Test func providersRegisteredAfterInitAreExecutable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let before = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + // Not registered yet: the helper simply does not exist in the context. + let beforeObserved = await observe(before) + #expect(beforeObserved.error != nil) + + tools.register(provider: LateProvider()) + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + let observed = await observe(call) + #expect(observed.result?.output?.objectValue?.string("value") == "hi") +} + +@Test func providersRegisteredAfterInitAppearInSearchAndTypeDeclarations() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + #expect(tools.capabilities().contains { $0.capability == "myapp.late.echo" } == false) + + tools.register(provider: LateProvider()) + + // Search must not advertise anything execution cannot invoke, and vice versa. + let reference = try #require(tools.capabilities().first { $0.capability == "myapp.late.echo" }) + #expect(reference.dts.contains("value: string;")) + #expect(tools.typeDeclarations().contains("late: {")) + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: #"async () => { return api.byJSName["myapp.late.echo"].summary; }"#) + ) + #expect(response.result == .string("Echo a value registered after init.")) +} diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift new file mode 100644 index 0000000..450f9e4 --- /dev/null +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -0,0 +1,62 @@ +import Foundation +import Testing +@testable import CodeMode + +/// Serialized: these assert on a dispatched block getting a thread, which is exactly +/// what the watchdog and event-loop suites make scarce when run alongside them. +@Suite(.serialized) struct CompletionWaitTests { + + // The pattern this replaces — a `var` written from a framework's completion + // queue and read after a semaphore wait whose timeout result was discarded — is + // both a data race and an ambiguity: the caller could not tell "timed out" from + // "the API returned nothing". + + @Test func completionWaitReturnsTheDeliveredValue() throws { + // 60s, not 5: other suites block GCD workers for the length of a timer (the + // synchronous bridge model holds a thread per running execution), so even an + // immediate `global().async` can wait seconds for a thread on a loaded + // runner. The assertion is that the delivered value comes back, not how + // fast — the thread pressure itself is what the async ABI (#9) removes. + let value = try CompletionWait.value(timeout: 60, operationName: "test") { complete in + DispatchQueue.global().async { complete([1, 2, 3]) } + } + #expect(value == [1, 2, 3]) + } + + @Test func completionWaitThrowsRatherThanReturningAnEmptyResult() { + do { + // Never completes: the old shape returned `.array([])`, indistinguishable + // from a genuinely empty fetch. + let value: [Int] = try CompletionWait.value(timeout: 0.1, operationName: "reminders.read") { _ in } + Issue.record("Expected a timeout, got \(value)") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + } + + @Test func completionWaitToleratesALateCallback() throws { + // The late write must land in the box rather than racing a caller's read. + let lateCallbackFinished = DispatchSemaphore(value: 0) + do { + let _: Int = try CompletionWait.value(timeout: 0.05, operationName: "test") { complete in + DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { + complete(7) + lateCallbackFinished.signal() + } + } + Issue.record("Expected a timeout") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + + // Generous on purpose: other suites block GCD workers for the length of a + // timer, so a 0.2s dispatch can be starved for seconds on a loaded runner. + // The assertion is that the late callback lands at all. + #expect(lateCallbackFinished.wait(timeout: .now() + 60) == .success) + } + + @Test func completionWaitReportsWhetherTheCallbackArrived() { + #expect(CompletionWait.completion(timeout: 5) { complete in complete() }) + #expect(CompletionWait.completion(timeout: 0.05) { _ in } == false) + } +} diff --git a/Tests/CodeModeTests/DefensiveDecodingTests.swift b/Tests/CodeModeTests/DefensiveDecodingTests.swift index 8a4485c..e9ede14 100644 --- a/Tests/CodeModeTests/DefensiveDecodingTests.swift +++ b/Tests/CodeModeTests/DefensiveDecodingTests.swift @@ -56,6 +56,50 @@ private func decodeRequest(_ json: String) throws -> JavaScriptExecutionRequest } } +// MARK: - Adversarial numbers must not trap + +@Test func executionRequestRejectsNonRepresentableTimeout() throws { + // `Int.init(_: Double)` traps on these. This decode path runs on + // model-authored JSON before any script does, so a trap here is an + // in-process kill of the host app. + for literal in ["1e300", "-1e300", "1e400", "-1e400"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":\#(literal)}"#) + } + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } + + for literal in ["inf", "-inf", "infinity", "nan", "NaN"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } +} + +@Test func executionRequestClampsTimeoutToAdvertisedBounds() throws { + // The schema advertises 1...60000; out-of-range values are clamped rather + // than failing the call. + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":600000}"#).timeoutMs == 60_000) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":0}"#).timeoutMs == 1) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":-5}"#).timeoutMs == 1) + #expect(JavaScriptExecutionRequest(code: "", allowedCapabilities: [], timeoutMs: 10_000_000).timeoutMs == 60_000) +} + +@Test func jsonValueIntConversionIsTotal() { + #expect(JSONValue.number(.infinity).intValue == nil) + #expect(JSONValue.number(-.infinity).intValue == nil) + #expect(JSONValue.number(.nan).intValue == nil) + #expect(JSONValue.number(1e300).intValue == nil) + #expect(JSONValue.number(-1e300).intValue == nil) + #expect(JSONValue.number(9.3e18).intValue == nil) + #expect(JSONValue.number(42).intValue == 42) + #expect(JSONValue.number(-42.9).intValue == -42) + #expect(JSONValue.number(Double(Int.max)).intValue == nil) + #expect(JSONValue.number(Double(Int.min)).intValue == Int.min) +} + @Test func executionRequestRoundTripsThroughCodable() throws { let original = JavaScriptExecutionRequest( code: "return 1;", @@ -118,6 +162,38 @@ private func makeEchoRegistry(argumentTypes: [String: CapabilityArgumentType]) - } } +@Test func registryRejectsNonFiniteDeclaredNumbers() throws { + let registry = makeEchoRegistry(argumentTypes: ["timeoutMs": .number]) + let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) + defer { cleanup(sandbox) } + + // Straight through as a JSON number... + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .number(.infinity)], + context: context + ) + } + // ...and smuggled through string coercion, which `Double.init(String)` accepts. + for spelling in ["inf", "-infinity", "nan"] { + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string(spelling)], + context: context + ) + } + } + + let ok = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string("2500")], + context: context + ) + #expect(ok.objectValue?["timeoutMs"] == .number(2_500)) +} + @Test func registryDoesNotCoerceNonBooleanStringsOrBadNumbers() throws { let registry = makeEchoRegistry(argumentTypes: ["count": .number, "flag": .bool]) let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) diff --git a/Tests/CodeModeTests/EventKitBridgeTests.swift b/Tests/CodeModeTests/EventKitBridgeTests.swift index 6fb95b9..da6c32a 100644 --- a/Tests/CodeModeTests/EventKitBridgeTests.swift +++ b/Tests/CodeModeTests/EventKitBridgeTests.swift @@ -135,3 +135,44 @@ import Testing #expect(observed.error?.code == "PERMISSION_DENIED") } + +// MARK: - Serialization of degenerate items + +#if canImport(EventKit) +import EventKit + +// EventKit's serialization logic executes nowhere today — not in unit tests, not +// in CI, not in evals — because the bridge hard-instantiates EKEventStore and the +// tests can only reach permission denial. These construct the model objects +// directly, which needs no permission, and cover the two implicitly-unwrapped +// fields that crashed the execution thread. + +@Test func eventSerializationSurvivesAnEventWithNoCalendar() throws { + let store = EKEventStore() + let event = EKEvent(eventStore: store) + event.title = "Orphaned" + event.startDate = Date(timeIntervalSince1970: 1_700_000_000) + event.endDate = event.startDate.addingTimeInterval(3_600) + + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned event, while + // every neighbouring field was nil-coalesced. + #expect(event.calendar == nil) + + let json = try #require(EventKitBridge.eventJSONForTesting(event).objectValue) + #expect(json.string("title") == "Orphaned") + #expect(json.string("calendarIdentifier") == "") + #expect(json.string("calendarTitle") == "") + #expect(json.string("notes") == "") +} + +@Test func reminderSerializationSurvivesAMissingTitle() throws { + let store = EKEventStore() + let reminder = EKReminder(eventStore: store) + + // `EKReminder.title` is `String!`. + let json = try #require(EventKitBridge.reminderJSONForTesting(reminder).objectValue) + #expect(json.string("title") == "") + #expect(json.string("calendarIdentifier") == "") + #expect(json.bool("isCompleted") == false) +} +#endif diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift new file mode 100644 index 0000000..97882b9 --- /dev/null +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -0,0 +1,731 @@ +import Foundation +import Testing +@testable import CodeMode + +/// Serialized: most of these hold a GCD worker for the length of a timer, and several +/// pin a slot deliberately. Run concurrently with each other on a small CI +/// runner they starve the global pool — and each other — for tens of seconds. +@Suite(.serialized) struct EventLoopTests { + + // `setTimeout` used to invoke its callback synchronously and ignore the delay. + // Three things were wrong at once: clearTimeout could never cancel, an error + // thrown by a callback propagated to setTimeout's *caller*, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was a + // hot loop that hammered remote APIs through fetch. + + @Test func setTimeoutDefersItsCallbackInsteadOfRunningItInline() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => { order.push('timer'); }, 0); + order.push('sync'); + await new Promise(resolve => setTimeout(resolve, 1)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let order = try #require(observed.result?.output?.objectValue?.array("order")) + #expect(order == [.string("sync"), .string("timer")]) + } + + @Test func setTimeoutActuallyWaitsTheRequestedDelay() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 150)); return 'done';", + allowedCapabilities: [], + timeoutMs: 5_000 + ) + ) + + #expect(observed.result?.output == .string("done")) + // The delay was previously ignored entirely, so a backoff loop spun. Under + // a virtual clock the assertion is exact: the runtime slept for precisely + // the delay it was asked for — not "at least" it, and not a wall-clock + // bound that a loaded runner could stretch. + #expect(clock.elapsed == .milliseconds(150)) + } + + @Test func clearTimeoutCancelsAPendingCallback() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const fired = []; + const id = setTimeout(() => { fired.push('should-not-run'); }, 5); + clearTimeout(id); + await new Promise(resolve => setTimeout(resolve, 20)); + return { fired }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("fired") == []) + } + + @Test func timersFireInDueOrderNotRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + // Registered late-first, so registration order and due order disagree. This + // must hold however far the host's sleep overshoots — if enough time passes + // for both to come due in one tick, they still fire shortest-delay first. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('b'), 10); + setTimeout(() => order.push('a'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("a"), .string("b")]) + } + + @Test func timersComingDueInOneTickStillFireInDueOrder() async throws { + // A slow machine wakes late. Modelled exactly: every sleep overshoots by + // 10ms, so the first wake (aimed at 5ms) lands at 15ms with *both* timers + // due in a single advanceTimers call — the case that sorting by + // registration id alone got wrong. Registered late-first so the two + // orderings disagree. Before the clock seam this condition could only be + // reproduced by a loaded CI runner. + let clock = VirtualClock(overshoot: .milliseconds(10)) + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('later'), 10); + setTimeout(() => order.push('sooner'), 5); + await new Promise(resolve => setTimeout(resolve, 40)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("sooner"), .string("later")]) + } + + @Test func timersDueAtTheSameInstantFireInRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + // Equal delays, so only registration order can break the tie. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('first'), 5); + setTimeout(() => order.push('second'), 5); + setTimeout(() => order.push('third'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect( + observed.result?.output?.objectValue?.array("order") + == [.string("first"), .string("second"), .string("third")] + ) + } + + @Test func anErrorInATimerCallbackBecomesADiagnosticNotACallerThrow() async throws { + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + setTimeout(() => { throw new Error('boom'); }, 1); + await new Promise(resolve => setTimeout(resolve, 20)); + return 'survived'; + """, + allowedCapabilities: [] + ) + ) + + // Previously this propagated to whoever called setTimeout, which is not how + // an event loop behaves. + #expect(observed.result?.output == .string("survived")) + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "TIMER_CALLBACK_ERROR" && $0.message.contains("boom") }) + } + + @Test func anUnsettleablePromiseIsReportedImmediately() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // No resolve path and no queued timer: nothing in this runtime can advance + // it, so sleeping to the deadline buys nothing. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(() => {}); return 1;", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) + // "Immediately" means the runtime never slept at all. + #expect(clock.elapsed == .zero) + } + + @Test func aTimerStillPendingAtTheDeadlineTimesOut() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // A timer due long after the deadline must not be waited out. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 30000)); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Stopped exactly at the 100ms deadline, never anywhere near the 30s timer. + #expect(clock.elapsed == .milliseconds(100)) + } + + @Test func aLongBackoffStaysCancellable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + + Task { + try? await Task.sleep(nanoseconds: 100_000_000) + call.cancel() + } + + let started = Date() + let observed = await observe(call) + #expect(observed.error?.code == "CANCELLED") + // Proves cancel interrupted the 20s timer rather than waiting it out. + #expect(Date().timeIntervalSince(started) < 15) + } + + // MARK: - Swallowed bridge failures + + @Test func aForgottenAwaitOnAFailedCallIsSurfacedAsADiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Nothing installs an unhandled-rejection hook, so this used to complete with + // the agent told "done" while a CAPABILITY_DENIED was silently discarded. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + apple.keychain.get({ key: 'never-allowed' }).catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 10)); + return 'looks fine'; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output == .string("looks fine")) + let diagnostics = try #require(observed.result?.diagnostics) + let swallowed = try #require(diagnostics.first { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" }) + #expect(swallowed.message.contains("keychain.read")) + #expect(swallowed.message.contains("CAPABILITY_DENIED")) + #expect(swallowed.suggestions.contains { $0.contains("missing await") }) + } + + @Test func aCleanRunGetsNoSwallowedFailureDiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ok.txt', data: 'x' }); return 'ok';", + allowedCapabilities: [.fsWrite] + ) + ) + + #expect(observed.result?.diagnostics.contains { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" } == false) + } + + // MARK: - Bounded concurrency + + @Test func concurrentExecutionsAllCompleteAndStayIsolated() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 3) + ) + defer { cleanup(sandbox) } + + // The concurrent execution queue is a core design point but had no test, and + // no width limit: each execution blocks its thread for the whole run, so N + // parallel calls meant N blocked GCD threads and N live JavaScriptCore VMs. + // More executions than slots must still all complete, each with its own + // context and its own result. + let results = try await withThrowingTaskGroup(of: (Int, JSONValue?).self) { group in + for index in 0..<12 { + group.addTask { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:worker-\(index).txt', data: 'value-\(index)' }); + const read = await apple.fs.read({ path: 'tmp:worker-\(index).txt' }); + return { index: \(index), text: read.text, leaked: typeof globalThis.__leaked }; + """, + allowedCapabilities: [.fsRead, .fsWrite], + timeoutMs: 20_000 + ) + ) + return (index, observed.result?.output) + } + } + + var collected: [Int: JSONValue?] = [:] + for try await (index, output) in group { + collected[index] = output + } + return collected + } + + #expect(results.count == 12) + for index in 0..<12 { + let output = try #require(results[index]??.objectValue) + #expect(output.int("index") == index) + #expect(output.string("text") == "value-\(index)") + // A fresh JSContext per execution: no state carries between them. + #expect(output.string("leaked") == "undefined") + } + } + + @Test func executionsBeyondTheSlotLimitQueueRatherThanFail() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1), + clock: VirtualClock() + ) + defer { cleanup(sandbox) } + + // With one slot these serialize; the point is that the extras wait for a slot + // instead of erroring, and that waiting for a slot does not count against the + // per-execution timeout. + let outputs = try await withThrowingTaskGroup(of: JSONValue?.self) { group in + for index in 0..<4 { + group.addTask { + try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, 60)); return \(index);", + allowedCapabilities: [], + timeoutMs: 500 + ) + ).result?.output + } + } + var collected: [JSONValue?] = [] + for try await output in group { + collected.append(output) + } + return collected + } + + #expect(outputs.compactMap { $0?.intValue }.sorted() == [0, 1, 2, 3]) + } + + @Test func aResultThatSettlesRightAtTheDeadlineIsNotDiscarded() async throws { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + // Timer and deadline at the same instant. The settled state must be read + // before the deadline is enforced: checking the deadline first turns a + // script that fulfilled exactly on time into a spurious timeout, discarding + // a result that already exists. Under a real clock this was a 12-iteration + // race; under a virtual one it is a single exact case. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 40)); return 'settled';", + allowedCapabilities: [], + timeoutMs: 40 + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("settled")) + } + + // MARK: - Lifetime and slot semantics + + @Test func iteratingEventsAfterDroppingTheCallDoesNotCancelTheExecution() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Swift does not keep a local alive to the end of its scope, so a host that + // reads `call.events` and never touches `call` again may have it released + // mid-iteration in an optimized build. Cancelling in the call's deinit cut + // that host off. Force the release explicitly here rather than hoping the + // optimizer does it. + var call: JavaScriptExecutionCall? = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: """ + console.log('start'); + await new Promise(resolve => setTimeout(resolve, 200)); + console.log('end'); + return 'finished'; + """, + allowedCapabilities: [] + ) + ) + let events = call!.events + call = nil + + var sawFinished = false + var sawEnd = false + for await event in events { + if case .finished = event { sawFinished = true } + if case let .log(entry) = event, entry.message == "end" { sawEnd = true } + } + + #expect(sawEnd) + #expect(sawFinished) + } + + @Test func droppingBothTheCallAndItsStreamCancelsTheExecution() async throws { + // One slot: if the dropped execution were still running, the second call + // could not start until its 20s timer elapsed. + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + func startAndDrop() async throws { + let dropped = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + _ = dropped.events + // Give it a moment to actually occupy the slot before we drop it. + try await Task.sleep(nanoseconds: 100_000_000) + } + try await startAndDrop() + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return 'got the slot';", allowedCapabilities: [], timeoutMs: 5_000) + ) + + #expect(observed.result?.output == .string("got the slot")) + #expect(Date().timeIntervalSince(started) < 15, "the dropped execution must have released its slot") + } + + @Test func cancellingWhileWaitingForASlotNeverRunsTheScript() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // Occupy the only slot. + let occupant = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 3000)); return 'occupant';", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + try await Task.sleep(nanoseconds: 100_000_000) + + // This one parks waiting for the slot. Cancel it there: it must report + // CANCELLED and must never have executed — the file it would write is the + // evidence. + let parked = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ran.txt', data: 'x' }); return 'ran';", + allowedCapabilities: [.fsWrite], + timeoutMs: 30_000 + ) + ) + try await Task.sleep(nanoseconds: 100_000_000) + parked.cancel() + + let parkedOutcome = await observe(parked) + #expect(parkedOutcome.error?.code == "CANCELLED") + #expect(FileManager.default.fileExists(atPath: sandbox.tmp.appendingPathComponent("ran.txt").path) == false) + + // The occupant is unaffected and the slot still hands over cleanly afterwards. + let occupantOutcome = await observe(occupant) + #expect(occupantOutcome.result?.output == .string("occupant")) + } + + @Test func cancellationInterruptsASleepImmediately() throws { + let controller = ExecutionCancellationController() + let entered = DispatchSemaphore(value: 0) + let woke = DispatchSemaphore(value: 0) + + // A polling sleep would not notice the cancel until its next slice; an + // interruptible one returns as soon as the cancel lands. Synchronous test on + // purpose: the sleeper is a real blocked thread, as it is in the runtime. + DispatchQueue.global().async { + entered.signal() + controller.sleep(until: Date(timeIntervalSinceNow: 30)) + woke.signal() + } + + // Wait for the sleeper to actually be scheduled before cancelling. On a + // starved runner the dispatch alone can take seconds, and cancelling before + // the sleep has begun measures scheduling, not the wake-up this test is + // about. The generous bound here is for scheduling; the tight one below is + // for the wake. + #expect(entered.wait(timeout: .now() + 60) == .success, "sleeper never got a thread") + let cancelledAt = Date() + controller.cancel() + + #expect(woke.wait(timeout: .now() + 10) == .success) + #expect(Date().timeIntervalSince(cancelledAt) < 5) + } + + @Test func sleepReturnsAtTheDeadlineWhenNotCancelled() { + let controller = ExecutionCancellationController() + let started = Date() + controller.sleep(until: Date(timeIntervalSinceNow: 0.1)) + #expect(Date().timeIntervalSince(started) >= 0.09) + #expect(controller.isCancelled == false) + } + + @Test func searchDoesNotWaitBehindALongExecution() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // Occupy the only execution slot with a script that writes a file after + // 3s. If search shared that slot it would wait for it, and the file would + // exist by the time search returned. + let occupant = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "console.log('running'); await new Promise(r => setTimeout(r, 3000)); await apple.fs.write({ path: 'tmp:occupant-done.txt', data: 'x' }); return 1;", + allowedCapabilities: [.fsWrite], + timeoutMs: 30_000 + ) + ) + // Synchronize on the occupant actually holding the slot, not on a sleep. + var running = false + for await event in occupant.events { + if case let .log(entry) = event, entry.message == "running" { running = true; break } + if case .finished = event { break } + } + #expect(running, "the occupant never started") + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: "async () => api.references.length") + ) + let occupantFinished = FileManager.default.fileExists(atPath: sandbox.tmp.appendingPathComponent("occupant-done.txt").path) + + #expect((response.result?.intValue ?? 0) > 0) + #expect(occupantFinished == false, "search waited for the execution slot") + occupant.cancel() + _ = await observe(occupant) + } + + // MARK: - Delays that are not ordinary numbers + + @Test func unrepresentableTimerDelaysTimeOutInsteadOfCrashingTheHost() async throws { + // `Int(_: Double)` traps on these, and the timer loop converted the delay + // straight from JavaScript: a capability-free `setTimeout(r, 1e300)` + // killed the host process. Each must end in an ordinary timeout, exactly + // at the deadline. + for delay in ["1e300", "Infinity", "Number.MAX_VALUE", "2 ** 53"] { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, \(delay))); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT", "delay \(delay)") + #expect(clock.elapsed == .milliseconds(100), "delay \(delay)") + } + } + + @Test func degenerateTimerDelaysFireImmediately() async throws { + // NaN, negative, and missing delays mean "as soon as possible", as on the web. + for delay in ["-5", "NaN", "'abc'", "undefined", "null", "-Infinity"] { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, \(delay))); return 'fired';", + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output == .string("fired"), "delay \(delay)") + #expect(clock.elapsed == .zero, "delay \(delay)") + } + } + + @Test func searchSurvivesAnUnrepresentableTimerDelay() async throws { + // Search runs the same settlement loop, with no capabilities at all. + let (tools, sandbox) = try makeTools(clock: VirtualClock()) + defer { cleanup(sandbox) } + + do { + _ = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: "async () => { await new Promise(r => setTimeout(r, 1e300)); return 1; }") + ) + Issue.record("Expected the search to time out") + } catch let error as CodeModeToolError { + #expect(error.code == "SEARCH_TIMEOUT") + } + } + + @Test func aScriptTamperingWithTheTimerInternalsCannotCrashTheHost() async throws { + // `__codemode` is on globalThis, so the JavaScript-side delay + // normalization is not a boundary: a script can replace the function the + // host reads, or push an entry directly. The Swift-side bound is what + // keeps the host alive; this exercises it with the normalization bypassed. + let tampering = [ + "globalThis.__codemode.nextTimerDelay = () => 1e300;", + "globalThis.__codemode.nextTimerDelay = () => Infinity;", + "globalThis.__codemode.timers.entries.push({ id: 1e9, fn: () => {}, args: [], dueIn: Infinity });", + ] + for tamper in tampering { + let clock = VirtualClock() + let (tools, sandbox) = try makeTools(clock: clock) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "\(tamper) await new Promise(() => {}); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT", "\(tamper)") + #expect(clock.elapsed == .milliseconds(100), "\(tamper)") + } + } + + // MARK: - Compiling scripts apart from the harness + + @Test func aScriptCannotCloseItsWrapperAndRunAtTopLevel() async throws { + // User code used to be spliced into `(async function(){ … })()`, so a + // script could close the wrapper and run at top level, outside the + // harness. Throwing `{ line: -Infinity }` from there crashed the host. + // Compiled on its own, each of these is just a syntax error. + let breakouts = [ + "return 1; })(); throw { line: -Infinity, message: 'x' }; (async function(){", + "return 1; })(); console.log('escaped'); (async function(){", + "return 1 }; console.log('escaped'); async function g(){", + ] + for code in breakouts { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + let observed = try await execute(tools, request: JavaScriptExecutionRequest(code: code, allowedCapabilities: [])) + #expect(observed.error?.code == "JS_SYNTAX_ERROR", "\(code)") + #expect(observed.result == nil, "\(code)") + } + } + + @Test func aSearchCannotCloseItsWrapperAndRunAtTopLevel() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + do { + _ = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: "async () => 1)); throw { line: -Infinity }; ((async () => 1") + ) + Issue.record("Expected JS_SYNTAX_ERROR") + } catch let error as CodeModeToolError { + #expect(error.code == "JS_SYNTAX_ERROR") + } + } + + @Test func aTrailingLineCommentDoesNotSwallowTheHarness() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + let observed = try await execute(tools, request: JavaScriptExecutionRequest(code: "return 7 // done", allowedCapabilities: [])) + #expect(observed.error == nil) + #expect(observed.result?.output == .number(7)) + + let found = try await tools.searchJavaScriptAPI(JavaScriptAPISearchRequest(code: "async () => 7 // done")) + #expect(found.result == .number(7)) + } + + @Test func syntaxErrorsReportTheScriptsOwnLine() async throws { + for (code, line) in [("const b = ;", 1), ("const a = 1;\nconst b = ;\nreturn a;", 2)] { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + let observed = try await execute(tools, request: JavaScriptExecutionRequest(code: code, allowedCapabilities: [])) + #expect(observed.error?.code == "JS_SYNTAX_ERROR") + #expect(observed.error?.line == line, "\(code)") + // JavaScriptCore reports the column of the compile call, not the error. + #expect(observed.error?.column == nil) + } + + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + do { + _ = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: "async () => {\n const value = ;\n}") + ) + Issue.record("Expected JS_SYNTAX_ERROR") + } catch let error as CodeModeToolError { + #expect(error.code == "JS_SYNTAX_ERROR") + #expect(error.line == 2) + } + } +} diff --git a/Tests/CodeModeTests/ExecutionLimitsTests.swift b/Tests/CodeModeTests/ExecutionLimitsTests.swift new file mode 100644 index 0000000..7dae8ec --- /dev/null +++ b/Tests/CodeModeTests/ExecutionLimitsTests.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing +@testable import CodeMode + +// Nothing bounded a script's output before: the whole result was stringified and +// copied into Swift, every console.log was retained and re-copied into each +// CodeModeToolError, and the events stream buffered without limit. One script +// could jetsam an iOS host. Every limit degrades with a diagnostic instead of +// failing, because for an LLM consumer a truncated result beats an OOM. + +@Test func oversizedResultsAreReplacedWithAParseableDescriptor() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 2_000, truncatedResultPreviewCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return Array.from({ length: 5000 }, (_, i) => 'entry-' + i);", + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("__codeModeTruncated") == true) + #expect((output.int("characterCount") ?? 0) > 2_000) + #expect(output.int("limit") == 2_000) + #expect((output.string("preview")?.count ?? 0) <= 200) + + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "RESULT_TRUNCATED" }) +} + +@Test func resultsWithinTheLimitAreUntouched() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 100_000) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return { ok: true, items: [1, 2, 3] };", allowedCapabilities: []) + ) + + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("ok") == true) + #expect(output.array("items")?.count == 3) + #expect(observed.result?.diagnostics.contains { $0.code == "RESULT_TRUNCATED" } == false) +} + +@Test func retainedLogsAreCappedWithANotice() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 25) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "for (let i = 0; i < 500; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + + let logs = try #require(observed.result?.logs) + #expect(logs.count == 25) + // The first N are kept: the beginning of a runaway log is where the cause is. + #expect(logs.first?.message == "line 0") + #expect(observed.result?.diagnostics.contains { $0.code == "LOG_LIMIT_REACHED" } == true) +} + +@Test func longLogMessagesAreElidedInTheMiddle() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogMessageCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "console.log('HEAD' + 'x'.repeat(100000) + 'TAIL'); return 'done';", + allowedCapabilities: [] + ) + ) + + let message = try #require(observed.result?.logs.first?.message) + #expect(message.count < 400) + #expect(message.hasPrefix("HEAD")) + #expect(message.hasSuffix("TAIL")) + #expect(message.contains("characters elided")) +} + +@Test func diagnosticsAndPermissionEventsAreCapped() { + let transcript = ExecutionTranscript( + limits: ExecutionLimits(maxDiagnostics: 3, maxPermissionEvents: 2) + ) + + for index in 0..<50 { + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "D\(index)", message: "m")) + transcript.record(permissionEvent: PermissionEvent(permission: .contacts, status: .granted)) + } + + let snapshot = transcript.snapshot() + #expect(snapshot.diagnostics.count == 3) + #expect(snapshot.permissionEvents.count == 2) +} + +@Test func theEventStreamBufferIsBounded() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 100_000, maxBufferedEvents: 16) + ) + defer { cleanup(sandbox) } + + // Never drained until the script has finished: with the default unbounded + // policy this buffers every event; bounded, it keeps only the newest. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "for (let i = 0; i < 5000; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + _ = try await call.result + + var received = 0 + for await _ in call.events { + received += 1 + } + #expect(received <= 16) +} diff --git a/Tests/CodeModeTests/ExecutionWatchdogTests.swift b/Tests/CodeModeTests/ExecutionWatchdogTests.swift index 1e6012d..06e2545 100644 --- a/Tests/CodeModeTests/ExecutionWatchdogTests.swift +++ b/Tests/CodeModeTests/ExecutionWatchdogTests.swift @@ -2,171 +2,205 @@ import Foundation import Testing @testable import CodeMode -@Test func executeTerminatesCPUBoundInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - let started = Date() - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 300 +/// Serialized: each of these spins a core at 100% for its budget. Seven at once on a +/// three-core runner is a stall, not a test. +@Suite(.serialized) struct ExecutionWatchdogTests { + + @Test func executeTerminatesCPUBoundInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 300 + ) ) - ) - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) -} + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A hang-guard, not a performance assertion: `while (true) {}` must be + // terminated at all. The error code above is the real check, and the bound is + // loose because a shared CI runner is not an idle machine. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - await Promise.resolve(); - while (true) {} - """, - allowedCapabilities: [], - timeoutMs: 300 - ) - ) + @Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") -} + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await Promise.resolve(); + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 300 + ) + ) -@Test func executeTimeoutTerminationIsNotCatchableByScript() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - try { - while (true) {} - } catch (error) { - return { swallowed: true }; - } - """, - allowedCapabilities: [], - timeoutMs: 300 + @Test func executeTimeoutTerminationIsNotCatchableByScript() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + try { + while (true) {} + } catch (error) { + return { swallowed: true }; + } + """, + allowedCapabilities: [], + timeoutMs: 300 + ) ) - ) - #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") -} + #expect(observed.result == nil) + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } -@Test func executeRecoversWithFreshContextAfterTerminatedLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func executeRecoversWithFreshContextAfterTerminatedLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - let timedOut = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 200 + let timedOut = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 200 + ) ) - ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") - - let recovered = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "return { ok: true };", - allowedCapabilities: [] + #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + + let recovered = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { ok: true };", + allowedCapabilities: [] + ) ) - ) - let payload = try requireJSONObject(from: try #require(recovered.result)) - #expect(payload["ok"] as? Bool == true) -} + let payload = try requireJSONObject(from: try #require(recovered.result)) + #expect(payload["ok"] as? Bool == true) + } -@Test func cancelInterruptsRunningInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func cancelInterruptsRunningInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - let call = try await tools.executeJavaScript( - JavaScriptExecutionRequest( - code: "while (true) {}", - allowedCapabilities: [], - timeoutMs: 60_000 + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "while (true) {}", + allowedCapabilities: [], + timeoutMs: 60_000 + ) ) - ) - try await Task.sleep(nanoseconds: 200_000_000) - call.cancel() - let observed = await observe(call) + try await Task.sleep(nanoseconds: 200_000_000) + call.cancel() + let observed = await observe(call) - #expect(observed.result == nil) - #expect(observed.error?.code == "CANCELLED") -} + #expect(observed.result == nil) + #expect(observed.error?.code == "CANCELLED") + } -@Test func executeReturnsResultThatSettlesRightAtDeadline() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - // A short CPU spin that finishes well under the timeout must return its - // value, not be discarded by a deadline check that races the settled state. - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: """ - let total = 0; - for (let i = 0; i < 200000; i++) { total += i; } - return { total }; - """, - allowedCapabilities: [], - timeoutMs: 2_000 + @Test func executeReturnsResultThatSettlesRightAtDeadline() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A short CPU spin that finishes well under the timeout must return its + // value, not be discarded by a deadline check that races the settled state. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + let total = 0; + for (let i = 0; i < 200000; i++) { total += i; } + return { total }; + """, + allowedCapabilities: [], + timeoutMs: 2_000 + ) ) - ) - let payload = try requireJSONObject(from: try #require(observed.result)) - #expect(payload["total"] != nil) - #expect(observed.error == nil) -} + let payload = try requireJSONObject(from: try #require(observed.result)) + #expect(payload["total"] != nil) + #expect(observed.error == nil) + } + + @Test func executeTerminatesInfiniteGetterDuringResultSerialization() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The result settles fine, but serializing it runs a runaway getter. The + // watchdog must still bound that phase instead of hanging the thread. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 300 + ) + ) + + #expect(observed.result?.output == nil) + // The watchdog fired, so this is a timeout — not the INVALID_RESULT + // "must be JSON-serializable" it used to report, which pointed the model at + // the wrong repair entirely. + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Same hang-guard reasoning: the runaway getter must be terminated at all. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func executeTerminatesInfiniteGetterDuringResultSerialization() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } - - // The result settles fine, but serializing it runs a runaway getter. The - // watchdog must still bound that phase instead of hanging the thread. - let started = Date() - let observed = try await execute( - tools, - request: JavaScriptExecutionRequest( - code: "return { get trap() { while (true) {} } };", - allowedCapabilities: [], - timeoutMs: 300 + @Test func serializationBudgetDoesNotScaleWithTheExecutionTimeout() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A long execution budget used to buy an equally long serialization budget + // (`max`, not `min`), so a runaway getter after a 30s-budget script got + // another 30s. The serialization phase is bounded independently. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 30_000 + ) ) - ) - #expect(observed.result?.output == nil) - #expect(observed.error != nil) - #expect(Date().timeIntervalSince(started) < 5) -} + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Proves serialization did not inherit the script's 30s budget. + #expect(Date().timeIntervalSince(started) < 20) + } -@Test func searchTerminatesCPUBoundInfiniteLoop() async throws { - let (tools, sandbox) = try makeTools() - defer { cleanup(sandbox) } + @Test func searchTerminatesCPUBoundInfiniteLoop() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } - do { - _ = try await tools.searchJavaScriptAPI( - JavaScriptAPISearchRequest( - code: "async () => { while (true) {} }" + do { + _ = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: "async () => { while (true) {} }" + ) ) - ) - Issue.record("Expected search to time out") - } catch let error as CodeModeToolError { - #expect(error.code == "SEARCH_TIMEOUT") + Issue.record("Expected search to time out") + } catch let error as CodeModeToolError { + #expect(error.code == "SEARCH_TIMEOUT") + } } } diff --git a/Tests/CodeModeTests/FileSystemBridgeTests.swift b/Tests/CodeModeTests/FileSystemBridgeTests.swift index 2cae75c..f44ff53 100644 --- a/Tests/CodeModeTests/FileSystemBridgeTests.swift +++ b/Tests/CodeModeTests/FileSystemBridgeTests.swift @@ -239,3 +239,434 @@ private final class RecordingCodeModeFileSystem: CodeModeFileSystem, @unchecked let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["text"] as? String == "fs-from-execute") } + +// MARK: - Destructive-destination guards + +@Test func fileSystemMoveRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + _ = try fs.write(arguments: ["path": .string("documents:keepme.txt"), "data": .string("precious")], context: context) + + // `documents:` resolves to the documents root itself, and containment admits + // it. The old unconditional removeItem here deleted the user's Documents. + for destination in ["documents:", "documents:/", "tmp:", "caches:"] { + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string(destination), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + Issue.record("Expected fs.move to refuse the root destination \(destination)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + + let survived = try fs.read(arguments: ["path": .string("documents:keepme.txt")], context: context) + #expect(try requireObject(survived).string("text") == "precious") +} + +@Test func fileSystemCopyRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string("documents:"), + "overwrite": .bool(true), + ], context: context) + } +} + +@Test func fileSystemDeleteRefusesASandboxRoot() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.delete(arguments: ["path": .string("documents:"), "recursive": .bool(true)], context: context) + Issue.record("Expected fs.delete to refuse a sandbox root") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.path)) +} + +@Test func fileSystemMoveAndCopyRequireExplicitOverwrite() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:b.txt"), "data": .string("destination")], context: context) + + do { + _ = try fs.copy(arguments: ["from": .string("tmp:a.txt"), "to": .string("tmp:b.txt")], context: context) + Issue.record("Expected fs.copy to refuse an existing destination without overwrite") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // The destination is untouched by the refused call. + let intact = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(intact).string("text") == "destination") + + _ = try fs.copy(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:b.txt"), + "overwrite": .bool(true), + ], context: context) + let replaced = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "source") +} + +@Test func fileSystemMoveOntoADirectoryAlsoRequiresRecursive() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dir/kept.txt"), "data": .string("kept")], context: context) + + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + ], context: context) + Issue.record("Expected fs.move onto a directory to require recursive=true") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == true) + + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == false) +} + +// MARK: - Size and decoding limits + +@Test func fileSystemReadRefusesFilesOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 64, maxWriteBytes: 1_024)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: [ + "path": .string("tmp:big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:big.txt")], context: context) + Issue.record("Expected fs.read to refuse a file over the read limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } +} + +@Test func fileSystemWriteRefusesPayloadsOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 16)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.write(arguments: [ + "path": .string("tmp:nested/big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + Issue.record("Expected fs.write to refuse a payload over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // A refused write leaves no directory behind. + #expect(try fs.exists(arguments: ["path": .string("tmp:nested")], context: context).boolValue == false) +} + +@Test func fileSystemReadReportsUndecodableUTF8InsteadOfEmptyText() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // 0xFF is never valid UTF-8. Returning "" here told the script the file was + // empty; it must be an error the script can repair with base64. + let binary = Data([0xFF, 0xFE, 0x00, 0x01]) + _ = try fs.write(arguments: [ + "path": .string("tmp:binary.bin"), + "data": .string(binary.base64EncodedString()), + "encoding": .string("base64"), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:binary.bin")], context: context) + Issue.record("Expected fs.read to fail on undecodable UTF-8") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + let base64 = try fs.read(arguments: [ + "path": .string("tmp:binary.bin"), + "encoding": .string("base64"), + ], context: context) + #expect(try requireObject(base64).string("base64") == binary.base64EncodedString()) +} + +// MARK: - Source-side guards and non-destructive replacement + +@Test func fileSystemMoveRefusesASandboxRootAsSource() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("documents:precious.txt"), "data": .string("keep")], context: context) + + // The destination-side guard alone let this move Documents into tmp:, which + // the OS may purge. + do { + _ = try fs.move(arguments: ["from": .string("documents:"), "to": .string("tmp:stolen")], context: context) + Issue.record("Expected fs.move to refuse a sandbox root as its source") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.appendingPathComponent("precious.txt").path)) +} + +@Test func fileSystemCopyRefusesASandboxRootAsSource() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.copy(arguments: ["from": .string("documents:"), "to": .string("tmp:copy")], context: context) + Issue.record("Expected fs.copy to refuse a sandbox root as its source") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } +} + +@Test func fileSystemOverwriteWithAMissingSourceKeepsTheDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("documents:important.txt"), "data": .string("keep")], context: context) + + // Previously: destination removed, then the move failed on the missing source. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: [ + "from": .string("tmp:does-not-exist"), + "to": .string("documents:important.txt"), + "overwrite": .bool(true), + ], context: context) + } + let survived = try fs.read(arguments: ["path": .string("documents:important.txt")], context: context) + #expect(try requireObject(survived).string("text") == "keep") +} + +@Test func fileSystemOverwriteOntoItselfIsRefusedAndKeepsTheFile() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:x.txt"), "data": .string("keep")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + + // from == to with overwrite: true used to delete the source. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: ["from": .string("tmp:x.txt"), "to": .string("tmp:x.txt"), "overwrite": .bool(true)], context: context) + } + #expect(try fs.exists(arguments: ["path": .string("tmp:x.txt")], context: context).boolValue == true) + + // A destination inside the source can never succeed either. + #expect(throws: (any Error).self) { + _ = try fs.move(arguments: ["from": .string("tmp:dir"), "to": .string("tmp:dir/inner")], context: context) + } +} + +/// Fails the copy itself, after the destination has been moved aside — the case +/// the old remove-then-act order turned into data loss. +private final class FailingCopyFileSystem: CodeModeFileSystem, @unchecked Sendable { + private let base = LocalCodeModeFileSystem() + func listDirectory(at url: URL) throws -> [CodeModeFileSystemEntry] { try base.listDirectory(at: url) } + func readData(at url: URL) throws -> Data { try base.readData(at: url) } + func writeData(_ data: Data, to url: URL) throws { try base.writeData(data, to: url) } + func moveItem(at sourceURL: URL, to destinationURL: URL) throws { try base.moveItem(at: sourceURL, to: destinationURL) } + func copyItem(at sourceURL: URL, to destinationURL: URL) throws { + throw CocoaError(.fileWriteOutOfSpace) + } + func removeItem(at url: URL) throws { try base.removeItem(at: url) } + func createDirectory(at url: URL, recursive: Bool) throws { try base.createDirectory(at: url, recursive: recursive) } + func attributesOfItem(at url: URL) throws -> CodeModeFileSystemAttributes { try base.attributesOfItem(at: url) } + func itemExists(at url: URL) -> Bool { base.itemExists(at: url) } + func access(at url: URL) -> CodeModeFileSystemAccess { base.access(at: url) } +} + +@Test func fileSystemFailedReplacementRestoresTheDestination() throws { + let fs = FileSystemBridge(fileSystem: FailingCopyFileSystem()) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:src.txt"), "data": .string("new")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dst.txt"), "data": .string("original")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: ["from": .string("tmp:src.txt"), "to": .string("tmp:dst.txt"), "overwrite": .bool(true)], context: context) + } + + let restored = try fs.read(arguments: ["path": .string("tmp:dst.txt")], context: context) + #expect(try requireObject(restored).string("text") == "original") + // And nothing left behind from the aside move. + let leftovers = try FileManager.default.contentsOfDirectory(atPath: sandbox.tmp.path).filter { $0.contains("codemode-replacing") } + #expect(leftovers.isEmpty) +} + +@Test func fileSystemSuccessfulReplacementLeavesNoAsideFile() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:src.txt"), "data": .string("new")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dst.txt"), "data": .string("original")], context: context) + + _ = try fs.move(arguments: ["from": .string("tmp:src.txt"), "to": .string("tmp:dst.txt"), "overwrite": .bool(true)], context: context) + + let replaced = try fs.read(arguments: ["path": .string("tmp:dst.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "new") + let leftovers = try FileManager.default.contentsOfDirectory(atPath: sandbox.tmp.path).filter { $0.contains("codemode-replacing") } + #expect(leftovers.isEmpty) +} + +@Test func fileSystemCopyAnswersToTheWriteLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 100)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.mkdir(arguments: ["path": .string("tmp:tree/nested")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:tree/a.txt"), "data": .string(String(repeating: "x", count: 60))], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:tree/nested/b.txt"), "data": .string(String(repeating: "y", count: 60))], context: context) + + // Each file is under the limit; the tree is not. + do { + _ = try fs.copy(arguments: ["from": .string("tmp:tree"), "to": .string("tmp:tree-copy")], context: context) + Issue.record("Expected fs.copy to refuse a tree over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:tree-copy")], context: context).boolValue == false) +} + +/// A host's own policy — here, one that narrows the default to two roots. Since +/// `allowedRoots` has no default, it must state its roots, and the bridge's root +/// guards apply to it exactly as to `DefaultPathPolicy`. +private struct TwoRootPolicy: PathPolicy { + let base: DefaultPathPolicy + let roots: [URL] + + func resolve(path: String) throws -> URL { + let url = try base.resolve(path: path) + guard roots.contains(where: { url.path.hasPrefix($0.resolvingSymlinksInPath().path) }) else { + throw BridgeError.pathViolation("outside this host's roots") + } + return url + } + + var allowedRoots: [URL] { roots } +} + +@Test func customPathPoliciesGetTheSameRootGuards() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + let policy = TwoRootPolicy( + base: DefaultPathPolicy(config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents)), + roots: [sandbox.tmp, sandbox.documents] + ) + let context = BridgeInvocationContext( + executionContext: .init(), + allowedCapabilities: Set(CapabilityID.allCases), + pathPolicy: policy, + artifactStore: InMemoryArtifactStore(), + permissionBroker: NoopPermissionBroker(), + auditLogger: SyncAuditLogger(), + transcript: ExecutionTranscript(), + cancellationController: ExecutionCancellationController() + ) + let fs = FileSystemBridge() + _ = try fs.write(arguments: ["path": .string("documents:keep.txt"), "data": .string("x")], context: context) + + for arguments: [String: JSONValue] in [ + ["from": .string("documents:"), "to": .string("tmp:stolen")], + ["from": .string("tmp:keep.txt"), "to": .string("documents:"), "overwrite": .bool(true), "recursive": .bool(true)], + ] { + do { + _ = try fs.move(arguments: arguments, context: context) + Issue.record("Expected the custom policy's root to be refused for \(arguments)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.appendingPathComponent("keep.txt").path)) +} + +// MARK: - Hidden entries count against the copy limit + +@Test func fileSystemCopyCountsHiddenFilesAgainstTheLimit() throws { + // Found by Codex review: the size walk listed directories with + // `.skipsHiddenFiles` while `copyItem` copies everything, so a 1,024-byte + // hidden file copied under a 4-byte limit. + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 4_096, maxWriteBytes: 4)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + let tree = sandbox.tmp.appendingPathComponent("tree") + try FileManager.default.createDirectory(at: tree, withIntermediateDirectories: true) + try Data(count: 1_024).write(to: tree.appendingPathComponent(".hidden")) + + do { + _ = try fs.copy(arguments: ["from": .string("tmp:tree"), "to": .string("tmp:tree-copy")], context: context) + Issue.record("Expected a hidden file over the limit to be refused") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(FileManager.default.fileExists(atPath: sandbox.tmp.appendingPathComponent("tree-copy").path) == false) +} + +@Test func fileSystemCopyCountsFilesInsideHiddenDirectories() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 4_096, maxWriteBytes: 100)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + // A visible file under the limit, and a hidden subtree that pushes it over. + let hiddenDir = sandbox.tmp.appendingPathComponent("tree/.cache/nested") + try FileManager.default.createDirectory(at: hiddenDir, withIntermediateDirectories: true) + try Data(count: 10).write(to: sandbox.tmp.appendingPathComponent("tree/visible.txt")) + try Data(count: 500).write(to: hiddenDir.appendingPathComponent("blob")) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: ["from": .string("tmp:tree"), "to": .string("tmp:tree-copy")], context: context) + } +} + +@Test func fileSystemListStillHidesHiddenEntries() throws { + // Display behavior is unchanged: the filtering moved from the storage layer + // to fs.list, where it no longer hides anything from limit accounting. + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + _ = try fs.write(arguments: ["path": .string("tmp:dir/visible.txt"), "data": .string("v")], context: context) + try Data("h".utf8).write(to: sandbox.tmp.appendingPathComponent("dir/.secret")) + + let names = try requireArray(fs.list(arguments: ["path": .string("tmp:dir")], context: context)) + .compactMap { $0.objectValue?.string("name") } + #expect(names == ["visible.txt"]) + + // While the storage layer reports everything, marked. + let entries = try LocalCodeModeFileSystem().listDirectory(at: sandbox.tmp.appendingPathComponent("dir")) + #expect(entries.first { $0.name == ".secret" }?.isHidden == true) + #expect(entries.first { $0.name == "visible.txt" }?.isHidden == false) +} diff --git a/Tests/CodeModeTests/HostConfigurationValidatorTests.swift b/Tests/CodeModeTests/HostConfigurationValidatorTests.swift index 671aa27..f35cf2f 100644 --- a/Tests/CodeModeTests/HostConfigurationValidatorTests.swift +++ b/Tests/CodeModeTests/HostConfigurationValidatorTests.swift @@ -201,3 +201,21 @@ import Testing #expect(issues.contains(where: { $0.key == "Apple Pay merchant configuration" && $0.severity == .warning })) #expect(issues.contains(where: { $0.key == "StoreKit configuration" && $0.severity == .warning })) } + +// MARK: - Configuration validation + +@Test func validatorWarnsWhenTheCapabilityGrantIsUnrestricted() { + // `.unrestricted` is the default for source compatibility, but it means the + // model-authored allowlist is the only one in force. A host should be told. + let issues = HostConfigurationValidator.validate(configuration: CodeModeConfiguration()) + let issue = issues.first { $0.key == "capabilityGrant" } + #expect(issue?.severity == .warning) + #expect(issue?.message.contains(".unrestricted") == true) +} + +@Test func validatorIsQuietWhenAGrantIsSet() { + for grant in [CapabilityGrant.only([.fsRead]), .none] { + let issues = HostConfigurationValidator.validate(configuration: CodeModeConfiguration(capabilityGrant: grant)) + #expect(issues.isEmpty, "unexpected issues for grant \(grant): \(issues)") + } +} diff --git a/Tests/CodeModeTests/HostRuntimeTests.swift b/Tests/CodeModeTests/HostRuntimeTests.swift index 158798a..489db5d 100644 --- a/Tests/CodeModeTests/HostRuntimeTests.swift +++ b/Tests/CodeModeTests/HostRuntimeTests.swift @@ -602,7 +602,11 @@ import Testing ) #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A promise with no resolve path and no pending timer is provably + // unsettleable under this runtime, so it is reported immediately instead of + // sleeping a thread until the deadline for a result that can never arrive. + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) } @Test func executeRecoversWithFreshContextAfterTimeout() async throws { @@ -617,7 +621,7 @@ import Testing timeoutMs: 30 ) ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + #expect(timedOut.error?.code == "JS_RUNTIME_ERROR") let recovered = try await execute( tools, @@ -700,16 +704,33 @@ import Testing let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } + // A long timer, not an unsettleable promise: the latter now fails fast with + // JS_RUNTIME_ERROR, which would win the race against the cancellation this + // test is about. let call = try await tools.executeJavaScript( JavaScriptExecutionRequest( code: """ - await new Promise(() => {}); + console.log('running'); + await new Promise(resolve => setTimeout(resolve, 20000)); return { never: true }; """, - allowedCapabilities: [] + allowedCapabilities: [], + timeoutMs: 60_000 ) ) + // Cancel only once the script is observed to be running. A `Task.yield()` + // before cancelling is not a guarantee under load: on a stalled runner the + // test task can be descheduled long enough for the 20s timer to elapse, and + // then a cancel of an already-finished execution is a no-op that this test + // would misreport as a runtime bug. + var running = false + for await event in call.events { + if case let .log(entry) = event, entry.message == "running" { running = true; break } + if case .finished = event { break } + } + #expect(running, "the script never reported starting") + let waiter = Task, Never> { do { return .success(try await call.result) @@ -720,7 +741,6 @@ import Testing } } - await Task.yield() waiter.cancel() let waiterResult = await waiter.value diff --git a/Tests/CodeModeTests/KeychainBridgeTests.swift b/Tests/CodeModeTests/KeychainBridgeTests.swift index 46b6946..722467f 100644 --- a/Tests/CodeModeTests/KeychainBridgeTests.swift +++ b/Tests/CodeModeTests/KeychainBridgeTests.swift @@ -1,8 +1,9 @@ import Foundation import Testing @testable import CodeMode +import Security -@Test func keychainRoundTripWriteReadDelete() throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainRoundTripWriteReadDelete() throws { let bridge = KeychainBridge(service: "CodeModeTests.\(UUID().uuidString)") let key = "token" @@ -20,7 +21,7 @@ import Testing #expect(postDelete == .null) } -@Test func keychainReadMissingValueReturnsNull() throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainReadMissingValueReturnsNull() throws { let bridge = KeychainBridge(service: "CodeModeTests.\(UUID().uuidString)") let readValue = try bridge.read(arguments: ["key": .string("missing")]) #expect(readValue == .null) @@ -37,7 +38,7 @@ import Testing } } -@Test func executeUsesKeychainBridge() async throws { +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func executeUsesKeychainBridge() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -58,3 +59,15 @@ import Testing let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["value"] as? String == "value-from-execute") } + +@Test func keychainFailuresDescribeTheStatusInsteadOfPrintingIt() { + // -34018 is the one host developers hit in their own unsigned test targets, + // and the model must learn it is not retryable. + let missing = KeychainBridge.failure("keychain.write", status: errSecMissingEntitlement) + #expect(missing.localizedDescription.contains("no keychain entitlement")) + #expect(missing.localizedDescription.contains("retrying will not help")) + + let other = KeychainBridge.failure("keychain.read", status: errSecDuplicateItem) + #expect(other.localizedDescription.contains("keychain.read failed:")) + #expect(other.localizedDescription.contains("(\(errSecDuplicateItem))")) +} diff --git a/Tests/CodeModeTests/NetworkBridgeTests.swift b/Tests/CodeModeTests/NetworkBridgeTests.swift index 5e444ae..ec72f4e 100644 --- a/Tests/CodeModeTests/NetworkBridgeTests.swift +++ b/Tests/CodeModeTests/NetworkBridgeTests.swift @@ -227,3 +227,74 @@ private final class StubHTTPURLProtocol: URLProtocol { #expect(result.string("stringValue") == "https://example.com/path?q=1") #expect(result.string("methodValue") == "https://example.com/path?q=1") } + +// MARK: - Ambient authority + +@Test func networkBridgeDefaultSessionCarriesNoAmbientCredentials() { + // The shipping default used to be `URLSession.shared`, which reads + // HTTPCookieStorage.shared and URLCredentialStorage.shared — so a script got + // authenticated session-riding against every origin the app is logged in to, + // and a Set-Cookie in a script-fetched response poisoned the app's jar. + let configuration = NetworkBridge.isolatedSession.configuration + #expect(configuration.httpCookieStorage == nil) + #expect(configuration.urlCredentialStorage == nil) + #expect(configuration.urlCache == nil) + #expect(configuration.httpShouldSetCookies == false) + #expect(configuration.httpCookieAcceptPolicy == .never) + #expect(NetworkBridge.isolatedSession !== URLSession.shared) +} + +@Test func networkFetchRefusesCredentialHeadersByDefault() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge(session: session) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + for header in ["Cookie", "cookie", "Authorization", "Proxy-Authorization", "proxy-authenticate"] { + do { + _ = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object([header: .string("secret")])]), + ], context: context) + Issue.record("Expected \(header) to be refused") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } + + // Ordinary headers are unaffected. + let ok = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["X-Unit": .string("plain")])]), + ], context: context) + #expect(try requireObject(ok).bool("ok") == true) +} + +@Test func networkFetchAllowsCredentialHeadersWhenHostOptsIn() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge( + session: session, + policy: NetworkAccessPolicy(allowsCredentialHeaders: true) + ) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + let result = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["Authorization": .string("Bearer t")])]), + ], context: context) + #expect(try requireObject(result).bool("ok") == true) + #expect(NetworkAccessPolicy.permissive.allowsCredentialHeaders) +} diff --git a/Tests/CodeModeTests/PolishFindingsTests.swift b/Tests/CodeModeTests/PolishFindingsTests.swift new file mode 100644 index 0000000..ac3f7cb --- /dev/null +++ b/Tests/CodeModeTests/PolishFindingsTests.swift @@ -0,0 +1,143 @@ +import Foundation +import Testing +@testable import CodeMode + +// MARK: - One ISO 8601 parser + +@Test func iso8601ParsingAcceptsFractionalSeconds() { + // `Date.toISOString()` in JavaScript always emits milliseconds, so this is + // the spelling a model produces by default. A bare ISO8601DateFormatter + // rejects it, and the bridges that used one silently substituted "now". + #expect(CodeModeDate.parse("2026-07-24T10:00:00.000Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00+01:00") != nil) + #expect(CodeModeDate.parse(" 2026-07-24T10:00:00Z ") != nil) + #expect(CodeModeDate.parse("not a date") == nil) + #expect(CodeModeDate.parse("") == nil) + #expect(CodeModeDate.parse(nil) == nil) +} + +@Test func requiredDatesFailInsteadOfSubstitutingADefault() { + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require("yesterday", argument: "start", capability: "calendar.read") + } + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require(nil, argument: "start", capability: "calendar.read") + } + // A present-but-unparseable optional is still an error; only absence defaults. + #expect(throws: (any Error).self) { + _ = try CodeModeDate.optional("soon", argument: "end", capability: "calendar.read") + } +} + +@Test func absentOptionalDatesStillFallBackToTheCallersDefault() throws { + let absent = try CodeModeDate.optional(nil, argument: "end", capability: "calendar.read") + #expect(absent == nil) +} + +// MARK: - Network policy + +@Test func standardPolicyBlocksBareSingleLabelHosts() { + let policy = NetworkAccessPolicy.standard + // These resolve onto the LAN through the DHCP search domain, exactly like + // *.local, but previously fell through to "allowed". + for host in ["router", "intranet", "wpad", "nas", "printer"] { + let url = URL(string: "http://\(host)/admin")! + #expect(policy.violationReason(for: url) != nil, "expected \(host) to be refused") + } + + // Ordinary public hosts are unaffected. + #expect(policy.violationReason(for: URL(string: "https://example.com/x")!) == nil) + #expect(policy.violationReason(for: URL(string: "https://api.example.co.uk/x")!) == nil) +} + +@Test func hostsCanStillAllowlistASingleLabelHost() { + let policy = NetworkAccessPolicy(allowedHosts: ["router"]) + #expect(policy.violationReason(for: URL(string: "http://router/admin")!) == nil) +} + +// MARK: - System UI destinations + +@Test func webPresentationRespectsTheNetworkAccessPolicy() throws { + let bridge = SystemUIBridge(networkAccessPolicy: .standard) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // Opening a page in a browser was exempt from the policy governing fetch, so + // a blocked origin could still be put in front of the user — or be the target + // of a script-started OAuth flow. + for capability in ["web", "auth"] { + let arguments: [String: JSONValue] = ["url": .string("http://169.254.169.254/latest/meta-data")] + do { + if capability == "web" { + _ = try bridge.presentWeb(arguments: arguments, context: context) + } else { + _ = try bridge.authenticateWeb(arguments: arguments, context: context) + } + Issue.record("Expected \(capability) presentation to be refused by the network policy") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } +} + +// MARK: - Path containment + +@Test func danglingSymlinksDoNotPassContainment() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let outsideRoot = sandbox.root.appendingPathComponent("outside", isDirectory: true) + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + + // The target deliberately does not exist: `fileExists` follows symlinks and + // so reported the link as a *missing component*, which was re-appended to the + // resolved ancestor and admitted instead of being resolved out of the root. + let link = sandbox.tmp.appendingPathComponent("escape") + try FileManager.default.createSymbolicLink( + at: link, + withDestinationURL: outsideRoot.appendingPathComponent("secret.txt") + ) + + #expect(throws: (any Error).self) { + _ = try policy.resolve(path: "tmp:escape") + } +} + +@Test func allowedRootsAreReportedByTheDefaultPolicy() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + let documentsRoot = try policy.resolve(path: "documents:") + let tmpRoot = try policy.resolve(path: "tmp:") + let insideTmp = try policy.resolve(path: "tmp:file.txt") + + #expect(policy.allowedRoots.count == 3) + #expect(policy.isAllowedRoot(documentsRoot)) + #expect(policy.isAllowedRoot(tmpRoot)) + #expect(policy.isAllowedRoot(insideTmp) == false) +} + +// MARK: - Diagnostics reach the stream + +@Test func errorSeverityDiagnosticsAreEmittedToTheEventStream() { + let received = LockedBox<[ToolDiagnostic]>([]) + let transcript = ExecutionTranscript { event in + if case let .diagnostic(diagnostic) = event { + received.set(received.get() + [diagnostic]) + } + } + + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "I", message: "info")) + transcript.record(diagnostic: ToolDiagnostic(severity: .warning, code: "W", message: "warning")) + // Previously filtered out, so a host watching the stream saw every diagnostic + // except the ones that mattered most. + transcript.record(diagnostic: ToolDiagnostic(severity: .error, code: "E", message: "error")) + + #expect(received.get().map(\.code) == ["I", "W", "E"]) +} diff --git a/Tests/CodeModeTests/TestSupport.swift b/Tests/CodeModeTests/TestSupport.swift index 3e2a23a..4dc756b 100644 --- a/Tests/CodeModeTests/TestSupport.swift +++ b/Tests/CodeModeTests/TestSupport.swift @@ -1,6 +1,26 @@ import Foundation import Testing @testable import CodeMode +import Security + +/// Whether this process can use the keychain at all. An unsigned SwiftPM test +/// bundle on the iOS simulator has no keychain access group, so every keychain +/// call fails with `errSecMissingEntitlement` (-34018). Tests whose subject *is* +/// keychain storage skip there, with that reason; tests that only touch the +/// keychain incidentally assert their real subject instead. +let keychainIsAvailable: Bool = { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: "CodeModeTests.keychain-probe", + kSecAttrAccount as String: UUID().uuidString, + kSecValueData as String: Data("probe".utf8), + ] + let status = SecItemAdd(query as CFDictionary, nil) + if status == errSecSuccess { + SecItemDelete(query as CFDictionary) + } + return status != errSecMissingEntitlement +}() struct TestSandbox { let root: URL @@ -35,6 +55,8 @@ func cleanup(_ sandbox: TestSandbox) { func makeTools( permissionBroker: any PermissionBroker = NoopPermissionBroker(), + capabilityGrant: CapabilityGrant = .unrestricted, + executionLimits: ExecutionLimits = .standard, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), systemUIPresenter: any SystemUIPresenter = UnavailableSystemUIPresenter(), eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), @@ -49,7 +71,8 @@ func makeTools( passKitClient: any PassKitClient = UnavailablePassKitClient(), storeKitClient: any StoreKitClient = UnavailableStoreKitClient(), codeModeProviders: [any CodeModeProvider] = [], - hostPlatform: HostPlatform = .current + hostPlatform: HostPlatform = .current, + clock: any RuntimeClock = RealClock() ) throws -> (CodeModeAgentTools, TestSandbox) { let sandbox = try makeTestSandbox() @@ -59,7 +82,9 @@ func makeTools( let configuration = CodeModeConfiguration( pathPolicy: pathPolicy, + capabilityGrant: capabilityGrant, fileSystem: fileSystem, + executionLimits: executionLimits, artifactStore: InMemoryArtifactStore(), permissionBroker: permissionBroker, auditLogger: SyncAuditLogger(), @@ -79,7 +104,7 @@ func makeTools( hostPlatform: hostPlatform ) - let tools = CodeModeAgentTools(config: configuration) + let tools = CodeModeAgentTools(config: configuration, clock: clock) return (tools, sandbox) } diff --git a/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift new file mode 100644 index 0000000..e7e9f9a --- /dev/null +++ b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift @@ -0,0 +1,324 @@ +import Foundation +import Testing +@testable import CodeMode + +// Cloudflare's central Code Mode insight is that models write markedly better +// code against real types. Before this the model got a flat `argumentTypes` map +// of six coarse cases plus a prose `resultSummary`; these tests pin the shape of +// what it gets now. + +private func reference( + capability: String = "fs.read", + jsNames: [String] = ["apple.fs.read", "fs.promises.readFile"], + required: [String] = ["path"], + optional: [String] = ["encoding"], + types: [String: CapabilityArgumentType] = ["path": .string, "encoding": .string], + hints: [String: String] = ["path": "File path using allowed root prefix."], + constraints: CapabilityArgumentConstraints = .init(allowedStringValues: ["encoding": ["utf8", "base64"]]) +) -> JavaScriptAPIReference { + JavaScriptAPIReference( + capability: capability, + jsNames: jsNames, + summary: "Read a file.", + tags: ["fs"], + example: "await apple.fs.read({ path: 'tmp:x.txt' })", + requiredArguments: required, + optionalArguments: optional, + argumentTypes: types, + argumentHints: hints, + argumentConstraints: constraints, + resultSummary: "Object with path plus text or base64." + ) +} + +@Test func declarationsTypeArgumentsAndCarryHintsAsDocComments() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + + #expect(dts.contains("path: string;")) + #expect(dts.contains("/** File path using allowed root prefix. */")) + // Required vs optional is expressed in the type, not left to prose. + #expect(dts.contains("encoding?:")) + #expect(dts.contains("path?:") == false) + #expect(dts.contains("Capability: fs.read")) + #expect(dts.contains("Returns: Object with path plus text or base64.")) +} + +@Test func constrainedArgumentsBecomeStringLiteralUnions() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // The constraint IS the type; a union beats `string` plus a prose hint the + // model has to notice and obey. + #expect(dts.contains(#"encoding?: "utf8" | "base64";"#)) +} + +@Test func nodeStyleAliasesAreDocumentedRatherThanMistypedAsObjectCalls() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // `fs.promises.readFile` is positional, which the catalog cannot express, so + // the generated signature must describe the namespaced helper and only + // mention the alias. + #expect(dts.contains("Aliases: fs.promises.readFile")) + #expect(dts.contains("positional")) +} + +@Test func dottedArgumentPathsBecomeNestedObjectTypes() { + let nested = reference( + capability: "network.fetch", + jsNames: ["fetch"], + required: ["url"], + optional: ["options.method", "options.timeoutMs"], + types: ["url": .string, "options.method": .string, "options.timeoutMs": .number], + hints: ["options.method": "HTTP method; defaults to GET."], + constraints: .none + ) + let dts = TypeScriptDeclarations.declaration(for: nested) + + #expect(dts.contains("options?: {")) + #expect(dts.contains("method?: string;")) + #expect(dts.contains("timeoutMs?: number;")) +} + +@Test func argumentlessHelpersTakeNoArgument() { + let none = reference( + capability: "location.permission.request", + jsNames: ["apple.location.requestPermission"], + required: [], + optional: [], + types: [:], + hints: [:], + constraints: .none + ) + #expect(TypeScriptDeclarations.declaration(for: none).contains("(): Promise")) +} + +@Test func theWholeSurfaceIsNamespacedAndPlatformFiltered() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .macOS) + defer { cleanup(sandbox) } + + let surface = tools.typeDeclarations() + + #expect(surface.contains("declare const apple: {")) + #expect(surface.contains("fs: {")) + #expect(surface.contains("read(args: {")) + // The Node-compat globals live in the hand-authored preamble, because their + // positional convention is not something the catalog can describe. + #expect(surface.contains("declare function fetch(")) + #expect(surface.contains("type CodeModeValue =")) + // iOS-only helpers must not be declared for a macOS host. + #expect(surface.contains("alarm: {") == false) +} + +@Test func iOSOnlyHelpersAppearForAniOSHost() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + #expect(tools.typeDeclarations().contains("alarm: {")) +} + +@Test func everyCapabilityCarriesItsOwnDeclarationForSearchResults() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let references = tools.capabilities() + #expect(references.isEmpty == false) + #expect(references.allSatisfy { $0.dts.isEmpty == false }) + + let read = try #require(references.first { $0.capability == "fs.read" }) + #expect(read.dts.contains("path: string;")) +} + +@Test func searchCanReturnDeclarationsAsItsPayload() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Code-driven search stays the filter; the payload is now TypeScript. + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: """ + async () => { + return api.references + .filter(ref => ref.tags.includes("filesystem")) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """ + ) + ) + let payload = try #require(response.result?.stringValue) + #expect(payload.contains("declare const apple: {")) + #expect(payload.contains("read(args: {")) + #expect(payload.contains("path: string;")) +} + +@Test func nodeAliasesCanPassTheOverwriteFlagTheDeclarationPromises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // fs.move/fs.copy now refuse an existing destination without an explicit + // overwrite, so the Node-style aliases had to gain a way to pass one — the + // preamble declares it, and this proves the runtime honours it. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:src.txt', data: 'source' }); + await apple.fs.write({ path: 'tmp:dst.txt', data: 'destination' }); + await fs.promises.copyFile('tmp:src.txt', 'tmp:dst.txt', { overwrite: true }); + return await fs.promises.readFile('tmp:dst.txt', 'utf8'); + """, + allowedCapabilities: [.fsWrite, .fsRead, .fsCopy] + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("source")) +} + +@Test(.enabled(if: keychainIsAvailable, "no keychain entitlement in this test host (errSecMissingEntitlement)")) func keychainHelpersAcceptTheObjectFormTheCatalogAdvertises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The catalog declared `key: string` as an object argument while the wrapper + // took a positional string, and its own example showed the positional form — + // so code written from the metadata sent "[object Object]" as the key. Both + // spellings now work, which is what makes the generated declaration true. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.keychain.set({ key: 'codemode-dts-test', value: 'object-form' }); + const viaObject = await apple.keychain.get({ key: 'codemode-dts-test' }); + const viaPositional = await apple.keychain.get('codemode-dts-test'); + await apple.keychain.delete({ key: 'codemode-dts-test' }); + return { viaObject: viaObject && viaObject.value, viaPositional: viaPositional && viaPositional.value }; + """, + allowedCapabilities: [.keychainRead, .keychainWrite, .keychainDelete] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.string("viaObject") == "object-form") + #expect(output.string("viaPositional") == "object-form") +} + +@Test func reservedWordHelpersRenderAsValidMemberNames() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + // `apple.keychain.delete` and the `export` helpers used to render as + // `function delete(` / `function export(` — invalid TypeScript. As object + // type members they are legal. + let surface = tools.typeDeclarations() + let reserved = ["break", "case", "catch", "class", "const", "continue", "debugger", "default", "delete", + "do", "else", "enum", "export", "extends", "false", "finally", "for", "function", "if", + "import", "in", "instanceof", "new", "null", "return", "super", "switch", "this", + "throw", "true", "try", "typeof", "var", "void", "while", "with"] + for word in reserved { + #expect(surface.contains("function \(word)(") == false, "reserved word '\(word)' declared as a function") + } + #expect(surface.contains("delete(args: {")) + #expect(surface.contains("declare namespace") == false) +} + +// MARK: - Declarations follow the grant + +@Test func typeDeclarationsFollowTheCapabilityGrant() async throws { + let (granted, grantedSandbox) = try makeTools(capabilityGrant: .only([.fsRead, .fsList]), hostPlatform: .iOS) + defer { cleanup(grantedSandbox) } + let (open, openSandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(openSandbox) } + + let narrow = granted.typeDeclarations() + let full = open.typeDeclarations() + + // Only what the host grants is advertised: never a helper the model would be + // denied, and never 21k tokens of surface when two helpers are permitted. + #expect(narrow.contains("read(args: {")) + #expect(narrow.contains("list(args: {")) + #expect(narrow.contains("keychain: {") == false) + #expect(narrow.contains("write(args: {") == false) + #expect(narrow.contains("declare function fetch("), "the preamble is always included") + #expect(narrow.count * 10 < full.count) +} + +@Test func typeDeclarationsCanBeNarrowedToAnExplicitSubset() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + + let subset = tools.typeDeclarations(for: [.keychainRead]) + #expect(subset.contains("keychain: {")) + #expect(subset.contains("get(args: {")) + // Not `fs: {` — the always-included preamble declares the Node `fs` shim. + #expect(subset.contains("read(args: {") == false) +} + +@Test func anUnrestrictedGrantDeclaresTheWholeSurface() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + let declared = tools.typeDeclarations() + for reference in tools.capabilities() { + let leaf = reference.jsNames.first { $0.contains(".") && !$0.hasPrefix("fs.promises.") }? + .split(separator: ".").last.map(String.init) + if let leaf { + #expect(declared.contains("\(leaf)("), "missing \(reference.capability)") + } + } +} + +// MARK: - Codable compatibility + +@Test func referencesEncodedBeforeDtsExistedStillDecode() throws { + // The shape a host cached before this branch: no `dts` key. The synthesized + // decoder threw keyNotFound on it. + let legacy = #"{"capability":"fs.read","capabilityKey":"fs.read","builtInCapability":"fs.read","jsNames":["apple.fs.read"],"summary":"Read a file.","tags":[],"example":"","requiredArguments":["path"],"optionalArguments":[],"argumentTypes":{"path":"string"},"argumentHints":{},"argumentConstraints":{"allowedStringValues":{}},"resultSummary":"Object."}"# + let decoded = try JSONDecoder().decode(JavaScriptAPIReference.self, from: Data(legacy.utf8)) + #expect(decoded.capability == "fs.read") + // Regenerated rather than left empty. + #expect(decoded.dts.contains("read(args: {")) + #expect(decoded.dts.contains("path: string;")) +} + +@Test func referencesRoundTripThroughCodableUnchanged() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + let original = tools.capabilities() + let decoded = try JSONDecoder().decode([JavaScriptAPIReference].self, from: JSONEncoder().encode(original)) + #expect(decoded == original) +} + +// MARK: - Artifacts for the compiler check + +/// Writes every generated declaration to `$CODEMODE_DTS_OUTPUT` for a real +/// `tsc --strict` pass (`scripts/check-typescript-declarations.sh`, run in CI). +/// The unit tests above check known failure shapes; only a compiler checks +/// that the output is TypeScript at all. Inert unless the variable is set. +@Test(.enabled(if: ProcessInfo.processInfo.environment["CODEMODE_DTS_OUTPUT"] != nil, "set CODEMODE_DTS_OUTPUT to write declarations for tsc")) +func writeTypeDeclarationsForCompilerCheck() async throws { + let output = URL(fileURLWithPath: try #require(ProcessInfo.processInfo.environment["CODEMODE_DTS_OUTPUT"])) + let surfaces = output.appendingPathComponent("surfaces", isDirectory: true) + let fragments = output.appendingPathComponent("fragments", isDirectory: true) + try FileManager.default.createDirectory(at: surfaces, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: fragments, withIntermediateDirectories: true) + + try TypeScriptDeclarations.preamble.write( + to: output.appendingPathComponent("preamble.d.ts"), atomically: true, encoding: .utf8 + ) + + // Every platform's surface, unrestricted, so no capability goes unchecked. + for platform in [HostPlatform.iOS, .macOS, .visionOS] { + let (tools, sandbox) = try makeTools(hostPlatform: platform) + defer { cleanup(sandbox) } + try tools.typeDeclarations().write( + to: surfaces.appendingPathComponent("\(platform).d.ts"), atomically: true, encoding: .utf8 + ) + } + + // Fragments from the widest platform surface. + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + let references = tools.capabilities() + for reference in references { + try reference.dts.write( + to: fragments.appendingPathComponent("\(reference.capability).d.ts"), atomically: true, encoding: .utf8 + ) + } + #expect(references.count > 100, "expected the full iOS catalog") +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8e51716..d28571b 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -532,7 +532,7 @@ }, "argumentHints" : { "callbackURLScheme" : "Optional custom URL scheme that completes the session.", - "prefersEphemeralSession" : "Whether to prefer a private browser session; default false.", + "prefersEphemeralSession" : "Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.", "timeoutMs" : "Optional timeout for waiting on callback\/cancellation.", "url" : "Absolute HTTP(S) authentication URL." }, @@ -559,7 +559,7 @@ ], "resultSummary" : "Object with action callback\/cancelled and callbackURL when available.", - "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication.", + "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy.", "tags" : [ "auth", "oauth", @@ -1852,10 +1852,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -1865,7 +1869,8 @@ "fs.promises.copyFile" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -1875,7 +1880,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Copy file\/directory within allowed sandbox roots.", + "summary" : "Copy file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2028,10 +2033,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -2041,7 +2050,8 @@ "fs.promises.rename" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -2051,7 +2061,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Move file\/directory within allowed sandbox roots.", + "summary" : "Move file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2061,10 +2071,14 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { - "encoding" : "utf8 (default) or base64.", + "encoding" : "utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.", "path" : "File path using allowed root prefix." }, "argumentTypes" : { @@ -2131,7 +2145,11 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { "data" : "UTF-8 text or base64 string depending on encoding.", @@ -2195,7 +2213,7 @@ "requiredPermissions" : [ ], - "resultSummary" : "Object with status\/granted fields and requested type arrays.", + "resultSummary" : "Object with status ('completed'), per-type writeAuthorization, and the requested type arrays. Read authorization is never disclosed by HealthKit, so an empty health.read may mean denial.", "summary" : "Request HealthKit authorization for requested read\/write types.", "tags" : [ "healthkit", @@ -2380,7 +2398,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.delete('auth_token')", + "example" : "await apple.keychain.delete({ key: 'auth_token' })", "id" : "keychain.delete", "jsNames" : [ "apple.keychain.delete" @@ -2413,7 +2431,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.get('auth_token')", + "example" : "await apple.keychain.get({ key: 'auth_token' })", "id" : "keychain.read", "jsNames" : [ "apple.keychain.get" @@ -2448,7 +2466,7 @@ "key" : "string", "value" : "string" }, - "example" : "await apple.keychain.set('auth_token', token)", + "example" : "await apple.keychain.set({ key: 'auth_token', value: token })", "id" : "keychain.write", "jsNames" : [ "apple.keychain.set" @@ -3265,7 +3283,7 @@ "argumentHints" : { "options.body" : "UTF-8 request body string.", "options.bodyBase64" : "Base64-encoded request body. Mutually exclusive with options.body.", - "options.headers" : "Object of header key\/value string pairs.", + "options.headers" : "Object of header key\/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.method" : "HTTP method; defaults to GET.", "options.responseEncoding" : "text (default) or base64.", "options.timeoutMs" : "Request and bridge wait timeout in milliseconds; defaults to 30000.", @@ -3300,7 +3318,7 @@ ], "resultSummary" : "Object with ok\/status\/statusText\/headers\/bodyText or bodyBase64.", - "summary" : "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + "summary" : "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", "tags" : [ "network", "http", diff --git a/scripts/check-typescript-declarations.sh b/scripts/check-typescript-declarations.sh new file mode 100755 index 0000000..d27518d --- /dev/null +++ b/scripts/check-typescript-declarations.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Compiles every TypeScript declaration CodeMode generates with `tsc --strict`. +# +# The Swift tests check known failure shapes (reserved-word function names, for +# one); only a real compiler checks that the output is valid TypeScript at all. +# +# scripts/check-typescript-declarations.sh [output-dir] +# +# TSC overrides the compiler command, e.g. `TSC=tsc` to use a local install. +# Default: TypeScript 7.0.2 via npx. +set -euo pipefail + +cd "$(dirname "$0")/.." +out="${1:-$(mktemp -d)}" +rm -rf "$out" +mkdir -p "$out" +tsc_cmd="${TSC:-npx --yes --package typescript@7.0.2 tsc}" + +echo "Writing declarations to $out" +CODEMODE_DTS_OUTPUT="$out" swift test --filter writeTypeDeclarationsForCompilerCheck + +surfaces=$(find "$out/surfaces" -name '*.d.ts' | wc -l | tr -d ' ') +fragments=$(find "$out/fragments" -name '*.d.ts' | wc -l | tr -d ' ') +# Guard against a vacuous pass: a filter typo or a skipped test would leave +# nothing for tsc to check, and an empty compilation succeeds. +if [ "$surfaces" -ne 3 ] || [ "$fragments" -lt 100 ]; then + echo "error: expected 3 surfaces and 100+ fragments, found $surfaces and $fragments" >&2 + exit 1 +fi + +# One compilation, not 124. The preamble stays a global script so fragments can +# use `CodeModeValue`, as they would in a host prompt. Each surface and fragment +# becomes its own module, so their top-level `declare const apple` stay local +# instead of colliding — each is checked exactly as it would be used alone. +for file in "$out"/surfaces/*.d.ts "$out"/fragments/*.d.ts; do + printf '\nexport {};\n' >> "$file" +done + +cat > "$out/tsconfig.json" <<'JSON' +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "lib": ["es2022"], + "types": [], + "skipLibCheck": false + }, + "include": ["preamble.d.ts", "surfaces/*.d.ts", "fragments/*.d.ts"] +} +JSON + +echo "Compiling $surfaces surfaces and $fragments fragments with: $tsc_cmd" +listed=$($tsc_cmd -p "$out/tsconfig.json" --listFilesOnly | grep -c "$out" || true) +expected=$((surfaces + fragments + 1)) +if [ "$listed" -lt "$expected" ]; then + echo "error: tsc would check $listed of $expected declaration files" >&2 + exit 1 +fi + +$tsc_cmd -p "$out/tsconfig.json" +echo "OK: all $expected declaration files compile under --strict"