diff --git a/.swiftlint-baseline.json b/.swiftlint-baseline.json index 7657a86..bbe45dd 100644 --- a/.swiftlint-baseline.json +++ b/.swiftlint-baseline.json @@ -452,10 +452,10 @@ {"violation":{"ruleDescription":"Enums should be explicitly assigned their raw values","ruleIdentifier":"explicit_enum_raw_value","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":6,"character":48},"reason":"Enums should be explicitly assigned their raw values","severity":"warning","ruleName":"Explicit Enum Raw Value"},"text":" case incompatibleModel, invalidSchema, unavailableTool, invalidContext, invalidVersion"}, {"violation":{"ruleDescription":"Enums should be explicitly assigned their raw values","ruleIdentifier":"explicit_enum_raw_value","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":6,"character":65},"reason":"Enums should be explicitly assigned their raw values","severity":"warning","ruleName":"Explicit Enum Raw Value"},"text":" case incompatibleModel, invalidSchema, unavailableTool, invalidContext, invalidVersion"}, {"violation":{"ruleDescription":"Enums should be explicitly assigned their raw values","ruleIdentifier":"explicit_enum_raw_value","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":6,"character":81},"reason":"Enums should be explicitly assigned their raw values","severity":"warning","ruleName":"Explicit Enum Raw Value"},"text":" case incompatibleModel, invalidSchema, unavailableTool, invalidContext, invalidVersion"}, -{"violation":{"ruleDescription":"Function bodies should not span too many lines","ruleIdentifier":"function_body_length","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":21,"character":5},"reason":"Function body should span 80 lines or less excluding comments and whitespace: currently spans 94 lines","severity":"error","ruleName":"Function Body Length"},"text":" func validationIssues(catalog: ModelCatalog, kits: [String: ToolKit]) -> [WorkflowValidationIssue] {"}, +{"violation":{"ruleDescription":"Function bodies should not span too many lines","ruleIdentifier":"function_body_length","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":21,"character":5},"reason":"Function body should span 40 lines or less excluding comments and whitespace: currently spans 69 lines","severity":"warning","ruleName":"Function Body Length"},"text":" func validationIssues(catalog: ModelCatalog, tools: ToolSources) -> [WorkflowValidationIssue] {"}, {"violation":{"ruleDescription":"Properties should have a type interface","ruleIdentifier":"explicit_type_interface","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":31,"character":13},"reason":"Properties should have a type interface","severity":"warning","ruleName":"Explicit Type Interface"},"text":" let knownModels = Set(catalog.ids)"}, -{"violation":{"ruleDescription":"Complexity of function bodies should be limited.","ruleIdentifier":"cyclomatic_complexity","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":50,"character":9},"reason":"Function should have complexity 8 or less; currently complexity is 33","severity":"error","ruleName":"Cyclomatic Complexity"},"text":" func visit(_ steps: [Step]) {"}, -{"violation":{"ruleDescription":"Function bodies should not span too many lines","ruleIdentifier":"function_body_length","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":50,"character":9},"reason":"Function body should span 40 lines or less excluding comments and whitespace: currently spans 72 lines","severity":"warning","ruleName":"Function Body Length"},"text":" func visit(_ steps: [Step]) {"}, +{"violation":{"ruleDescription":"Complexity of function bodies should be limited.","ruleIdentifier":"cyclomatic_complexity","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":37,"character":9},"reason":"Function should have complexity 8 or less; currently complexity is 20","severity":"error","ruleName":"Cyclomatic Complexity"},"text":" func visit(_ steps: [Step]) {"}, +{"violation":{"ruleDescription":"Function bodies should not span too many lines","ruleIdentifier":"function_body_length","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":37,"character":9},"reason":"Function body should span 40 lines or less excluding comments and whitespace: currently spans 47 lines","severity":"warning","ruleName":"Function Body Length"},"text":" func visit(_ steps: [Step]) {"}, {"violation":{"ruleDescription":"Properties should have a type interface","ruleIdentifier":"explicit_type_interface","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":73,"character":25},"reason":"Properties should have a type interface","severity":"warning","ruleName":"Explicit Type Interface"},"text":" let candidates = [step.model, step.fallback].compactMap { $0 }.filter { knownModels.contains($0) }"}, {"violation":{"ruleDescription":"Else and catch should be on the same line, one space after the previous declaration","ruleIdentifier":"statement_position","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":91,"character":93},"reason":"Else and catch should be on the same line, one space after the previous declaration","severity":"warning","ruleName":"Statement Position"},"text":" if let kit = kits[id] { if kit.server == nil { tools += kit.tools } }"}, {"violation":{"ruleDescription":"Else and catch should be on the same line, one space after the previous declaration","ruleIdentifier":"statement_position","location":{"file":"Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift","line":97,"character":104},"reason":"Else and catch should be on the same line, one space after the previous declaration","severity":"warning","ruleName":"Statement Position"},"text":" if let tool = kit.tools.first(where: { $0.name == name }) { tools = [tool] }"}, diff --git a/CHANGELOG.md b/CHANGELOG.md index c36b733..a87aa7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ - `Field.Kind.object` groups fields, so `extract` steps and tool parameters can ask for lists of records such as `.list(.object([...]))`. Each object reads as a dictionary. Exhaustive `switch`es over `Field.Kind` need a case for it. +- Host tools: `HostTool`s in a `HostToolKit`, passed as `WorkflowExecutor(hostToolKits:)`, are + tools the app writes in Swift, such as one that edits the app's own state. Agent and Use a Tool + steps select them by kit id, so workflows stay Codable. Calls that change something follow the + write policy; local-only executions may call them. ### Fixed diff --git a/Documentation/Usage.md b/Documentation/Usage.md index 877bc13..5d88e87 100644 --- a/Documentation/Usage.md +++ b/Documentation/Usage.md @@ -171,6 +171,38 @@ prevents fallback. This is not an exactly-once guarantee. Group reusable tools in a `ToolKit`, pass it to the executor and select it with `kits: ["weather"]`. + + +### 🧩 Give an agent a Swift tool + +When the action belongs to your app, such as editing an order on screen, write +the tool in Swift and pass it in a `HostToolKit`: + +```swift +let order = HostToolKit(id: "order", name: "Order", tools: [ + HostTool(name: "addLine", description: "Adds a product to the order.", parameters: [ + Field("product", .text), + Field("quantity", .number), + ]) { arguments in + await cart.add(arguments) // Your app's own state + return .text("Added") + }, +]) +let workflow = Workflow( + name: "Order by voice" +) { + Step.agent("answer", "Update the order.", kits: ["order"], model: .appleOnDevice) +} +let report = try await WorkflowExecutor(writePolicy: .allow, hostToolKits: [order]) + .run(workflow, input: .text("Two beers, please")) +``` + +The workflow only names the kit, so it stays Codable; the body stays in +memory. Tools change something unless you pass `changesSomething: false`, and +then follow the write policy like requests (`.ask` sees a `POST host:order`). +Local-only executions may call them, since they reach no network. Use a Tool +steps call them directly with `Step.useTool("add", kit: "order", tool: "addLine")`. + ## ☁️ Choose your models diff --git a/Sources/ModelRelay/Execution/Recovery/WorkflowExecutor+Checkpoints.swift b/Sources/ModelRelay/Execution/Recovery/WorkflowExecutor+Checkpoints.swift index 912318e..945e551 100644 --- a/Sources/ModelRelay/Execution/Recovery/WorkflowExecutor+Checkpoints.swift +++ b/Sources/ModelRelay/Execution/Recovery/WorkflowExecutor+Checkpoints.swift @@ -8,7 +8,7 @@ extension WorkflowExecutor { try workflow.validateContexts() try workflow.validateApprovals(recoverable: true) try preflight(workflow) - return try workflow.recoveryDefinition(kits: Array(toolKits.values)) + return try workflow.recoveryDefinition(kits: toolSources.declaredKits) } func validate(_ checkpoint: WorkflowExecutionCheckpoint, workflow: Workflow, revision: String) throws { diff --git a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Agents.swift b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Agents.swift index 58bda67..648fb42 100644 --- a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Agents.swift +++ b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Agents.swift @@ -50,14 +50,16 @@ extension WorkflowExecutor { var servers: [ConnectedToolKit] = [] for id in scope.policy.tools == .denied ? [] : kitIDs ?? [] { - guard let kit = toolKits[id] else { - throw RelayError.stepFailed(step: step.name, reason: "the tool set “\(id)” isn't available") - } - if kit.server != nil { - let connection: any ToolServerConnection = try await connect(kit, step: step.name, policy: scope.policy) - servers.append(ConnectedToolKit(kit: kit, connection: connection, tools: try await serverTools(kit, connection, step: step.name))) - } else { + switch toolSources[id] { + case let .requests(kit): specs += kit.tools + case let .server(kit): + let connection: any ToolServerConnection = try await connect(kit, step: step.name, policy: scope.policy) + servers.append(ConnectedToolKit(kit: kit, connection: connection, tools: try await serverTools(kit, connection, step: step.name), reachesNetwork: true)) + case let .host(host): + servers.append(ConnectedToolKit(kit: host.toolKit, connection: host.connection, tools: host.tools.map(\.serverTool), reachesNetwork: false)) + case .ambiguous, nil: + throw RelayError.stepFailed(step: step.name, reason: "the tool set “\(id)” isn't available") } } @@ -69,6 +71,7 @@ private struct ConnectedToolKit: Sendable { let kit: ToolKit let connection: any ToolServerConnection let tools: [ServerTool] + let reachesNetwork: Bool } private struct AgentToolDefinitions: Sendable { @@ -89,6 +92,7 @@ private struct AgentToolDefinitions: Sendable { tool: $0, kit: server.kit, connection: server.connection, + reachesNetwork: server.reachesNetwork, step: execution.step.name, context: execution.context, log: log diff --git a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Policies.swift b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Policies.swift index 2aa91de..9a2e7cf 100644 --- a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Policies.swift +++ b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Policies.swift @@ -15,15 +15,16 @@ extension WorkflowExecutor { for value in scope.valuePolicies.values { policy.data = policy.data.combined(with: value) } + // Only request tools read workflow values through templates. case let .agent(_, _, kits): - for id in kits ?? [] { - for tool in toolKits[id]?.tools ?? [] { + for case let .requests(kit)? in (kits ?? []).map({ toolSources[$0] }) { + for tool in kit.tools { references += ([tool.request.url, tool.request.body ?? ""] + tool.request.headers.map(\.value)) .flatMap { Template.references(in: $0) } } } case let .useTool(kitID, toolName, _): - if let tool = toolKits[kitID]?.tools.first(where: { $0.name == toolName }) { + if case let .requests(kit)? = toolSources[kitID], let tool = kit.tools.first(where: { $0.name == toolName }) { references += ([tool.request.url, tool.request.body ?? ""] + tool.request.headers.map(\.value)) .flatMap { Template.references(in: $0) } } diff --git a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Tools.swift b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Tools.swift index d42eb15..b7a78d9 100644 --- a/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Tools.swift +++ b/Sources/ModelRelay/Execution/Steps/WorkflowExecutor+Tools.swift @@ -7,25 +7,19 @@ extension WorkflowExecutor { let trace: StepTrace = execution.trace guard case let .useTool(kitID, toolName, arguments) = step.kind else { throw InvalidOutput(reason: "Expected a tool step") } - guard let kit = toolKits[kitID] else { - throw RelayError.stepFailed(step: step.name, reason: "the tool set “\(kitID)” isn't available") - } let rendered = arguments.mapValues { Template.render($0, scope.resolve) } let value: Value - if kit.server != nil { + switch toolSources[kitID] { + case let .host(host): + let tools: [ServerTool] = host.tools.map(\.serverTool) + let call: ConnectedToolCall = .init(kit: host.toolKit, connection: host.connection, tools: tools, name: toolName, arguments: rendered) + value = try await runConnectedTool(call, execution: execution, scope: scope) + case let .server(kit): let connection = try await connect(kit, step: step.name, policy: scope.policy) - guard let tool = try await serverTools(kit, connection, step: step.name).first(where: { $0.name == toolName }) else { - throw RelayError.stepFailed(step: step.name, reason: "\(kit.name) has no tool “\(toolName)”") - } - let typed = ServerArguments.typed(rendered, for: tool) - let call = ServerCallDescription.prepared(kit: kit, tool: tool.name, arguments: typed) - trace.input = call.description - try scope.policy.checkTool(changesSomething: tool.changesSomething, step: step.name) - try scope.control.beginTool(tool.name, step: step.name) - - let invocation: ServerToolInvocation = .init(tool: tool, connection: connection, arguments: typed, request: call) - value = try await callServerTool(invocation, execution: execution, control: scope.control) - } else { + let tools: [ServerTool] = try await serverTools(kit, connection, step: step.name) + let call: ConnectedToolCall = .init(kit: kit, connection: connection, tools: tools, name: toolName, arguments: rendered) + value = try await runConnectedTool(call, execution: execution, scope: scope) + case let .requests(kit): guard let spec = kit.tools.first(where: { $0.name == toolName }) else { throw RelayError.stepFailed(step: step.name, reason: "\(kit.name) has no tool “\(toolName)”") } @@ -33,11 +27,28 @@ extension WorkflowExecutor { if reference.hasPrefix("args.") { return rendered[String(reference.dropFirst("args.".count))].map(Value.text) } return scope.resolve(reference) } + case .ambiguous, nil: + throw RelayError.stepFailed(step: step.name, reason: "the tool set “\(kitID)” isn't available") } scope.record(execution.record(value)) return value } + private func runConnectedTool(_ call: ConnectedToolCall, execution: WorkflowStepExecution, scope: WorkflowExecutionScope) async throws -> Value { + let step: String = execution.step.name + guard let tool = call.tools.first(where: { $0.name == call.name }) else { + throw RelayError.stepFailed(step: step, reason: "\(call.kit.name) has no tool “\(call.name)”") + } + let typed: [String: Value] = ServerArguments.typed(call.arguments, for: tool) + let request: PreparedRequest = ServerCallDescription.prepared(kit: call.kit, tool: tool.name, arguments: typed) + execution.trace.input = request.description + try scope.policy.checkTool(changesSomething: tool.changesSomething, step: step) + try scope.control.beginTool(tool.name, step: step) + + let invocation: ServerToolInvocation = .init(tool: tool, connection: call.connection, arguments: typed, request: request) + return try await callServerTool(invocation, execution: execution, control: scope.control) + } + private func callServerTool(_ invocation: ServerToolInvocation, execution: WorkflowStepExecution, control: ExecutionControl) async throws -> Value { var dispatched: Bool = false @@ -86,6 +97,15 @@ extension WorkflowExecutor { } } +/// One named tool of a set whose tools come from a connection: a server, or the host app. +private struct ConnectedToolCall: Sendable { + let kit: ToolKit + let connection: any ToolServerConnection + let tools: [ServerTool] + let name: String + let arguments: [String: String] +} + private struct ServerToolInvocation: Sendable { let tool: ServerTool let connection: any ToolServerConnection diff --git a/Sources/ModelRelay/Execution/WorkflowExecutor.swift b/Sources/ModelRelay/Execution/WorkflowExecutor.swift index 7a1eb83..9a31bcc 100644 --- a/Sources/ModelRelay/Execution/WorkflowExecutor.swift +++ b/Sources/ModelRelay/Execution/WorkflowExecutor.swift @@ -5,7 +5,7 @@ public actor WorkflowExecutor { let catalog: ModelCatalog let stepTimeout: Duration let context: RequestContext - let toolKits: [String: ToolKit] + let toolSources: ToolSources let toolServers: (any ToolServerConnector)? let policy: WorkflowExecutionPolicy let limits: WorkflowExecutionLimits @@ -17,7 +17,8 @@ public actor WorkflowExecutor { /// fallback model, if any). Request steps need `requests`; those that change /// something also need a `writePolicy` that allows them. Custom bodies /// use the same cooperative timeout and run once, without fallback. - /// Tool sets with a server need `toolServers`. + /// Tool sets with a server need `toolServers`; `hostToolKits` are tool sets + /// the app implements in Swift. public init( catalog: ModelCatalog = ModelCatalog(), stepTimeout: Duration = .seconds(30), @@ -26,6 +27,7 @@ public actor WorkflowExecutor { writePolicy: WritePolicy = .deny, toolKits: [ToolKit] = [], toolServers: (any ToolServerConnector)? = nil, + hostToolKits: [HostToolKit] = [], policy: WorkflowExecutionPolicy = .unrestricted, limits: WorkflowExecutionLimits = .unlimited ) { @@ -34,7 +36,7 @@ public actor WorkflowExecutor { self.limits = limits self.stepTimeout = stepTimeout self.toolServers = toolServers - self.toolKits = Dictionary(toolKits.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + self.toolSources = ToolSources(kits: toolKits, hostKits: hostToolKits) self.modelRuntime = WorkflowModelRuntime(catalog: catalog, stepTimeout: stepTimeout) self.context = RequestContext(requests: requests, secrets: secrets, writePolicy: writePolicy, timeout: stepTimeout) } @@ -91,7 +93,7 @@ public actor WorkflowExecutor { /// Does not load providers, retrieve credentials, or discover remote tools. public func validate(_ workflow: Workflow) -> [WorkflowValidationIssue] { - workflow.validationIssues(catalog: catalog, kits: toolKits) + workflow.validationIssues(catalog: catalog, tools: toolSources) } func preflight(_ workflow: Workflow) throws { diff --git a/Sources/ModelRelay/Tools/HostTool.swift b/Sources/ModelRelay/Tools/HostTool.swift new file mode 100644 index 0000000..0e0de32 --- /dev/null +++ b/Sources/ModelRelay/Tools/HostTool.swift @@ -0,0 +1,90 @@ +import Foundation + +/// A tool the app implements in Swift, such as one that edits the app's own +/// state. Agents and Use a Tool steps select it through a `HostToolKit` by id, +/// so the workflow stays Codable while the body stays in memory. +/// +/// Bodies have the host application's authority, like custom steps. A call +/// that changes something asks the executor's `writePolicy` first (it sees a +/// POST to `host:` whose body names the tool and its arguments), runs +/// once and is never retried. Host tools reach no network themselves, so +/// local-only executions may call them. +public struct HostTool: Sendable { + public let name: String + public let description: String + public let parameters: [Field] + /// Unless the tool is marked read-only, it may change something and needs + /// the same permission as a write request. + public let changesSomething: Bool + private let body: @Sendable ([String: Value]) async throws -> Value + + public init( + name: String, + description: String, + parameters: [Field] = [], + changesSomething: Bool = true, + body: @escaping @Sendable (_ arguments: [String: Value]) async throws -> Value + ) { + self.name = name + self.description = description + self.parameters = parameters + self.changesSomething = changesSomething + self.body = body + } + + /// How the tool describes itself to models and policies. + var serverTool: ServerTool { + ServerTool( + name: name, + description: description, + parameters: parameters, + required: parameters.map(\.name), + changesSomething: changesSomething + ) + } + + func call(_ arguments: [String: Value]) async throws -> Value { + try await body(arguments) + } +} + +/// Host tools shared by many steps. Steps name it by `id`, like a `ToolKit`; +/// the two share one set of ids. +public struct HostToolKit: Sendable, Identifiable { + public let id: String + public let name: String + public let tools: [HostTool] + + public init(id: String, name: String, tools: [HostTool]) { + self.id = id + self.name = name + self.tools = tools + } + + /// The kit as tool runtimes read it: no request tools and no server. + var toolKit: ToolKit { + ToolKit(id: id, name: name, tools: []) + } + + var connection: HostToolConnection { + HostToolConnection(kit: self) + } +} + +/// A host tool kit as an in-process tool server: no connector, no network. +struct HostToolConnection: ToolServerConnection { + let kit: HostToolKit + + func tools() async throws -> [ServerTool] { + kit.tools.map(\.serverTool) + } + + func call(_ tool: String, arguments: [String: Value]) async throws -> Value { + guard let host = kit.tools.first(where: { $0.name == tool }) else { + throw InvalidOutput(reason: "\(kit.name) has no tool “\(tool)”") + } + try Task.checkCancellation() + + return try await host.call(arguments) + } +} diff --git a/Sources/ModelRelay/Tools/Runtime/ServerToolCall.swift b/Sources/ModelRelay/Tools/Runtime/ServerToolCall.swift index ecc8916..d61073f 100644 --- a/Sources/ModelRelay/Tools/Runtime/ServerToolCall.swift +++ b/Sources/ModelRelay/Tools/Runtime/ServerToolCall.swift @@ -2,11 +2,13 @@ import AnyLanguageModel import Foundation /// A server tool as an AnyLanguageModel `Tool`. Calls that may change -/// something ask the write policy first, like write requests. +/// something ask the write policy first, like write requests. Host tool kits +/// use it too, as servers that don't reach the network. struct ServerToolCall: Tool { let tool: ServerTool let kit: ToolKit let connection: any ToolServerConnection + let reachesNetwork: Bool let step: String let context: RequestContext let log: ToolCallLog @@ -26,7 +28,7 @@ struct ServerToolCall: Tool { @concurrent func call(arguments: GeneratedContent) async throws -> String { try Task.checkCancellation() try log.record(tool.name) - try context.policy.checkNetwork(step: step) + if reachesNetwork { try context.policy.checkNetwork(step: step) } try context.policy.checkTool(changesSomething: tool.changesSomething, step: step) var values: [String: Value] = [:] if case let .dictionary(fields) = Value(arguments) { values = fields } @@ -58,7 +60,8 @@ enum ServerCallDescription { ) return PreparedRequest( method: .post, - url: URL(string: kit.server?.url ?? "") ?? URL(string: "about:blank")!, + // Host tool kits have no server: their calls read as `host:`. + url: URL(string: kit.server?.url ?? "host:\(kit.id)") ?? URL(string: "about:blank")!, headers: [], body: body, secrets: [] diff --git a/Sources/ModelRelay/Tools/ToolSource.swift b/Sources/ModelRelay/Tools/ToolSource.swift new file mode 100644 index 0000000..6c8ab91 --- /dev/null +++ b/Sources/ModelRelay/Tools/ToolSource.swift @@ -0,0 +1,47 @@ +/// Where the tools of one tool set id come from. Steps name tool sets by id, +/// so `ToolKit`s and `HostToolKit`s share one set of ids. +enum ToolSource: Sendable { + /// Requests defined in the kit. + case requests(ToolKit) + /// Tools a server offers, discovered when a step runs. + case server(ToolKit) + /// Tools the app wrote in Swift. + case host(HostToolKit) + /// The id names both a tool kit and a host tool kit: neither is used. + case ambiguous(ToolKit, HostToolKit) +} + +/// The executor's tool sets by id. +struct ToolSources: Sendable { + private let sources: [String: ToolSource] + + /// The first kit with an id wins within each list, as before host tools. + init(kits: [ToolKit], hostKits: [HostToolKit]) { + var sources: [String: ToolSource] = [:] + for kit in kits where sources[kit.id] == nil { + sources[kit.id] = kit.server == nil ? .requests(kit) : .server(kit) + } + var hosts: Set = [] + for host in hostKits where hosts.insert(host.id).inserted { + switch sources[host.id] { + case let .requests(kit), let .server(kit): sources[host.id] = .ambiguous(kit, host) + default: sources[host.id] = .host(host) + } + } + self.sources = sources + } + + subscript(id: String) -> ToolSource? { + sources[id] + } + + /// The Codable kits, which recovery checkpoints fingerprint. + var declaredKits: [ToolKit] { + sources.values.compactMap { source in + switch source { + case let .requests(kit), let .server(kit), let .ambiguous(kit, _): kit + case .host: nil + } + } + } +} diff --git a/Sources/ModelRelay/Workflow/Validation/Workflow+ToolValidation.swift b/Sources/ModelRelay/Workflow/Validation/Workflow+ToolValidation.swift new file mode 100644 index 0000000..a319ffd --- /dev/null +++ b/Sources/ModelRelay/Workflow/Validation/Workflow+ToolValidation.swift @@ -0,0 +1,70 @@ +import Foundation + +extension Workflow { + /// Preflight issues with the tools an agent or Use a Tool step selects. + /// Server tools are discovered when the step runs, so they aren't checked here. + func toolIssues(for step: Step, sources: ToolSources) -> [WorkflowValidationIssue] { + let selection: ToolSelection = .init(step: step, sources: sources) + var issues: [WorkflowValidationIssue] = selection.unavailable.map { .init(step: step.name, kind: .unavailableTool, detail: $0) } + + let names: [String] = selection.requests.map(\.name) + selection.hosts.map(\.name) + if Set(names).count != names.count { issues.append(.init(step: step.name, kind: .invalidSchema, detail: "tool names must be unique")) } + let parameters: [[Field]] = selection.requests.map(\.parameters) + selection.hosts.map(\.parameters) + issues += parameters.flatMap(\.schemaProblems).map { .init(step: step.name, kind: .invalidSchema, detail: $0) } + + return issues + referenceIssues(in: selection.requests, step: step.name) + } + + /// Local kit templates can read workflow values too. No discovery/network here. + private func referenceIssues(in tools: [AgentTool], step: String) -> [WorkflowValidationIssue] { + let available: Set = Set(availableSteps(before: step).map(\.name) + ["input"]) + let texts: [String] = tools.flatMap { [$0.request.url, $0.request.body ?? ""] + $0.request.headers.map(\.value) } + + return texts.flatMap { Template.references(in: $0) } + .filter { !$0.hasPrefix("secret.") && !$0.hasPrefix("args.") } + .filter { !available.contains($0.split(separator: ".").first.map(String.init) ?? "") } + .map { .init(step: step, kind: .unavailableReference, detail: $0) } + } +} + +/// The tools a step selects from the executor's tool sets. +private struct ToolSelection { + var requests: [AgentTool] = [] + var hosts: [HostTool] = [] + /// Why a selected kit or tool isn't available. + var unavailable: [String] = [] + + init(step: Step, sources: ToolSources) { + switch step.kind { + case let .agent(_, own, ids): + requests = own + for id in ids ?? [] { + add(sources[id], id: id, tool: nil) + } + case let .useTool(id, name, _): + add(sources[id], id: id, tool: name) + default: + break + } + } + + private mutating func add(_ source: ToolSource?, id: String, tool name: String?) { + switch source { + case let .requests(kit): requests += pick(kit.tools, name: name) { $0.name } + case .server: break + case let .host(host): hosts += pick(host.tools, name: name) { $0.name } + case .ambiguous: unavailable.append("\(id) names both a tool kit and a host tool kit") + case nil: unavailable.append(id) + } + } + + /// Every tool for an agent; the named one for Use a Tool. + private mutating func pick(_ tools: [Tool], name: String?, _ toolName: (Tool) -> String) -> [Tool] { + guard let name else { return tools } + guard let tool = tools.first(where: { toolName($0) == name }) else { + unavailable.append(name) + return [] + } + return [tool] + } +} diff --git a/Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift b/Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift index 434fb12..df65a21 100644 --- a/Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift +++ b/Sources/ModelRelay/Workflow/Validation/Workflow+Validation.swift @@ -18,7 +18,7 @@ public struct WorkflowValidationError: Error, Equatable, Sendable { } extension Workflow { - func validationIssues(catalog: ModelCatalog, kits: [String: ToolKit]) -> [WorkflowValidationIssue] { + func validationIssues(catalog: ModelCatalog, tools: ToolSources) -> [WorkflowValidationIssue] { var issues: [WorkflowValidationIssue] = [] func issue(_ step: String, _ kind: WorkflowValidationIssue.Kind, _ detail: String) { issues.append(.init(step: step, kind: kind, detail: detail)) @@ -63,7 +63,7 @@ extension Workflow { issue(step.name, .incompatibleModel, "no configured model supports this step") } } - var tools: [AgentTool] = [] + issues += toolIssues(for: step, sources: tools) switch step.kind { case let .extract(_, values): fields(values, step: step.name) case let .classify(_, labels): @@ -72,19 +72,6 @@ extension Workflow { } case let .decide(questions): if let problem = Question.problem(in: questions) { issue(step.name, .invalidSchema, problem) } - case let .agent(_, own, ids): - tools = own - for id in ids ?? [] { - if let kit = kits[id] { if kit.server == nil { tools += kit.tools } } - else { issue(step.name, .unavailableTool, id) } - } - case let .useTool(id, name, _): - if let kit = kits[id] { - if kit.server == nil { - if let tool = kit.tools.first(where: { $0.name == name }) { tools = [tool] } - else { issue(step.name, .unavailableTool, name) } - } - } else { issue(step.name, .unavailableTool, id) } case let .parallel(children): visit(children) case let .ifEquals(_, _, yes, no): visit(yes); visit(no) case let .switchOn(_, cases, otherwise): @@ -95,19 +82,6 @@ extension Workflow { }; visit(otherwise) default: break } - if Set(tools.map(\.name)).count != tools.count { issue(step.name, .invalidSchema, "tool names must be unique") } - for tool in tools { - fields(tool.parameters, step: step.name) - } - // Local kit templates can read workflow values too. No discovery/network here. - for tool in tools { - let available = Set(availableSteps(before: step.name).map(\.name) + ["input"]) - let texts = [tool.request.url, tool.request.body ?? ""] + tool.request.headers.map(\.value) - for ref in texts.flatMap({ Template.references(in: $0) }) where !ref.hasPrefix("secret.") && !ref.hasPrefix("args.") { - let base = ref.split(separator: ".").first.map(String.init) ?? "" - if !available.contains(base) { issue(step.name, .unavailableReference, ref) } - } - } } } visit(steps) diff --git a/Tests/ModelRelayTests/Models/LiveAppleTests.swift b/Tests/ModelRelayTests/Models/LiveAppleTests.swift index 366bcd7..140bff2 100644 --- a/Tests/ModelRelayTests/Models/LiveAppleTests.swift +++ b/Tests/ModelRelayTests/Models/LiveAppleTests.swift @@ -1,4 +1,5 @@ import Foundation +import Synchronization import Testing @testable import ModelRelay @@ -45,6 +46,30 @@ struct LiveAppleTests { #expect(report.output["queue"] == .text("shipping")) } + @Test func onDeviceModelCallsAHostTool() async throws { + let added: Mutex<[[String: Value]]> = .init([]) + let kit: HostToolKit = .init(id: "order", name: "Order", tools: [ + HostTool(name: "addLine", description: "Adds a product to the order.", parameters: [ + Field("product", .choice(["Beer", "Cola", "Fries"])), + Field("quantity", .number), + ]) { arguments in + added.withLock { $0.append(arguments) } + return .text("Added") + }, + ]) + let workflow: Workflow = .init(name: "Order", steps: [ + Step( + "answer", + .agent(instruction: "Update the order with addLine, then say what you added.", tools: [], kits: ["order"]), + model: .appleOnDevice + ), + ]) + let executor: WorkflowExecutor = .init(writePolicy: .allow, hostToolKits: [kit]) + let report: WorkflowExecutionReport = try await executor.run(workflow, input: .text("Two beers, please.")) + print("LIVE host tool:", report.output.textValue, "| calls:", added.withLock { $0 }, "|", report.duration) + #expect(added.withLock { $0 }.contains(["product": .text("Beer"), "quantity": .number(2)])) + } + @Test func onDeviceModelExtractsListsOfObjects() async throws { let workflow: Workflow = .init(name: "Order", steps: [Step("order", .extract(instruction: "Turn the order into lines.", fields: [ Field("lines", .list(.object([ diff --git a/Tests/ModelRelayTests/Tools/HostToolTests.swift b/Tests/ModelRelayTests/Tools/HostToolTests.swift new file mode 100644 index 0000000..b1f8c34 --- /dev/null +++ b/Tests/ModelRelayTests/Tools/HostToolTests.swift @@ -0,0 +1,150 @@ +import Foundation +import Synchronization +import Testing +@testable import ModelRelay + +/// An order the host app owns; its tools change it in place. +private final class Order: Sendable { + private let state: Mutex<[String: Value]> = .init([:]) + var lines: [String: Value] { state.withLock { $0 } } + + func kit(onAdd: @escaping @Sendable () -> Void = {}) -> HostToolKit { + HostToolKit(id: "order", name: "Order", tools: [ + HostTool(name: "add_line", description: "Adds a product to the order.", parameters: [ + Field("product", .text), + Field("quantity", .number), + Field("extras", .list(.object([Field("name", .text), Field("quantity", .number)]))), + ]) { arguments in + onAdd() + let product: String = arguments["product"]?.textValue ?? "" + self.state.withLock { $0[product] = .dictionary(arguments) } + return .text("Added \(product)") + }, + HostTool( + name: "find_product", + description: "Finds a product on the menu.", + parameters: [Field("query", .text)], + changesSomething: false + ) { arguments in + .dictionary(["id": .text("p-1"), "name": .text(arguments["query"]?.textValue ?? "")]) + }, + ]) + } +} + +private final class Approvals: Sendable { + private let seen: Mutex<[String]> = .init([]) + var requests: [String] { seen.withLock { $0 } } + + func policy(_ answer: Bool) -> WritePolicy { + .ask { request in + self.seen.withLock { $0.append(request.description) } + return answer + } + } +} + +@Suite struct HostToolTests { + private let agent: Workflow = .init(name: "Order", steps: [ + Step("answer", .agent(instruction: "Update the order.", tools: [], kits: ["order"]), model: "small"), + ]) + + @Test func agentsCallHostToolsAndWritesAskFirst() async throws { + let order: Order = .init() + let approvals: Approvals = .init() + let model: ScriptedModel = .init("small", replies: [ + #"{"product": "Beer", "quantity": 2, "extras": [{"name": "Lemon", "quantity": 1}]}"#, "Added.", + ]) + let executor: WorkflowExecutor = .init(catalog: .scripted(["small": model]), writePolicy: approvals.policy(true), hostToolKits: [order.kit()]) + + let report: WorkflowExecutionReport = try await executor.run(agent, input: .text("Two beers with lemon")) + + #expect(report.output == .text("Added.")) + #expect(report.records.first?.toolCalls == ["add_line"]) + #expect(order.lines["Beer"]?["extras"] == .list([.dictionary(["name": .text("Lemon"), "quantity": .number(1)])])) + let request: String = try #require(approvals.requests.first) + #expect(request.hasPrefix("POST host:order")) + #expect(request.contains(#""tool" : "add_line""#)) + #expect(model.prompts.last?.contains("[add_line] \"Added Beer\"") == true) + } + + @Test func deniedWritesNeverRunTheBody() async { + let order: Order = .init() + let runs: Mutex = .init(0) + let model: ScriptedModel = .init("small", replies: [#"{"product": "Beer", "quantity": 1, "extras": []}"#, "Done."]) + let executor: WorkflowExecutor = .init(catalog: .scripted(["small": model]), hostToolKits: [order.kit { runs.withLock { $0 += 1 } }]) + + await #expect(throws: (any Error).self) { + try await executor.run(agent, input: .text("A beer")) + } + + #expect(runs.withLock { $0 } == 0) + #expect(order.lines.isEmpty) + } + + @Test func readOnlyExecutionsOfferOnlyReadOnlyHostTools() async throws { + let model: ScriptedModel = .init("small", replies: ["Nothing to change."]) + let executor: WorkflowExecutor = .init(catalog: .scripted(["small": model]), hostToolKits: [Order().kit()], policy: .readOnly) + + _ = try await executor.run(agent, input: .text("Is there beer?")) + + #expect(model.visibleTools == [["find_product"]]) + } + + @Test func localOnlyExecutionsMayCallHostTools() async throws { + let model: ScriptedModel = .init("small", replies: [#"{"query": "Beer"}"#, "Found it."]) + let catalog: ModelCatalog = .init().registering("small", location: .local) { model } + let executor: WorkflowExecutor = .init(catalog: catalog, hostToolKits: [Order().kit()], policy: .localOnly) + + let report: WorkflowExecutionReport = try await executor.run(agent, input: .text("Is there beer?")) + + #expect(report.records.first?.toolCalls == ["find_product"]) + #expect(report.output == .text("Found it.")) + } + + @Test func useAToolCallsHostToolsWithTypedArguments() async throws { + let order: Order = .init() + let workflow: Workflow = .init(name: "Order", steps: [ + Step("add", .useTool(kit: "order", tool: "add_line", arguments: ["product": "{{input}}", "quantity": "3", "extras": "[]"])), + ]) + let executor: WorkflowExecutor = .init(writePolicy: .allow, hostToolKits: [order.kit()]) + + let report: WorkflowExecutionReport = try await executor.run(workflow, input: .text("Cola")) + + #expect(report.output == .text("Added Cola")) + #expect(order.lines["Cola"] == .dictionary(["product": .text("Cola"), "quantity": .number(3), "extras": .list([])])) + } + + @Test func missingAmbiguousAndInvalidHostToolsArePreflightIssues() async { + let invalid: HostToolKit = .init(id: "menu", name: "Menu", tools: [ + HostTool(name: "pick", description: "Picks.", parameters: [Field("size", .choice([]))]) { _ in .null }, + ]) + let executor: WorkflowExecutor = .init( + toolKits: [ToolKit(id: "order", name: "Order API", tools: [])], + hostToolKits: [Order().kit(), invalid] + ) + let workflow: Workflow = .init(name: "Order", steps: [ + Step("both", .useTool(kit: "order", tool: "add_line", arguments: [:])), + Step("missing", .useTool(kit: "menu", tool: "remove_line", arguments: [:])), + Step("pick", .useTool(kit: "menu", tool: "pick", arguments: [:])), + ]) + + let issues: [WorkflowValidationIssue] = await executor.validate(workflow) + + #expect(issues == [ + .init(step: "both", kind: .unavailableTool, detail: "order names both a tool kit and a host tool kit"), + .init(step: "missing", kind: .unavailableTool, detail: "remove_line"), + .init(step: "pick", kind: .invalidSchema, detail: "choices must be nonempty and unique"), + ]) + } + + @Test func hostToolKitsLeaveRecoveryDefinitionsAsTheyWere() async throws { + let kits: [ToolKit] = [ToolKit(id: "api", name: "API", tools: [])] + let workflow: Workflow = .init(name: "Order", steps: [Step("note", .template("{{input}}"))]) + + let without: String = try await WorkflowExecutor(toolKits: kits).recoveryDefinition(workflow, revision: "1") + let with: String = try await WorkflowExecutor(toolKits: kits, hostToolKits: [Order().kit()]).recoveryDefinition(workflow, revision: "1") + + #expect(with == without) + } +} diff --git a/skills/modelrelay/SKILL.md b/skills/modelrelay/SKILL.md index ec49bea..4c906fe 100644 --- a/skills/modelrelay/SKILL.md +++ b/skills/modelrelay/SKILL.md @@ -131,6 +131,7 @@ Build one `WorkflowExecutor` per configuration, with explicit dependencies: | `secrets:` | The app's `SecretProvider`, backed by secure storage | | `writePolicy:` | `.deny` (default), `.ask { request in … }` or `.allow` | | `toolKits:` / `toolServers:` | `ToolKit`s and an `MCPConnector` | +| `hostToolKits:` | `HostToolKit`s: tools the app writes in Swift | | `policy:` / `limits:` | `.localOnly`, `.readOnly`, `.noTools`; `WorkflowExecutionLimits` | - `run` returns the report. `execute` adds live `updates`; keep the handle diff --git a/skills/modelrelay/references/api-map.md b/skills/modelrelay/references/api-map.md index 45cdf9d..be49634 100644 --- a/skills/modelrelay/references/api-map.md +++ b/skills/modelrelay/references/api-map.md @@ -58,6 +58,7 @@ Package setup for each product: `Documentation/Models.md#setup` and | Credentials | `SecretProvider` with `{{secret.NAME}}` | `Documentation/Usage.md#http-requests-and-agent-tools` | | Write decisions | `WritePolicy` | `Documentation/ContextAndPolicies.md#execution-policies` | | Reusable tools | `ToolKit`, `AgentTool`, `ToolServer` | `Documentation/Usage.md#http-requests-and-agent-tools` | +| Tools written in Swift | `HostToolKit`, `HostTool` | `Documentation/Usage.md#host-tools` | | MCP access | `MCPConnector`, `MCPSignIn` | `Documentation/Integration.md#mcp-and-oauth` | | Durable review state | `WorkflowExecutionCheckpointStore` | `Documentation/Recovery.md#storage-contract-atomic-durable-compare-and-swap` |