From 512237ad62b57c1f6ce130dea138f3652124788a Mon Sep 17 00:00:00 2001 From: shuv1337 Date: Thu, 12 Nov 2026 12:17:53 -0800 Subject: [PATCH] feat(self-host): deliver feedback to an operator webhook --- .github/workflows/fork-ci.yml | 2 + apps/docs/content/run/usage-analytics.md | 7 +- apps/hosted/self-host/README.md | 31 ++++ apps/hosted/self-host/compose.yaml | 3 + .../src/implementation/feedback-webhook.ts | 75 ++++++++ .../self-host/src/implementation/routes.ts | 14 +- apps/hosted/server/src/contracts/feedback.ts | 2 +- e2e/support/managed-server.ts | 2 + e2e/test-plan.ts | 24 +++ e2e/tests/docker-release.spec.ts | 111 +++++++++++ e2e/tests/feedback-webhook.spec.ts | 174 ++++++++++++++++++ .../executor/skills/executor/feedback.md | 4 +- .../executor-skill-digests.gen.ts | 2 +- 13 files changed, 441 insertions(+), 10 deletions(-) create mode 100644 apps/hosted/self-host/src/implementation/feedback-webhook.ts create mode 100644 e2e/tests/feedback-webhook.spec.ts diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index 68f99c869e..7466530ca8 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -67,6 +67,8 @@ jobs: - name: Security scenarios run: | bun run e2e:self-host -- --test-name 'application_type|fetch without a User-Agent names Executor|cannot create an all-apps connection|cannot widen a connection or mint|fails to load fails only its own reads' + - name: Operator feedback scenarios + run: bun run e2e:self-host --test-name 'self-host feedback webhook|self-host with DO_NOT_TRACK|self-host analytics pseudonymize|Hosted feedback enforces' # Run the large cold catalog alone, not beside the functional scenarios' servers. - name: Cold MCP discovery run: bun run e2e:self-host --test-name 'Cold MCP discovery serves 8549' diff --git a/apps/docs/content/run/usage-analytics.md b/apps/docs/content/run/usage-analytics.md index 55db45015a..d286c82867 100644 --- a/apps/docs/content/run/usage-analytics.md +++ b/apps/docs/content/run/usage-analytics.md @@ -18,8 +18,11 @@ EXECUTOR_DISABLE_ANALYTICS=1 ``` Any value other than empty, `0`, `false`, `no` or `off` turns analytics off. -Nothing is recorded or sent, including feedback: the `feedback.submit` tool then -returns `FeedbackDisabled`. +Nothing is recorded or sent to analytics, including feedback: the `feedback.submit` +tool then returns `FeedbackDisabled` unless the self-host operator configured +`EXECUTOR_FEEDBACK_WEBHOOK_URL`. That independent receiver takes precedence for +feedback and receives the submitted message, sender identity and instance origin; +the anonymous analytics guarantees below do not apply to that explicit delivery. ## What is sent diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index a9f9597fc2..e1890e3f83 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -161,6 +161,37 @@ organization, and leaves it unset otherwise. Releases before the one that added the record refuse every sign-in once a database holds a second organization, so do not roll back below it after one exists. +### Operator feedback receiver + +Set `EXECUTOR_FEEDBACK_WEBHOOK_URL` to send explicit `feedback.submit` submissions +from the API and Executor app to your own HTTP(S) receiver. Optionally set +`EXECUTOR_FEEDBACK_WEBHOOK_SECRET` for an `Authorization: Bearer` header. Keep both +in your host's secret store; credential-bearing webhook URLs are secrets too. +Use HTTPS except for a trusted local receiver. URLs with embedded username/password +credentials are refused; query-string tokens are supported. + +The receiver gets a JSON POST containing `message`, `submittedAt` (ISO 8601), +`organization: { id }`, `user: { id, email? }`, `caller` (available authenticated +source/client/app attribution), and `instanceOrigin`. Unlike anonymous analytics, +this intentionally identifies the sender to the instance operator. Never include +credentials in feedback. `title` ("Executor feedback") and `body` (the first 1,500 +message characters plus the sender) also support notification webhook services +without a separate adapter. An app's managed API key may identify its owner, not +the outer agent; unavailable agent attribution is not inferred from message text. + +Only a 2xx response within 10 seconds returns `accepted`. Redirects, connection +failures, timeouts and rejection return `FeedbackUnavailable`, without disclosing +the receiver URL or response. Delivery is not retried automatically; after an +ambiguous timeout a caller retry may duplicate feedback. `Idempotency-Key` is a +unique per-delivery ID, not a deduplication key across separate caller submissions. + +The webhook takes precedence over feedback analytics and works even with +`DO_NOT_TRACK=1`, `EXECUTOR_DISABLE_ANALYTICS=1`, or no analytics destination. +Other usage analytics are unchanged. With the URL unset or empty, existing +analytics feedback behavior is unchanged, including `FeedbackDisabled` when +analytics are unavailable. Submissions are not stored in the product database: +there is no feedback table, so durable retention belongs to the receiver for now. + ### Trace sampling Recording every span made Motel the busiest process under load, so the product diff --git a/apps/hosted/self-host/compose.yaml b/apps/hosted/self-host/compose.yaml index 15151e6b52..58f54c0dd0 100644 --- a/apps/hosted/self-host/compose.yaml +++ b/apps/hosted/self-host/compose.yaml @@ -17,6 +17,9 @@ services: SSO_CLIENT_SECRET: ${SSO_CLIENT_SECRET:-} SSO_ALLOWED_DOMAINS: ${SSO_ALLOWED_DOMAINS:-} EXECUTOR_ENCRYPTION_KEY: + # Optional operator-owned feedback receiver, independent of anonymous analytics. + EXECUTOR_FEEDBACK_WEBHOOK_URL: + EXECUTOR_FEEDBACK_WEBHOOK_SECRET: # Optional client-IP assertion, accepted only from the configured proxy peers. EXECUTOR_TRUSTED_PROXY_HEADER: EXECUTOR_TRUSTED_PROXIES: diff --git a/apps/hosted/self-host/src/implementation/feedback-webhook.ts b/apps/hosted/self-host/src/implementation/feedback-webhook.ts new file mode 100644 index 0000000000..dcd3cdb9ea --- /dev/null +++ b/apps/hosted/self-host/src/implementation/feedback-webhook.ts @@ -0,0 +1,75 @@ +/** Explicit feedback delivery to an operator-owned receiver, independent of usage analytics. */ +import { CurrentUsage, ProductAnalytics } from "@executor-js/hosted-server"; +import { FeedbackUnavailable } from "@executor-js/telemetry/product-analytics"; +import { Config, DateTime, Effect, Option, Redacted, Schema } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import { SqlClient } from "effect/sql"; + +const Person = Schema.Struct({ email: Schema.String }); + +/** An unset URL leaves the existing analytics sink (including its opt-out) untouched. */ +export const selfHostFeedbackWebhook = (origin: string) => + Effect.gen(function* () { + const configured = yield* Config.Redacted("EXECUTOR_FEEDBACK_WEBHOOK_URL").pipe(Config.option); + if (Option.isNone(configured) || Redacted.value(configured.value).trim() === "") + return undefined; + const destination = URL.parse(Redacted.value(configured.value)); + if ( + destination === null || + !["https:", "http:"].includes(destination.protocol) || + destination.username !== "" || + destination.password !== "" + ) { + yield* Effect.logWarning( + "EXECUTOR_FEEDBACK_WEBHOOK_URL must be an HTTP(S) URL without embedded credentials", + ); + return () => Effect.fail(new FeedbackUnavailable()); + } + const secret = yield* Config.Redacted("EXECUTOR_FEEDBACK_WEBHOOK_SECRET").pipe(Config.option); + const sql = yield* SqlClient.SqlClient; + + const submit: typeof ProductAnalytics.Service.submitFeedback = (feedback) => + Effect.gen(function* () { + const caller = yield* CurrentUsage; + const rows = yield* sql`select email from "user" where id = ${feedback.userId}`; + const person = Schema.decodeUnknownOption(Person)(rows[0]); + const user = { + id: feedback.userId, + ...(Option.isSome(person) ? { email: person.value.email } : {}), + }; + const submittedAt = DateTime.formatIso(yield* DateTime.now); + const who = [user.email ?? user.id, caller.client_name].filter(Boolean).join(" / "); + const payload = { + message: feedback.message, + submittedAt, + organization: { id: feedback.organizationId }, + user, + caller, + instanceOrigin: origin, + // Common notification receivers can consume these without a translation service. + title: "Executor feedback", + body: `${feedback.message.slice(0, 1500)}\nFrom: ${who.slice(0, 300)}`, + }; + const request = HttpClientRequest.post(destination.href).pipe( + HttpClientRequest.bodyJsonUnsafe(payload), + HttpClientRequest.setHeader("Idempotency-Key", crypto.randomUUID()), + ); + const response = yield* HttpClient.execute( + Option.isSome(secret) && Redacted.value(secret.value) !== "" + ? HttpClientRequest.bearerToken(request, secret.value) + : request, + ); + if (response.status < 200 || response.status >= 300) + return yield* new FeedbackUnavailable(); + }).pipe( + Effect.timeout("10 seconds"), + Effect.tapError(() => Effect.logWarning("Feedback webhook delivery failed")), + Effect.mapError(() => new FeedbackUnavailable()), + // Do not trace the URL, authorization or response: all may contain receiver secrets. + Effect.provideService(HttpClient.TracerDisabledWhen, () => true), + Effect.provideService(HttpClient.TracerPropagationEnabled, false), + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" }), + Effect.provide(FetchHttpClient.layer), + ); + return submit; + }); diff --git a/apps/hosted/self-host/src/implementation/routes.ts b/apps/hosted/self-host/src/implementation/routes.ts index 7068328b6b..a1595ca386 100644 --- a/apps/hosted/self-host/src/implementation/routes.ts +++ b/apps/hosted/self-host/src/implementation/routes.ts @@ -58,6 +58,7 @@ import { selfHostApi } from "./api.ts"; import { selfHostMcp } from "../mcp.ts"; import { SelfHostAuth } from "../auth.ts"; import { selfHostAnalytics } from "./product-analytics.ts"; +import { selfHostFeedbackWebhook } from "./feedback-webhook.ts"; import type { SourceTextFile as SourceFile } from "@executor-js/sdk/core"; import type { selfHostExecutorServices } from "./executor-services.ts"; @@ -74,11 +75,16 @@ export const selfHostRouteMap = (options: { const { skills, egress, executorServices, dashboard } = options; const auth = yield* SelfHostAuth.pipe(Effect.provide(executorServices)); const analytics = yield* selfHostAnalytics; - /** Requests and background schedules record through this instance's sink unless it opted out. */ + const webhook = yield* selfHostFeedbackWebhook(auth.origin); + const feedbackSink = analytics?.product ?? (yield* ProductAnalytics); + /** The operator's feedback receiver remains available when usage analytics are off. */ const observed = (effect: Effect.Effect) => - analytics === undefined - ? effect - : effect.pipe(Effect.provideService(ProductAnalytics, analytics.product)); + effect.pipe( + Effect.provideService( + ProductAnalytics, + webhook === undefined ? feedbackSink : { ...feedbackSink, submitFeedback: webhook }, + ), + ); const scheduler = yield* Effect.gen(function* () { const executor = yield* Effect.flatten(HostedExecutor); const authorize = yield* ScheduledAuthority; diff --git a/apps/hosted/server/src/contracts/feedback.ts b/apps/hosted/server/src/contracts/feedback.ts index d9688cda56..a4f54b6302 100644 --- a/apps/hosted/server/src/contracts/feedback.ts +++ b/apps/hosted/server/src/contracts/feedback.ts @@ -19,7 +19,7 @@ export const HostedFeedback = HttpApiGroup.make("feedback") .annotate(OpenApi.Summary, "Submit feedback") .annotate( OpenApi.Description, - "Send feedback about Executor to the Executor team. Returns accepted only after ingestion succeeds. Returns FeedbackDisabled when this instance has analytics turned off. Do not include credentials or other sensitive information.", + "Send feedback about Executor to this instance's configured receiver (the Executor team by default). Returns accepted only after ingestion succeeds. Returns FeedbackDisabled when neither analytics nor an operator feedback receiver is available. Do not include credentials or other sensitive information.", ), ) .middleware(RequireOrganization); diff --git a/e2e/support/managed-server.ts b/e2e/support/managed-server.ts index e3f5791a3e..38dc1f8cc9 100644 --- a/e2e/support/managed-server.ts +++ b/e2e/support/managed-server.ts @@ -49,6 +49,8 @@ class ServerFailed extends Schema.TaggedError()("ServerFailed", { const maxClockOffset = 40 * 86_400_000; /** Operator settings a scenario may turn on between product generations. */ export const OperatorSettings = Schema.Struct({ + EXECUTOR_FEEDBACK_WEBHOOK_URL: Schema.optionalKey(Schema.String), + EXECUTOR_FEEDBACK_WEBHOOK_SECRET: Schema.optionalKey(Schema.String), EXECUTOR_URL_ALLOW_LOOPBACK_HTTP: Schema.optionalKey(Schema.String), EXECUTOR_URL_ALLOW_HTTP_ORIGINS: Schema.optionalKey(Schema.String), NODE_EXTRA_CA_CERTS: Schema.optionalKey(Schema.NonEmptyString), diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index dec7f4f35d..2d2e702496 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -3218,6 +3218,30 @@ export const scenarios = plan({ local: na("Local feedback has no organization; the local analytics scenario covers it."), }, }, + feedbackWebhook: { + fixtures: "actors", + file: "feedback-webhook.spec.ts", + title: "self-host feedback webhook awaits acceptance and preserves analytics fallback", + managementApps: ["owner"], + targets: { + "self-host": scheduled, + cloud: na("Operator feedback webhooks are self-host only."), + local: na("Operator feedback webhooks are self-host only."), + }, + }, + feedbackWebhookOptOut: { + fixtures: "actors", + file: "feedback-webhook.spec.ts", + title: + "self-host feedback webhook works with analytics disabled and surfaces receiver failures", + managementApps: ["owner"], + serverEnvironment: { DO_NOT_TRACK: "1" }, + targets: { + "self-host": scheduled, + cloud: na("Operator feedback webhooks are self-host only."), + local: na("Operator feedback webhooks are self-host only."), + }, + }, selfHostAnalytics: { fixtures: "actors", file: "instance-analytics.spec.ts", diff --git a/e2e/tests/docker-release.spec.ts b/e2e/tests/docker-release.spec.ts index 989072739a..9f820280f7 100644 --- a/e2e/tests/docker-release.spec.ts +++ b/e2e/tests/docker-release.spec.ts @@ -42,6 +42,117 @@ const pinnedWorkerd = Effect.gen(function* () { return `workerd ${manifest.devDependencies.workerd.replace(/^\d+\.(\d{4})(\d{2})(\d{2})\.\d+$/, "$1-$2-$3")}`; }); +/** Exercise the product Worker, not Bun: workerd supports manual redirects but not error mode. */ +it.live("released image delivers operator feedback with analytics disabled", () => + Effect.scoped( + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const image = yield* Config.String("EXECUTOR_E2E_DOCKER_IMAGE"); + const id = `executor-feedback-${randomBytes(8).toString("hex")}`; + const run = (args: readonly string[]) => + processes.string(ChildProcess.make("docker", args, { stderr: "inherit" })); + yield* Effect.acquireRelease(run(["volume", "create", id]), () => + run(["volume", "rm", id]).pipe(Effect.orDie), + ); + yield* Effect.acquireRelease( + run([ + "run", + "--detach", + "--network", + "none", + "--name", + id, + "--volume", + `${id}:/app/data`, + "--env", + "BETTER_AUTH_URL=http://localhost:4400", + "--env", + "DO_NOT_TRACK=1", + "--env", + "EXECUTOR_FEEDBACK_WEBHOOK_URL=http://127.0.0.1:4567/feedback", + "--env", + "EXECUTOR_FEEDBACK_WEBHOOK_SECRET=synthetic-native-feedback", + image, + ]), + () => run(["rm", "--force", "--volumes", id]).pipe(Effect.orDie), + ); + // The receiver and HTTP driver share only this container's network namespace. + // No host ports, extra Docker networks or external services are used. + const result = yield* run([ + "run", + "--rm", + "--network", + `container:${id}`, + "node:24.22.1-bookworm-slim", + "node", + "--input-type=module", + "-e", + ` +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { setTimeout } from "node:timers/promises"; +const received = []; +let status = 204; +const receiver = createServer(async (request, response) => { + let text = ""; + for await (const chunk of request) text += chunk; + received.push({ authorization: request.headers.authorization, payload: JSON.parse(text) }); + response.writeHead(status, { location: "/feedback" }); + response.end(); +}); +await new Promise(resolve => receiver.listen(4567, "127.0.0.1", resolve)); +try { + const origin = "http://localhost:4400"; + const request = (path, data, cookie) => fetch(origin + path, { + method: data === undefined ? "GET" : "POST", + headers: { origin, "content-type": "application/json", ...(cookie ? { cookie } : {}) }, + ...(data === undefined ? {} : { body: JSON.stringify(data) }) + }); + let healthy = false; + for (let attempt = 0; attempt < 200; attempt++) { + try { if ((await request("/health")).ok) { healthy = true; break; } } catch {} + await setTimeout(250); + } + assert.ok(healthy, "native product did not become healthy"); + const setup = await request("/api/auth/self-host/setup", { + name: "Feedback Owner", email: "feedback@example.test", + password: "Synthetic-feedback-password-123!", organizationName: "Feedback lab" + }); + assert.equal(setup.status, 200); + const cookie = setup.headers.getSetCookie().map(part => part.split(";")[0]).join("; "); + const organizations = await request("/api/auth/organization/list", undefined, cookie); + const [organization] = await organizations.json(); + const path = "/api/organizations/" + organization.id + "/feedback"; + const accepted = await request(path, { message: "Native feedback" }, cookie); + assert.equal(accepted.status, 200); + assert.deepEqual(await accepted.json(), { status: "accepted" }); + assert.equal(received.length, 1); + assert.equal(received[0].authorization, "Bearer synthetic-native-feedback"); + assert.equal(received[0].payload.user.email, "feedback@example.test"); + assert.equal(received[0].payload.organization.id, organization.id); + assert.equal(received[0].payload.message, "Native feedback"); + assert.equal(received[0].payload.instanceOrigin, origin); + for (const failure of [302, 503]) { + status = failure; + const rejected = await request(path, { message: "Native receiver failure" }, cookie); + assert.equal(rejected.status, 503); + assert.equal((await rejected.json())._tag, "FeedbackUnavailable"); + } + assert.equal(received.length, 3, "redirect was followed or delivery retried"); + console.log("Native feedback: accepted, attributed, redirects and failures refused"); +} finally { + receiver.closeAllConnections(); + await new Promise(resolve => receiver.close(resolve)); +} + `, + ]); + expect(result).toContain( + "Native feedback: accepted, attributed, redirects and failures refused", + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + for (const mode of ["explicit", "local", "railway"] as const) it.live(`released image keeps login and encrypted credentials across restart (${mode})`, () => Effect.scoped( diff --git a/e2e/tests/feedback-webhook.spec.ts b/e2e/tests/feedback-webhook.spec.ts new file mode 100644 index 0000000000..694ecd30bf --- /dev/null +++ b/e2e/tests/feedback-webhook.spec.ts @@ -0,0 +1,174 @@ +/** Real self-host feedback requests wait for the operator's HTTP receiver. */ +import { expect, layer } from "@effect/vitest"; +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Schema } from "effect"; +import { HttpRouter, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http"; +import { createServer } from "node:http"; +import { scenarios } from "../test-plan.ts"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { managementApp } from "../support/management-app.ts"; +import { Target } from "../support/platform.ts"; +import { serverControl } from "../support/server-control.ts"; +import { deliveredAnalytics } from "../support/instance-analytics.ts"; + +const Payload = Schema.Struct({ + message: Schema.String, + submittedAt: Schema.String, + organization: Schema.Struct({ id: Schema.String }), + user: Schema.Struct({ id: Schema.String, email: Schema.String }), + caller: Schema.Struct({ source: Schema.String }), + instanceOrigin: Schema.String, + title: Schema.String, + body: Schema.String, +}); + +const exercise = (analyticsOff: boolean) => + Effect.gen(function* () { + const api = yield* Api; + const actors = yield* Actors; + const target = yield* Target; + const root = `/api/organizations/${actors.organization.id}`; + const post = (message: string) => + api.request(actors.owner, "POST", `${root}/feedback`, { message }); + if (analyticsOff) { + const disabled = yield* post("No receiver configured"); + expect(disabled.status).toBe(409); + expect(disabled.body).toMatchObject({ _tag: "FeedbackDisabled" }); + } + const received = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(release, undefined)); + const requests: { authorization: string | undefined; payload: typeof Payload.Type }[] = []; + let status = 204; + let hold = true; + const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + requests.push({ + authorization: request.headers.authorization, + payload: yield* request.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(Payload))), + }); + yield* Deferred.succeed(received, undefined); + if (hold) yield* Deferred.await(release); + if (status === 408) return yield* Effect.never; + return HttpServerResponse.empty({ status, headers: { location: "/feedback" } }); + }).pipe(Effect.orDie); + const services = yield* Layer.build( + Layer.fresh( + HttpRouter.serve(HttpRouter.add("POST", "/feedback", handler), { + disableLogger: true, + disableListenLog: true, + }).pipe( + Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 })), + ), + ), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("Feedback receiver needs TCP"); + const receiver = `http://127.0.0.1:${server.address.port}/feedback`; + yield* serverControl("stop"); + yield* serverControl("environment", 200, { + EXECUTOR_FEEDBACK_WEBHOOK_URL: receiver, + EXECUTOR_FEEDBACK_WEBHOOK_SECRET: analyticsOff ? "" : "synthetic-feedback-secret", + }); + yield* serverControl("start"); + const viewer = yield* body( + Schema.Struct({ userId: Schema.String }), + yield* api.request(actors.owner, "GET", "/api/viewer"), + ); + let completed = false; + const pending = yield* post("Operator feedback").pipe( + Effect.tap(() => + Effect.sync(() => { + completed = true; + }), + ), + Effect.forkChild, + ); + yield* Deferred.await(received); + expect(completed).toBe(false); + expect(requests).toHaveLength(1); + expect(requests[0]).toMatchObject({ + authorization: analyticsOff ? undefined : "Bearer synthetic-feedback-secret", + payload: { + message: "Operator feedback", + organization: { id: actors.organization.id }, + user: { id: viewer.userId, email: expect.stringContaining("@") }, + instanceOrigin: target.metadata.origin, + title: "Executor feedback", + caller: { source: "dashboard" }, + body: expect.stringContaining("From: "), + }, + }); + expect(Number.isNaN(Date.parse(requests[0]!.payload.submittedAt))).toBe(false); + hold = false; + yield* Deferred.succeed(release, undefined); + const accepted = yield* Fiber.join(pending); + expect(accepted.status).toBe(200); + expect(accepted.body).toEqual({ status: "accepted" }); + + // Exercise the actual Executor app route as an agent does, not just the hosted endpoint. + const { app } = yield* managementApp(actors.owner); + const called = yield* api.request(actors.owner, "POST", `${root}/apps/${app.id}/tools/call`, { + tool: "feedback.submit", + kind: "mutation", + input: { body: { message: "Agent operator feedback" } }, + }); + expect(called.status, JSON.stringify(called.body)).toBe(200); + expect(called.body).toEqual({ status: "accepted" }); + expect(requests.at(-1)?.payload.message).toBe("Agent operator feedback"); + expect(requests.at(-1)?.payload.caller.source).toBe("api"); + + status = 503; + const rejected = yield* post("Receiver failure"); + expect(rejected.status).toBe(503); + expect(rejected.body).toMatchObject({ _tag: "FeedbackUnavailable" }); + expect(JSON.stringify(rejected.body)).not.toContain(receiver); + expect(requests).toHaveLength(3); + status = 302; + const redirected = yield* post("Receiver redirect"); + expect(redirected.status).toBe(503); + expect(redirected.body).toMatchObject({ _tag: "FeedbackUnavailable" }); + expect(requests).toHaveLength(4); + if (analyticsOff) { + status = 408; + const timedOut = yield* post("Receiver never responds"); + expect(timedOut.status).toBe(503); + expect(timedOut.body).toMatchObject({ _tag: "FeedbackUnavailable" }); + expect(requests).toHaveLength(5); + } + + // Empty URL is the same as unset; a secret alone must not enable delivery. + yield* serverControl("stop"); + yield* serverControl("environment", 200, { EXECUTOR_FEEDBACK_WEBHOOK_URL: "" }); + yield* serverControl("start"); + const fallback = yield* post("Feedback after disabling the webhook"); + expect(fallback.status).toBe(analyticsOff ? 409 : 200); + expect(requests).toHaveLength(analyticsOff ? 5 : 4); + yield* serverControl("stop"); + const fs = yield* FileSystem.FileSystem; + const logs = yield* fs.readFileString(`${target.directory}/server.log`); + expect(logs).not.toContain(receiver); + expect(logs).not.toContain("synthetic-feedback-secret"); + expect(logs).toContain("Feedback webhook delivery failed"); + const analytics = yield* deliveredAnalytics; + if (analyticsOff) expect(analytics.text).toBe(""); + else { + expect( + analytics.events.some( + (event) => event.properties.message === "Feedback after disabling the webhook", + ), + ).toBe(true); + expect( + analytics.events.some((event) => event.properties.message === "Operator feedback"), + ).toBe(false); + } + }); + +layer(HostedLive, { excludeTestServices: true })("Self-host feedback webhook", (it) => { + it.effect(scenarios.feedbackWebhook.title, (context) => withHostedCase(context, exercise(false))); + it.effect(scenarios.feedbackWebhookOptOut.title, (context) => + withHostedCase(context, exercise(true)), + ); +}); diff --git a/packages/app-templates/executor/skills/executor/feedback.md b/packages/app-templates/executor/skills/executor/feedback.md index a26b7ec2fd..89171a8ab4 100644 --- a/packages/app-templates/executor/skills/executor/feedback.md +++ b/packages/app-templates/executor/skills/executor/feedback.md @@ -1,7 +1,7 @@ # Send feedback -Executor's team reads feedback from agents and the people they work for. Send -it with the Executor app's `feedback.submit` tool. Find its path with +Executor's team reads feedback from agents and the people they work for. A +self-host operator may configure their own receiver instead. Send feedback with the Executor app's `feedback.submit` tool. Find its path with `tools.search` and its signature with `tools.search.describe`. Hosted Executor also needs the organization from `context.get`. If the tool reports that feedback is disabled on this instance, skip it. diff --git a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts index 8fec88bac8..b84d23ae64 100644 --- a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts +++ b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts @@ -17,5 +17,5 @@ export const executorSkillDigests: Readonly> = { "app-authoring/workflows.md": "1496edbdb85e138715efa138c63068a11674e6188179f6ee0a92b81e6d9d63e2", "code-mode/SKILL.md": "683999bd21e33e53a0cc63fef2caace026e12e3f7c7544d5cf30da7d3cc45586", "executor/SKILL.md": "d98a4bfa40ee0afbf8c31fac67b27b69c400ee2c3b434479c08deb72f0e07e99", - "executor/feedback.md": "11397bd260917e5a21e92897a19368d0426bf5944a7c081adc2c061d1ea797cd" + "executor/feedback.md": "29e684fc80cef9f9fae46e8fe183c310a64f12d2737dbe0bcb855a4450dfd95b" };