From 0ae1961370b67c4af30af203fe407085effe0837 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 21:16:26 -0700 Subject: [PATCH 1/6] fix(search): harden live search accuracy, provider queries, and request cost - GitHub: batch code search by bytes under its 1,000-byte query limit, send date bounds as one updated:start..end range, keep qualifiers outside grouped text, and exclude dateless code from dated searches - Calendar: keep one verified copy of a meeting shared across calendars and order dated agendas by start - Clean Gmail, Calendar, and Confluence text; report Google quota 403s as rate limits - Report partial only for degraded coverage; drop cursors that would skip merged-out results - Share one account session between search and read; filter before verifying, verify in parallel, pass GitLab evidence, and reuse listed accounts - Pin DNS on the request as well as the agent (Bun ignores agent lookups), and reuse pinned keep-alive connections with compressed responses - Route the Search MCP chat tool to the assistant chat endpoint --- .../connectors/google-workspace/api-errors.ts | 15 +- apps/sim/connectors/utils.ts | 20 +- .../core/security/input-validation.server.ts | 64 +- ...ecure-fetch-connection-pool.server.test.ts | 148 ++++ .../lib/knowledge/application/chat.test.ts | 2 +- apps/sim/lib/knowledge/application/chat.ts | 2 +- apps/sim/lib/knowledge/search/diagnostics.ts | 7 + apps/sim/lib/sim-search/live/README.md | 4 +- .../lib/sim-search/live/account-session.ts | 125 ++++ apps/sim/lib/sim-search/live/accounts.ts | 148 ++-- .../lib/sim-search/live/application.test.ts | 174 +++++ apps/sim/lib/sim-search/live/application.ts | 694 +++++++++--------- apps/sim/lib/sim-search/live/atlassian.ts | 132 ++-- apps/sim/lib/sim-search/live/coda-mcp.test.ts | 2 +- apps/sim/lib/sim-search/live/coda-mcp.ts | 26 +- apps/sim/lib/sim-search/live/dates.test.ts | 2 +- apps/sim/lib/sim-search/live/dates.ts | 6 + .../sim/lib/sim-search/live/github-service.ts | 5 +- apps/sim/lib/sim-search/live/github.ts | 178 +++-- apps/sim/lib/sim-search/live/gitlab.ts | 17 +- .../sim-search/live/google-service.test.ts | 12 +- .../sim/lib/sim-search/live/google-service.ts | 29 +- apps/sim/lib/sim-search/live/google.ts | 203 ++--- apps/sim/lib/sim-search/live/http.test.ts | 46 ++ apps/sim/lib/sim-search/live/http.ts | 55 +- apps/sim/lib/sim-search/live/pages.test.ts | 39 + apps/sim/lib/sim-search/live/pages.ts | 37 +- apps/sim/lib/sim-search/live/policy.ts | 12 +- .../sim/lib/sim-search/live/providers.test.ts | 186 ++++- .../sim-search/live/service-session.test.ts | 7 +- .../lib/sim-search/live/service-session.ts | 14 +- apps/sim/lib/sim-search/live/slack.ts | 9 +- apps/sim/lib/sim-search/live/text.test.ts | 32 + apps/sim/lib/sim-search/live/text.ts | 57 ++ apps/sim/lib/sim-search/live/types.ts | 16 +- 35 files changed, 1837 insertions(+), 688 deletions(-) create mode 100644 apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts create mode 100644 apps/sim/lib/sim-search/live/account-session.ts create mode 100644 apps/sim/lib/sim-search/live/pages.test.ts create mode 100644 apps/sim/lib/sim-search/live/text.test.ts create mode 100644 apps/sim/lib/sim-search/live/text.ts diff --git a/apps/sim/connectors/google-workspace/api-errors.ts b/apps/sim/connectors/google-workspace/api-errors.ts index c1b67f95bd6..8ca0b403d16 100644 --- a/apps/sim/connectors/google-workspace/api-errors.ts +++ b/apps/sim/connectors/google-workspace/api-errors.ts @@ -54,6 +54,13 @@ const RATE_LIMIT_REASONS = new Set([ 'RATE_LIMIT_EXCEEDED', ]) +/** Whether a Google error reason reports an exhausted rate or usage quota rather than a denial. */ +export function isGoogleQuotaReason(reason: string): boolean { + return ( + RATE_LIMIT_REASONS.has(reason) || reason === 'dailyLimitExceeded' || reason === 'quotaExceeded' + ) +} + export function safeGoogleErrorReasons(reasons: readonly string[]): string[] { return [...new Set(reasons.filter((reason) => SAFE_REASONS.has(reason)))] } @@ -119,13 +126,7 @@ export class GoogleApiError extends ConnectorSourceError { const safeReasons = safeGoogleErrorReasons(reasons) const suffix = safeReasons.length ? ` (${safeReasons.join(', ')})` : '' const category = - status === 429 || - safeReasons.some( - (reason) => - RATE_LIMIT_REASONS.has(reason) || - reason === 'dailyLimitExceeded' || - reason === 'quotaExceeded' - ) + status === 429 || safeReasons.some(isGoogleQuotaReason) ? 'rate_limit' : status >= 500 ? 'provider_unavailable' diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 88aee7614f7..a85d26048e2 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -362,15 +362,25 @@ export function looksLikeHtml(value: string): boolean { * punctuation as numeric references, which previously reached the index verbatim. */ export function htmlToPlainText(html: string): string { - const text = html - .replace(/<[^>]*>/g, ' ') - .replace(HTML_ENTITY_PATTERN, (raw: string, hex?: string, decimal?: string, named?: string) => { + return decodeHtmlEntities(html.replace(/<[^>]*>/g, ' ')) + .replace(/\s+/g, ' ') + .trim() +} + +/** + * Decodes HTML character references without touching markup or whitespace. Use for text a + * provider HTML-escapes but does not mark up, such as Gmail message snippets. + */ +export function decodeHtmlEntities(text: string): string { + return text.replace( + HTML_ENTITY_PATTERN, + (raw: string, hex?: string, decimal?: string, named?: string) => { if (named !== undefined) return NAMED_ENTITIES[named] ?? raw if (hex !== undefined) return decodeCharacterReference(raw, Number.parseInt(hex, 16)) if (decimal !== undefined) return decodeCharacterReference(raw, Number.parseInt(decimal, 10)) return raw - }) - return text.replace(/\s+/g, ' ').trim() + } + ) } /** diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 9c638d35fc1..8c89ed838c9 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -356,6 +356,17 @@ export interface SecureFetchOptions { proxyUrl?: string /** Hide credential-derived URL details from validation logs. */ logUrlValidationDetails?: boolean + /** + * Ask for a gzip, deflate, or brotli body. The body is decoded before it is returned, and + * `maxResponseBytes` bounds the decoded bytes, so a compression bomb still stops at the cap. + */ + acceptCompressed?: boolean + /** + * Reuses keep-alive connections to the same pinned address across requests. A connection is + * only ever reused for the IP it was opened to, so every request keeps its DNS pinning. The + * owner must call {@link PinnedConnectionPool.destroy} once its requests have finished. + */ + connectionPool?: PinnedConnectionPool /** * Where this request's URL came from. Carried on the options so the same * policy is re-applied to every redirect hop rather than re-derived — a hop @@ -447,6 +458,43 @@ function resolveRedirectUrl(baseUrl: string, location: string): string { } } +/** Keep-alive agents keyed by protocol, host, port, and the pinned address they connect to. */ +export interface PinnedConnectionPool { + /** Undefined once destroyed, so a late request falls back to a single-use pinned agent. */ + agent(isHttps: boolean, host: string, port: number, resolvedIP: string): http.Agent | undefined + destroy(): void +} + +/** + * Creates a request-scoped pool of pinned keep-alive agents. Reusing a connection skips the TCP + * and TLS handshakes that otherwise dominate short provider API calls. + */ +export function createPinnedConnectionPool(): PinnedConnectionPool { + const agents = new Map() + let destroyed = false + return { + agent(isHttps, host, port, resolvedIP) { + if (destroyed) return undefined + const key = JSON.stringify([isHttps, host, port, resolvedIP]) + let agent = agents.get(key) + if (!agent) { + const options: http.AgentOptions = { + keepAlive: true, + lookup: createPinnedLookup(resolvedIP), + } + agent = isHttps ? new https.Agent(options) : new http.Agent(options) + agents.set(key, agent) + } + return agent + }, + destroy() { + destroyed = true + for (const agent of agents.values()) agent.destroy() + agents.clear() + }, + } +} + /** * Creates a DNS lookup function that always returns a pre-resolved IP address. * Use this to prevent DNS rebinding (TOCTOU) attacks when connecting to @@ -1088,6 +1136,11 @@ export async function secureFetchWithPinnedIP( const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort let agent: http.Agent | undefined + /** + * Bun ignores a `lookup` set on an Agent and honors one on the request, while Node honors + * both. A pinned direct connection sets it in both places so pinning holds in either runtime. + */ + let pinnedLookup: LookupFunction | undefined if (outboundDispatcher) { agent = undefined } else if (options.proxyUrl) { @@ -1096,12 +1149,16 @@ export async function secureFetchWithPinnedIP( // targets tunnel via CONNECT, http targets use absolute-URI forwarding. agent = isHttps ? new HttpsProxyAgent(options.proxyUrl) : new HttpProxyAgent(options.proxyUrl) } else { - const lookup = createPinnedLookup(resolvedIP) - const agentOptions: http.AgentOptions = { lookup } - agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions) + pinnedLookup = createPinnedLookup(resolvedIP) + agent = + options.connectionPool?.agent(isHttps, parsed.hostname, port, resolvedIP) ?? + (isHttps + ? new https.Agent({ lookup: pinnedLookup }) + : new http.Agent({ lookup: pinnedLookup })) } const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} + if (options.acceptCompressed) sanitizedHeaders['accept-encoding'] = 'gzip, deflate, br' if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) { sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT } @@ -1123,6 +1180,7 @@ export async function secureFetchWithPinnedIP( method: options.method || 'GET', headers: sanitizedHeaders, agent, + ...(pinnedLookup ? { lookup: pinnedLookup } : {}), timeout: options.timeout || 300000, } diff --git a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts new file mode 100644 index 00000000000..97d67cbc308 --- /dev/null +++ b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts @@ -0,0 +1,148 @@ +/** + * @vitest-environment node + */ +import http from 'node:http' +import type { AddressInfo } from 'node:net' +import { gzipSync } from 'node:zlib' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/security/dns', () => ({ + resolveHostAddresses: vi.fn(), + preferIpv4: (addresses: string[]) => addresses[0], +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + isHosted: false, + getEgressAllowedHosts: () => undefined, + getEgressAllowedIpRanges: () => undefined, + isLegacyPrivateDatabaseAccessAllowed: () => false, + getProxyUrl: () => undefined, +})) + +import { + createPinnedConnectionPool, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +const servers: http.Server[] = [] + +afterEach(() => { + for (const server of servers.splice(0)) { + server.closeAllConnections() + server.close() + } +}) + +/** Starts a loopback server that counts the TCP connections it accepts. */ +async function startServer(handler: http.RequestListener) { + const server = http.createServer(handler) + servers.push(server) + let connections = 0 + server.on('connection', () => connections++) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + return { + origin: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + connections: () => connections, + } +} + +describe('secureFetchWithPinnedIP connection reuse', () => { + it('reuses one pinned connection across requests that share a pool', async () => { + const server = await startServer((_req, res) => res.end('ok')) + const pool = createPinnedConnectionPool() + try { + for (let index = 0; index < 3; index++) { + const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + connectionPool: pool, + }) + await expect(response.text()).resolves.toBe('ok') + } + expect(server.connections()).toBe(1) + } finally { + pool.destroy() + } + }) + + it('opens a fresh connection per request without a pool', async () => { + const server = await startServer((_req, res) => res.end('ok')) + for (let index = 0; index < 2; index++) { + const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + await response.text() + } + expect(server.connections()).toBe(2) + }) + + it('never shares an agent between different pinned addresses', () => { + const pool = createPinnedConnectionPool() + try { + const first = pool.agent(true, 'api.example.com', 443, '203.0.113.1') + expect(pool.agent(true, 'api.example.com', 443, '203.0.113.1')).toBe(first) + expect(pool.agent(true, 'api.example.com', 443, '203.0.113.2')).not.toBe(first) + expect(pool.agent(true, 'other.example.com', 443, '203.0.113.1')).not.toBe(first) + } finally { + pool.destroy() + } + }) +}) + +describe('secureFetchWithPinnedIP compressed responses', () => { + it('asks for compression only when requested and returns the decoded body', async () => { + const encodings: (string | undefined)[] = [] + const server = await startServer((req, res) => { + encodings.push(req.headers['accept-encoding']) + if (req.headers['accept-encoding']?.includes('gzip')) { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(gzipSync(Buffer.from('{"ok":true}'))) + } else { + res.end('{"ok":true}') + } + }) + const compressed = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + acceptCompressed: true, + }) + await expect(compressed.json()).resolves.toEqual({ ok: true }) + const plain = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + await expect(plain.json()).resolves.toEqual({ ok: true }) + expect(encodings).toEqual(['gzip, deflate, br', undefined]) + }) +}) + +describe('secureFetchWithPinnedIP address pinning', () => { + it.each([ + ['a fresh agent', undefined], + ['a pooled agent', createPinnedConnectionPool()], + ])('pins the address on the request itself with %s', async (_label, pool) => { + const server = await startServer((_req, res) => res.end('ok')) + const request = vi.spyOn(http, 'request') + try { + const response = await secureFetchWithPinnedIP( + server.origin.replace('127.0.0.1', 'localhost'), + '127.0.0.1', + { + profile: 'configuredEndpoint', + connectionPool: pool, + } + ) + await response.text() + const options = request.mock.calls[0]?.[0] as http.RequestOptions + const lookup = options.lookup as unknown as ( + hostname: string, + options: object, + callback: (error: Error | null, address: string, family: number) => void + ) => void + const resolved = await new Promise((resolve) => + lookup('localhost', {}, (_error, address) => resolve(address)) + ) + expect(resolved).toBe('127.0.0.1') + } finally { + request.mockRestore() + pool?.destroy() + } + }) +}) diff --git a/apps/sim/lib/knowledge/application/chat.test.ts b/apps/sim/lib/knowledge/application/chat.test.ts index b1e6fc00650..2ea5c6de390 100644 --- a/apps/sim/lib/knowledge/application/chat.test.ts +++ b/apps/sim/lib/knowledge/application/chat.test.ts @@ -140,7 +140,7 @@ describe('organization Search Assistant chat', () => { userId: 'member-1', organizationId: 'org-1', chatId: 'private-chat', - goRoute: '/api/mothership/execute', + goRoute: '/api/mothership', interactive: false, autoExecuteTools: true, secretActorUserId: null, diff --git a/apps/sim/lib/knowledge/application/chat.ts b/apps/sim/lib/knowledge/application/chat.ts index cd49037d280..dc0ccdb0498 100644 --- a/apps/sim/lib/knowledge/application/chat.ts +++ b/apps/sim/lib/knowledge/application/chat.ts @@ -172,7 +172,7 @@ export const organizationSearchChat: OperationUseCase< organizationId, chatId, simRequestId: messageId, - goRoute: '/api/mothership/execute', + goRoute: '/api/mothership', interactive: false, autoExecuteTools: true, secretActorUserId: null, diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index fe0086eed4c..95ab1a6505b 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -64,6 +64,13 @@ export type SearchStage = | 'source_overview.searchable' | 'access_batch.connectors' | 'access_batch.live_proof' + | 'live.policies' + | 'live.accounts' + | 'live.resolve' + | 'live.session' + | 'live.search' + | 'live.verify' + | 'live.read' /** Fixed, content-free fields. Never pass queries, filters, document identities, SQL, or errors. */ export interface SearchDiagnosticMetadata { diff --git a/apps/sim/lib/sim-search/live/README.md b/apps/sim/lib/sim-search/live/README.md index 86bf763a9fb..f58cfaa420f 100644 --- a/apps/sim/lib/sim-search/live/README.md +++ b/apps/sim/lib/sim-search/live/README.md @@ -48,7 +48,7 @@ Selected labels are alternatives. Source date/query settings are authoritative r ### Google Calendar -Member mode searches calendars accessible through the member's account. Service mode verifies the account's primary-calendar identity, Directory customer, and source user selection before delegating to that same Workspace user. Selected calendar IDs constrain retrieval and source verification; `primary` means that member's primary calendar. The admin picker browses the delegated administrator's calendar list and stores `primary` as a per-member alias. Service search delegates with only `calendar.events.readonly` (plus `admin.directory.user.readonly` for the Directory check), the same scope as the indexed crawl; all-day events take the calendar's time zone from the events list response. The picker additionally needs `calendar.readonly` ([CalendarList authorization](https://developers.google.com/workspace/calendar/api/v3/reference/calendarList/list)). +Member mode searches calendars accessible through the member's account. A meeting visible in several calendars is returned once, preferring the member's primary calendar, and date-bounded agendas are ordered by scheduled start across calendars. Service mode verifies the account's primary-calendar identity, Directory customer, and source user selection before delegating to that same Workspace user. Selected calendar IDs constrain retrieval and source verification; `primary` means that member's primary calendar. The admin picker browses the delegated administrator's calendar list and stores `primary` as a per-member alias. Service search delegates with only `calendar.events.readonly` (plus `admin.directory.user.readonly` for the Directory check), the same scope as the indexed crawl; all-day events take the calendar's time zone from the events list response. The picker additionally needs `calendar.readonly` ([CalendarList authorization](https://developers.google.com/workspace/calendar/api/v3/reference/calendarList/list)). Each event must exist under the source's delegated token, be in an allowed calendar, not be cancelled, and overlap the source's configured rolling time window. The existing default is 30 days before and after the request. A stable UTC-day envelope around that window is intersected with the user's date bounds in the provider query, ensuring recurring events expand and query bounds stay stable between pages. The exact rolling source window is still checked for each result. Nonoverlapping date ranges return no results; continuations spanning a UTC-day change may require a fresh search. A source search query is checked with Calendar's event search and exact event-ID matching. All-day events use the calendar's timezone. Attendee details follow the source's include-attendees setting. The member's API access still determines which event details they can see. @@ -58,7 +58,7 @@ Member mode only. Search uses the connected user's Slack real-time search grant ### GitHub -Member mode searches issues, code, and repositories permitted by the connected token. Explicit repository/organization/user qualifiers narrow the user's query. Default discovery is bounded to up to 100 affiliated repositories, and provider pagination/search caps still apply. +Member mode searches issues, code, and repositories permitted by the connected token. Explicit repository/organization/user qualifiers narrow the user's query. Default discovery is bounded to up to 100 affiliated repositories, sent as `repo:` qualifiers in at most four batches per search kind. Code batches stay under code search's 1,000-byte query limit, issue batches are larger, and a long query that cannot fit every repository reports the searched subset. Date bounds become one `updated:start..end` range, since GitHub ORs repeated qualifiers. Provider pagination/search caps still apply. REST code search returns no file dates and accepts no date qualifier, so date-filtered searches cover issues and pull requests only. In service mode, an administrator connects a GitHub App installation and selects repositories one by one in Sources. Each source pins the provider-verified repository ID and may narrow code files by directory and extension. Search queries the member's own GitHub connection with `repo:` qualifiers drawn only from active sources. For each candidate, Sim checks that the current App installation still covers that repository, mints a repository-scoped read token, and compares repository and owner IDs returned under both the App and member tokens. It then checks the per-repository code filters. Reads use the member token and repeat these checks. A personal repository outside the selected sources is never searched, even if the member can access it. GitHub REST code search covers the default branch; live Sources therefore do not offer a branch setting. diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts new file mode 100644 index 00000000000..b84e7cc452a --- /dev/null +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -0,0 +1,125 @@ +import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' +import type { ResourceOwner } from '@/lib/core/resource-scope' +import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server' +import type { ResolvedLiveAccount } from '@/lib/sim-search/live/accounts' +import { createCodaMcpClient, readCodaMcp, searchCodaMcp } from '@/lib/sim-search/live/coda-mcp' +import { createAdminGitLabSession } from '@/lib/sim-search/live/gitlab-admin' +import { createNativeClient } from '@/lib/sim-search/live/http' +import { createPolicyVerifier } from '@/lib/sim-search/live/policy' +import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' +import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' +import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' +import { readNativeProvider, searchNativeProvider } from '@/lib/sim-search/live/providers' +import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search' +import { createLiveServiceSession } from '@/lib/sim-search/live/service-session' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +type Reference = Pick< + NativeDocument, + 'id' | 'container' | 'kind' | 'revision' | 'threadId' | 'accessMetadata' +> + +/** One member account's provider clients and the source boundary that bounds its results. */ +export interface LiveAccountSession { + /** The effective policy: the service source's settings in service mode, else the member's. */ + policy: LiveSearchPolicy + /** Service verification covered a bounded subset of the source's configured users. */ + servicePartial: boolean + search(input: NativeSearchInput): Promise + /** True only when the document is inside the source boundary and the member may read it. */ + verify(document: Reference): Promise + read(reference: Reference, filters?: WorkspaceSearchFilters): Promise + /** Checks a fetched document against the source settings saved now, not at session start. */ + verifyCurrent(document: NativeDocument): Promise +} + +interface OpenLiveAccountSessionInput { + owner: ResourceOwner + userId: string + resolved: ResolvedLiveAccount + policies: Record + signal: AbortSignal + pool?: PinnedConnectionPool +} + +/** + * Opens the clients a search or read of one account needs: the member's provider client (or + * Coda MCP client), an administrator GitLab session, and the source verifier. Search and read + * share this so both apply exactly the same boundary. + */ +export async function openLiveAccountSession( + input: OpenLiveAccountSessionInput +): Promise { + const { owner, userId, resolved, signal } = input + const { account } = resolved + const provider = account.provider + const origin = + 'origin' in resolved ? resolved.origin : LIVE_SEARCH_PROVIDER_CATALOG[provider].origin + const client = + account.type === 'managed_mcp' + ? null + : createNativeClient({ origin, accessToken: resolved.accessToken, signal, pool: input.pool }) + const admin = + 'adminSource' in resolved && client + ? await createAdminGitLabSession({ + owner, + userId, + source: resolved.adminSource, + token: resolved.accessToken, + client, + signal, + }) + : undefined + const mcp = client ? undefined : await createCodaMcpClient(owner, userId, account.id, signal) + + /** A service source replaces the member policy and verifies with its own credential. */ + const sourceBoundary = async (memberPolicy: LiveSearchPolicy) => { + const service = await createLiveServiceSession({ + owner, + userId, + provider, + policy: memberPolicy, + member: client, + mcp, + signal, + pool: input.pool, + }) + if (service) return service + const verifyPolicy = createPolicyVerifier(provider, memberPolicy, client, origin, mcp) + return { + policy: memberPolicy, + partial: false, + scopeSearch: undefined, + verify: (document: Reference) => verifyPolicy(document, document.accessMetadata), + } + } + const boundary = await sourceBoundary(livePolicyFor(input.policies, provider)) + + return { + policy: boundary.policy, + servicePartial: boundary.partial, + async search(search) { + const scoped = boundary.scopeSearch ? boundary.scopeSearch(search) : search + if (scoped === null) return { documents: [] } + if (admin) return admin.search(scoped) + return searchWithinPolicy(provider, client, scoped, (request) => + client ? searchNativeProvider(provider, client, request) : searchCodaMcp(mcp!, request) + ) + }, + async verify(document) { + if (!(await boundary.verify(document))) return false + return !admin || admin.verify(document, document.accessMetadata) + }, + read(reference, filters) { + if (admin) return admin.read(reference) + if (client) return readNativeProvider(provider, client, reference, boundary.policy, filters) + return readCodaMcp(mcp!, reference.id) + }, + async verifyCurrent(document) { + const current = await sourceBoundary( + livePolicyFor(await loadLiveSearchPolicies(owner), provider) + ) + return current.verify(document) + }, + } +} diff --git a/apps/sim/lib/sim-search/live/accounts.ts b/apps/sim/lib/sim-search/live/accounts.ts index a7da24b1901..047480f9f2e 100644 --- a/apps/sim/lib/sim-search/live/accounts.ts +++ b/apps/sim/lib/sim-search/live/accounts.ts @@ -4,6 +4,7 @@ import { and, eq, inArray, isNull, ne } from 'drizzle-orm' import { liveSearchProviderSchema } from '@/lib/api/contracts/mothership-assistant-tools' import { type ResourceOwner, + type ResourceScope, resourceScopeFields, resourceScopeFromOwner, } from '@/lib/core/resource-scope' @@ -27,74 +28,84 @@ import { } from '@/lib/sim-search/live/provider-catalog' import type { LiveAccount } from '@/lib/sim-search/live/types' +async function listOAuthRows(scope: ResourceScope, userId: string) { + if (scope.kind === 'workspace') return getPersonalOAuthCredentials(scope.workspaceId, userId) + const [managed, personal] = await Promise.all([ + getOwnOrganizationManagedOAuthCredentials({ organizationId: scope.organizationId, userId }), + db + .select({ + id: credential.id, + providerId: credential.providerId, + displayName: credential.displayName, + }) + .from(credential) + .innerJoin(account, eq(account.id, credential.accountId)) + .where( + and( + resourceScopeCondition(credential, scope), + eq(credential.type, 'oauth'), + eq(account.userId, userId), + eq(account.providerId, credential.providerId), + ne(credential.providerId, 'slack') + ) + ), + ]) + return [ + ...managed.map((row) => ({ ...row, type: 'managed_oauth' as const })), + ...personal.flatMap((row) => + row.providerId ? [{ ...row, providerId: row.providerId, type: 'oauth' as const }] : [] + ), + ] +} + +async function listCodaTokenRows(scope: ResourceScope, userId: string) { + const rows = await db + .select({ + id: credential.id, + providerId: credential.providerId, + displayName: credential.displayName, + }) + .from(credential) + .where( + and( + resourceScopeCondition(credential, scope), + eq(credential.type, 'service_account'), + eq(credential.providerId, 'coda-service-account'), + eq(credential.createdBy, userId), + isNull(credential.revokedAt) + ) + ) + return rows.map((row) => ({ + ...row, + providerId: 'coda-service-account', + type: 'service_account' as const, + })) +} + /** Personal provider accounts and ACL-gated administrator-managed GitLab sources. */ export async function listLiveAccounts( owner: ResourceOwner, userId: string ): Promise { const scope = resourceScopeFromOwner(owner) - const oauth = - scope.kind === 'workspace' - ? await getPersonalOAuthCredentials(scope.workspaceId, userId) - : [ - ...( - await getOwnOrganizationManagedOAuthCredentials({ - organizationId: scope.organizationId, - userId, - }) - ).map((row) => ({ ...row, type: 'managed_oauth' as const })), - ...( - await db - .select({ - id: credential.id, - providerId: credential.providerId, - displayName: credential.displayName, - }) - .from(credential) - .innerJoin(account, eq(account.id, credential.accountId)) - .where( - and( - resourceScopeCondition(credential, scope), - eq(credential.type, 'oauth'), - eq(account.userId, userId), - eq(account.providerId, credential.providerId), - ne(credential.providerId, 'slack') - ) - ) - ).flatMap((row) => - row.providerId ? [{ ...row, providerId: row.providerId, type: 'oauth' as const }] : [] - ), - ] - const workspaceContext = + /** A workspace's organization is known only after its context loads; an organization's is not. */ + const loadApprovals = (organizationId?: string | null) => + organizationId ? listOrganizationSearchApprovals(organizationId) : null + const [oauth, coda, workspaceContext, organizationApprovals] = await Promise.all([ + listOAuthRows(scope, userId), + listCodaTokenRows(scope, userId), scope.kind === 'workspace' - ? await resolveKnowledgeWorkspaceContext({ workspaceId: scope.workspaceId }) - : undefined - const organizationId = - scope.kind === 'organization' ? scope.organizationId : workspaceContext?.workspaceOrganizationId - const approvals = organizationId ? await listOrganizationSearchApprovals(organizationId) : null + ? resolveKnowledgeWorkspaceContext({ workspaceId: scope.workspaceId }) + : undefined, + scope.kind === 'organization' ? loadApprovals(scope.organizationId) : null, + ]) + const approvals = + organizationApprovals ?? (await loadApprovals(workspaceContext?.workspaceOrganizationId)) const denied = new Set( approvals ? liveSearchProviderSchema.options.filter((provider) => approvals.get(provider) !== true) : [] ) - const coda = ( - await db - .select({ - id: credential.id, - providerId: credential.providerId, - displayName: credential.displayName, - }) - .from(credential) - .where( - and( - resourceScopeCondition(credential, scope), - eq(credential.type, 'service_account'), - eq(credential.providerId, 'coda-service-account'), - eq(credential.createdBy, userId), - isNull(credential.revokedAt) - ) - ) - ).map((row) => ({ ...row, providerId: 'coda-service-account', type: 'service_account' as const })) const candidates = [...oauth, ...coda].flatMap((row) => { const provider = liveSearchProviderForCredential(row.providerId) return provider && supportsLiveSearchMode(provider, 'member') && !denied.has(provider) @@ -110,11 +121,11 @@ export async function listLiveAccounts( ] : [] }) - const visible = workspaceContext - ? await filterWorkspaceAccountCredentials(workspaceContext, candidates) - : candidates - const mcp = denied.has('coda') ? [] : await listCodaMcpSearchAccounts(owner, userId) - const admin = denied.has('gitlab') ? [] : await listAdminGitLabAccounts(owner) + const [visible, mcp, admin] = await Promise.all([ + workspaceContext ? filterWorkspaceAccountCredentials(workspaceContext, candidates) : candidates, + denied.has('coda') ? [] : listCodaMcpSearchAccounts(owner, userId), + denied.has('gitlab') ? [] : listAdminGitLabAccounts(owner), + ]) if (visible.length === 0) return [...mcp, ...admin] // Metadata is fetched in one batch; a fresh binding check still precedes token resolution. const rows = await db @@ -152,10 +163,25 @@ export async function listLiveAccounts( ] } +/** Re-lists the member's accounts so a reference from an earlier request is checked afresh. */ export async function resolveLiveAccount(owner: ResourceOwner, userId: string, accountId: string) { const current = (await listLiveAccounts(owner, userId)).find((row) => row.id === accountId) if (!current) throw new NativeSearchError('reconnect', 'This search connection is no longer available.') + return resolveListedLiveAccount(owner, userId, current) +} + +export type ResolvedLiveAccount = Awaited> + +/** + * Resolves credentials for an account {@link listLiveAccounts} returned earlier in the same + * request, which already performed the binding and revocation checks. + */ +export async function resolveListedLiveAccount( + owner: ResourceOwner, + userId: string, + current: LiveAccount +) { if (current.type === 'admin_source') return resolveAdminGitLabAccount(owner, current) if (current.type === 'managed_mcp') return { account: current, accessToken: '', mcp: true } const scope = resourceScopeFromOwner(owner) diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index d5c1da01caf..cc51ac8c910 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -49,6 +49,10 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ vi.mock('@/lib/sim-search/live/accounts', () => ({ listLiveAccounts: mocks.accounts, resolveLiveAccount: mocks.resolveAccount, + resolveListedLiveAccount: async (owner: unknown, userId: string, listed: { id: string }) => ({ + ...(await mocks.resolveAccount(owner, userId, listed.id)), + account: listed, + }), })) vi.mock('@/lib/sim-search/live/providers', () => ({ NATIVE_SEARCH_GUIDANCE: 'Live coverage', @@ -233,6 +237,176 @@ describe('authorized live retrieval', () => { expect.objectContaining({ filters }) ) }) + it('reports more matches as a cursor, not as degraded coverage', async () => { + mocks.search.mockResolvedValue({ documents: [document], nextCursor: 'next', hasMore: true }) + const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 1 } }) + expect(result.retrieval.status).toBe('complete') + expect(result.live?.accounts[0]).toMatchObject({ status: 'ok', nextCursor: 'next' }) + }) + it('keeps more matches partial when a date order must cover them', async () => { + mocks.search.mockResolvedValue({ documents: [document], nextCursor: 'next' }) + const result = await searchLiveKnowledge.execute({ + principal, + input: { ...input, filters: { sortBy: 'newest' } }, + }) + expect(result.retrieval.status).toBe('partial') + }) + it('reports candidates that could not be verified and keeps the verified ones', async () => { + mocks.search.mockResolvedValue({ + documents: [document, { ...document, id: 'other', url: 'https://docs.google.com/other' }], + }) + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async ({ id }: { id: string }) => { + if (id === 'other') throw new NativeSearchError('unavailable', 'Budget') + return true + }, + }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual(['doc']) + expect(result.live?.accounts[0]).toMatchObject({ + status: 'partial', + message: expect.stringContaining('could not be verified'), + }) + }) + it('fails the account when a grant is revoked during verification', async () => { + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async () => { + throw new NativeSearchError('reconnect', 'Revoked') + }, + }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results).toEqual([]) + expect(result.live?.accounts[0]).toMatchObject({ status: 'reconnect', message: 'Revoked' }) + }) + it('verifies GitLab candidates with the evidence from their own search response', async () => { + const gitlab = { + ...account, + id: 'gitlab-source:source', + provider: 'gitlab', + type: 'admin_source', + } + const evidence = { confidential: false, authorId: 7, assigneeIds: [] } + mocks.accounts.mockResolvedValue([gitlab]) + mocks.resolveAccount.mockResolvedValue({ + account: gitlab, + accessToken: 'admin-secret', + origin: 'https://gitlab.company.com', + adminSource: { id: 'source', config: { project: '42' } }, + }) + mocks.adminSearch.mockResolvedValue({ + documents: [ + { ...document, id: '5', container: '42', kind: 'issues', accessMetadata: evidence }, + ], + }) + await searchLiveKnowledge.execute({ principal, input }) + expect(mocks.adminVerify).toHaveBeenCalledWith(expect.objectContaining({ id: '5' }), evidence) + }) + it('merges equal ranks in a stable account order', async () => { + const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' } + mocks.accounts.mockResolvedValue([gmail, account]) + mocks.search.mockImplementation(async (provider: string) => ({ + documents: [{ ...document, id: provider, url: `https://docs.google.com/${provider}` }], + })) + const result = await searchLiveKnowledge.execute({ principal, input }) + mocks.accounts.mockResolvedValue([account, gmail]) + const reversed = await searchLiveKnowledge.execute({ principal, input }) + const order = (data: typeof result) => data.results.map((row) => row.connectorType) + expect(order(reversed)).toEqual(order(result)) + }) + it('scales the read window with the requested chunk limit', async () => { + mocks.read.mockResolvedValue({ ...document, content: 'x'.repeat(30_000) }) + const search = await searchLiveKnowledge.execute({ principal, input }) + const read = (limit: number) => + readLiveDocument.execute({ + principal, + input: { + workspaceId: 'workspace', + documentId: search.results[0]!.documentId, + limit, + resultSecretRegistry: new ResolvedSecretTraceRegistry([]), + }, + }) + expect((await read(3)).chunks[0]?.content).toHaveLength(8000) + expect((await read(8)).chunks[0]?.content).toHaveLength(24_000) + }) + it('centers the preview on the query match', async () => { + mocks.search.mockResolvedValue({ + documents: [{ ...document, content: `${'filler '.repeat(500)}launch checklist` }], + }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results[0]?.content).toContain('launch checklist') + }) + it('keeps the first verified copy of an item a provider returned twice', async () => { + mocks.search.mockResolvedValue({ + documents: [ + { + ...document, + id: 'primary-copy', + url: 'https://calendar.google.com/a', + dedupeKey: 'meeting', + }, + { + ...document, + id: 'team-copy', + url: 'https://calendar.google.com/b', + dedupeKey: 'meeting', + }, + ], + }) + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async ({ id }: { id: string }) => id === 'team-copy', + }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual([ + 'team-copy', + ]) + }) + it('reports undated exclusions only for documents the member may read', async () => { + mocks.search.mockResolvedValue({ + documents: [{ ...document, id: 'hidden', modifiedAt: undefined }], + }) + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async () => false, + }) + const result = await searchLiveKnowledge.execute({ + principal, + input: { ...input, filters: { modifiedAfter: '2026-01-01T00:00:00Z' } }, + }) + expect(result.live?.accounts[0]).toMatchObject({ status: 'ok' }) + expect(result.live?.accounts[0]?.message).toBeUndefined() + }) + it('drops the cursor of an account whose results were cut from the merge', async () => { + const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' } + mocks.accounts.mockResolvedValue([account, gmail]) + mocks.search.mockImplementation(async (provider: string) => ({ + documents: [1, 2].map((rank) => ({ + ...document, + id: `${provider}-${rank}`, + url: `https://docs.google.com/${provider}-${rank}`, + })), + nextCursor: 'next', + })) + const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 2 } }) + expect(result.results).toHaveLength(2) + for (const status of result.live?.accounts ?? []) { + expect(status.nextCursor).toBeUndefined() + expect(status.message).toContain('Search this account alone') + } + }) + it('reports partial coverage when more matches exist but none could be returned', async () => { + mocks.search.mockResolvedValue({ documents: [], hasMore: true }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.retrieval.status).toBe('partial') + expect(result.live?.accounts[0]?.message).toContain('More matches exist') + }) it('rejects invalid dates before resolving provider credentials', async () => { await expect( searchLiveKnowledge.execute({ diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 6a3759084f1..692a8faf887 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -1,4 +1,5 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { compareStrings } from '@sim/utils/string' import { z } from 'zod' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' import type { @@ -19,37 +20,37 @@ import { resourceScopeFromOwner, resourceScopeKey, } from '@/lib/core/resource-scope' +import { createPinnedConnectionPool } from '@/lib/core/security/input-validation.server' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { isKnowledgeSourceUrl } from '@/lib/knowledge/search/citation' -import { listLiveAccounts, resolveLiveAccount } from '@/lib/sim-search/live/accounts' -import { createCodaMcpClient, readCodaMcp, searchCodaMcp } from '@/lib/sim-search/live/coda-mcp' +import { measureSearchStage } from '@/lib/knowledge/search/diagnostics' +import { matchPassage } from '@/lib/knowledge/search/snippet' import { + type LiveAccountSession, + openLiveAccountSession, +} from '@/lib/sim-search/live/account-session' +import { + listLiveAccounts, + resolveListedLiveAccount, + resolveLiveAccount, +} from '@/lib/sim-search/live/accounts' +import { + dateSortDirection, hasDateBounds, matchesSourceDates, sourceDate, sourceDateType, } from '@/lib/sim-search/live/dates' -import { createAdminGitLabSession } from '@/lib/sim-search/live/gitlab-admin' -import { createNativeClient, NativeSearchError } from '@/lib/sim-search/live/http' -import { createPolicyVerifier } from '@/lib/sim-search/live/policy' -import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' -import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog' -import { - NATIVE_SEARCH_GUIDANCE, - readNativeProvider, - searchNativeProvider, -} from '@/lib/sim-search/live/providers' -import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search' -import { createLiveServiceSession } from '@/lib/sim-search/live/service-session' -import type { - LiveAccount, - NativeDocument, - NativePage, - NativeSearchInput, -} from '@/lib/sim-search/live/types' +import { NativeSearchError } from '@/lib/sim-search/live/http' +import { joinMessages } from '@/lib/sim-search/live/pages' +import { loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' +import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog' +import { NATIVE_SEARCH_GUIDANCE } from '@/lib/sim-search/live/providers' +import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -137,27 +138,42 @@ export function matchesLiveFilters( } return true } -function resultFor( +/** A provider result with the reference that binds it to this member, scope, and account. */ +interface LiveCandidate { + document: NativeDocument + documentId: string +} + +function candidateFor( document: NativeDocument, account: LiveAccount, owner: ResourceOwner, - userId: string, + userId: string +): LiveCandidate { + return { + document, + documentId: encodeLiveReference({ + v: 1, + user: userId, + scope: resourceScopeKey(resourceScopeFromOwner(owner)), + account: account.id, + provider: account.provider, + id: document.id, + ...(document.container ? { container: document.container } : {}), + ...(document.kind ? { kind: document.kind } : {}), + ...(document.revision ? { revision: document.revision } : {}), + ...(document.threadId ? { threadId: document.threadId } : {}), + }), + } +} + +function resultFor( + { document, documentId }: LiveCandidate, + account: LiveAccount, rank: number, + previewQuery: string, registry?: ResolvedSecretTraceRegistry ): WorkspaceKnowledgeSearchResult { - const documentId = encodeLiveReference({ - v: 1, - user: userId, - scope: resourceScopeKey(resourceScopeFromOwner(owner)), - account: account.id, - provider: account.provider, - id: document.id, - ...(document.container ? { container: document.container } : {}), - ...(document.kind ? { kind: document.kind } : {}), - ...(document.revision ? { revision: document.revision } : {}), - ...(document.threadId ? { threadId: document.threadId } : {}), - }) - // The shared UI wire shape retains its index fields; live results never use them as identifiers. return { documentId, knowledgeBaseId: '', @@ -178,12 +194,83 @@ function resultFor( sourceDate: sourceDate(document, account.provider) ?? null, sourceDateType: sourceDateType(account.provider, document), author: document.author ? safeContent(document.author, registry) : null, - content: safeContent(document.content, registry).slice(0, 1800), + content: matchPassage(safeContent(document.content, registry), previewQuery, PREVIEW_CHARACTERS) + .content, chunkIndex: 0, similarity: 1 / (60 + rank), } } +/** Accounts searched at once; bounds token refresh and provider fan-out in large organizations. */ +const ACCOUNT_CONCURRENCY = 4 +/** Candidates verified at once; each verification is one or more provider requests. */ +const VERIFY_CONCURRENCY = 5 +const MAX_ACCOUNTS = 20 +/** Previews center on the query's longest matching term, like indexed passages. */ +const PREVIEW_CHARACTERS = 1800 +/** + * Characters a read returns per three requested chunks. `limit` counts index chunks elsewhere, + * so the default of three keeps one window and the maximum of eight returns three. + */ +const READ_WINDOW_CHARACTERS = 8000 + +/** + * Verifies candidates in rank order. A revoked grant fails the whole account; any other + * verification failure only omits that candidate, and is reported as incomplete coverage. + */ +async function verifyCandidates(session: LiveAccountSession, candidates: LiveCandidate[]) { + const outcomes = await mapWithConcurrency( + candidates, + VERIFY_CONCURRENCY, + async ({ document }) => { + try { + return (await session.verify(document)) ? ('permitted' as const) : ('denied' as const) + } catch (error) { + if (error instanceof NativeSearchError && error.status === 'reconnect') throw error + return 'unverified' as const + } + } + ) + return { + permitted: candidates.filter((_, index) => outcomes[index] === 'permitted'), + unverified: outcomes.includes('unverified'), + } +} + +/** Keeps the first of candidates a provider marked as the same item, such as a shared meeting. */ +function firstOfEachDocument(candidates: LiveCandidate[]): LiveCandidate[] { + const seen = new Set() + return candidates.filter(({ document }) => { + if (!document.dedupeKey) return true + if (seen.has(document.dedupeKey)) return false + seen.add(document.dedupeKey) + return true + }) +} + +/** Whether a result lacks the date metadata the requested date filters need. */ +function lacksFilterDate( + document: NativeDocument, + provider: string, + filters?: WorkspaceSearchFilters +): boolean { + return ( + (Boolean(filters?.startDate || filters?.endDate) && !sourceDate(document, provider)) || + (Boolean(filters?.modifiedAfter || filters?.modifiedBefore) && + !Number.isFinite(Date.parse(document.modifiedAt ?? ''))) + ) +} + +/** Stable account order so equal-rank results from different accounts always merge the same way. */ +function compareAccounts(left: LiveAccount, right: LiveAccount): number { + return ( + LIVE_SEARCH_PROVIDER_IDS.indexOf(left.provider) - + LIVE_SEARCH_PROVIDER_IDS.indexOf(right.provider) || + compareStrings(left.displayName, right.displayName) || + compareStrings(left.id, right.id) + ) +} + export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.search, resolveContext: ({ input }: { input: LiveSearchInput }) => resolveKnowledgeOwnerContext(input), @@ -202,221 +289,208 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ throw new OrchestrationError('validation', 'Invalid live search query or result limit') if (input.filters) input = { ...input, filters: workspaceSearchFiltersSchema.parse(input.filters) } + const filters = input.filters if ( - input.filters?.startDate && - input.filters.endDate && - Date.parse(input.filters.startDate) >= Date.parse(input.filters.endDate) + filters?.startDate && + filters.endDate && + Date.parse(filters.startDate) >= Date.parse(filters.endDate) ) throw new OrchestrationError('validation', 'endDate must be after startDate') const queries = input.nativeQueries ? nativeSearchQueriesSchema.parse(input.nativeQueries) : undefined - if (queries?.some((query) => !query.query) && !hasDateBounds(input.filters)) + if (queries?.some((query) => !query.query) && !hasDateBounds(filters)) throw new OrchestrationError('validation', 'Empty native queries require a date bound') const searchSignal = input.signal ? AbortSignal.any([input.signal, AbortSignal.timeout(20_000)]) : AbortSignal.timeout(20_000) - const policies = await loadLiveSearchPolicies(input) - const allAccounts = await listLiveAccounts(input, userId) - const eligible = allAccounts.filter( - (account) => - (!input.filters?.source || input.filters.source === account.provider) && - (!queries || - queries.some( - (query) => - query.provider === account.provider && - (!query.accountId || query.accountId === account.id) - )) - ) - const selected = eligible.slice(0, 20) - const accounts: LiveSearchAccountStatus[] = [] - const results: WorkspaceKnowledgeSearchResult[] = [] - // Four accounts at a time bounds token refresh and API fanout across large organizations. - for (let offset = 0; offset < selected.length; offset += 4) { - const batch = await Promise.all( - selected.slice(offset, offset + 4).map(async (account) => { - const status = { - accountId: account.id, - provider: account.provider, - displayName: account.displayName, - } - const signal = AbortSignal.any([searchSignal, AbortSignal.timeout(12_000)]) - try { - signal.throwIfAborted() - const resolved = await resolveLiveAccount(input, userId, account.id) - const client = - resolved.account.type === 'managed_mcp' - ? null - : createNativeClient({ - origin: - 'origin' in resolved - ? resolved.origin - : LIVE_SEARCH_PROVIDER_CATALOG[account.provider].origin, - accessToken: resolved.accessToken, - signal, - }) - const native = queries?.find( - (query) => - query.provider === account.provider && - (!query.accountId || query.accountId === account.id) - ) - let policy = livePolicyFor(policies, account.provider) - const admin = - 'adminSource' in resolved && client - ? await createAdminGitLabSession({ - owner: input, - userId, - source: resolved.adminSource, - token: resolved.accessToken, - client, - signal, - }) - : undefined - const mcp = client - ? undefined - : await createCodaMcpClient(input, userId, account.id, signal) - const service = await createLiveServiceSession({ - owner: input, - userId, - provider: account.provider, - policy, - member: client, - mcp, - signal, - }) - if (service) policy = service.policy - const verify = - service?.verify ?? - createPolicyVerifier( - account.provider, - policy, - client, - 'origin' in resolved - ? resolved.origin - : LIVE_SEARCH_PROVIDER_CATALOG[account.provider].origin, - mcp - ) - const searchInput: NativeSearchInput = { - filters: input.filters, - policy, - query: input.query, - native, - limit: input.topK, - scopes: resolved.account.scopes, - } - const scopedInput = service?.scopeSearch - ? service.scopeSearch(searchInput) - : searchInput - const page: NativePage = - scopedInput === null - ? { documents: [] } - : admin - ? await admin.search(scopedInput) - : await searchWithinPolicy(account.provider, client, scopedInput, (scoped) => - client - ? searchNativeProvider(account.provider, client, scoped) - : searchCodaMcp(mcp!, scoped) - ) - const permitted: NativeDocument[] = [] - let unverified = false - for (const document of page.documents) { - try { - if ((await verify(document)) && (!admin || (await admin.verify(document)))) - permitted.push(document) - } catch (error) { - if (error instanceof NativeSearchError && error.status === 'reconnect') throw error - unverified = true - } - } - const rows = permitted - .filter((document) => document.id) - .map((document, index) => ({ - document, - result: resultFor( - document, - account, - input, - userId, - index + 1, - input.resultSecretRegistry - ), - })) - const matching = rows - .filter(({ document, result }) => - matchesLiveFilters(document, result.documentId, account.provider, input.filters) - ) - .map(({ result }) => result) - return { - status: { - ...status, - status: - (input.filters?.sortBy && - input.filters.sortBy !== 'relevance' && - rows.some( - ({ document }) => - !Number.isFinite(Date.parse(sourceDate(document, account.provider) ?? '')) - )) || - unverified || - service?.partial || - page.partial || - page.nextCursor || - matching.length < rows.length - ? ('partial' as const) - : ('ok' as const), - message: - [ - page.message, - service?.partial - ? 'Service account verification covered a bounded subset of the configured users. Narrow the source user list for complete coverage; external Drive users can only search files also visible to the source administrator.' - : undefined, - (input.filters?.startDate || input.filters?.endDate) && - rows.some( - ({ document }) => - !Number.isFinite(Date.parse(sourceDate(document, account.provider) ?? '')) - ) - ? 'Some results lacked date metadata and were excluded; date coverage is incomplete.' - : undefined, - input.filters?.sortBy && input.filters.sortBy !== 'relevance' - ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.' - : undefined, - ] - .filter(Boolean) - .join(' ') || undefined, - nextCursor: page.nextCursor, - }, - results: matching, - } - } catch (error) { - input.signal?.throwIfAborted() - const failure = - error instanceof NativeSearchError - ? error - : signal.aborted - ? new NativeSearchError( - 'timeout', - 'Provider search timed out. Narrow the query or try again.' - ) - : new NativeSearchError( - 'unavailable', - 'This account could not be searched. Check the connection and try again.' - ) - return { - status: { - ...status, - status: failure.status, - message: failure.message, - retryAfterSeconds: failure.retryAfterSeconds, - }, - results: [], - } - } - }) + const nativeFor = (account: LiveAccount) => + queries?.find( + (query) => + query.provider === account.provider && + (!query.accountId || query.accountId === account.id) ) - for (const item of batch) { - accounts.push(item.status) - results.push(...item.results) + const [policies, allAccounts] = await Promise.all([ + measureSearchStage('live.policies', () => loadLiveSearchPolicies(input)), + measureSearchStage('live.accounts', () => listLiveAccounts(input, userId)), + ]) + const eligible = allAccounts + .filter( + (account) => + (!filters?.source || filters.source === account.provider) && + (!queries || nativeFor(account)) + ) + .sort(compareAccounts) + const selected = eligible.slice(0, MAX_ACCOUNTS) + const direction = dateSortDirection(filters) + const dateSorted = Boolean(direction) + const pool = createPinnedConnectionPool() + const searchAccount = async ( + account: LiveAccount + ): Promise<{ status: LiveSearchAccountStatus; results: WorkspaceKnowledgeSearchResult[] }> => { + const status = { + accountId: account.id, + provider: account.provider, + displayName: account.displayName, + } + /** Cancels requests still in flight once the account settles, including after a failure. */ + const settled = new AbortController() + const signal = AbortSignal.any([searchSignal, AbortSignal.timeout(12_000), settled.signal]) + try { + signal.throwIfAborted() + const resolved = await measureSearchStage('live.resolve', () => + resolveListedLiveAccount(input, userId, account) + ) + const session = await measureSearchStage('live.session', () => + openLiveAccountSession({ owner: input, userId, resolved, policies, signal, pool }) + ) + const native = nativeFor(account) + const page = await measureSearchStage('live.search', () => + session.search({ + filters, + policy: session.policy, + query: input.query, + native, + limit: input.topK, + scopes: resolved.account.scopes, + }) + ) + const candidates = page.documents + .filter((document) => document.id) + .map((document) => candidateFor(document, account, input, userId)) + /** + * Local filters run first so provider verification is spent only on eligible results. + * Undated results are verified too, so their exclusion is reported only when readable. + */ + const { permitted, unverified } = await measureSearchStage('live.verify', () => + verifyCandidates( + session, + candidates.filter( + ({ document, documentId }) => + matchesLiveFilters(document, documentId, account.provider, filters) || + lacksFilterDate(document, account.provider, filters) + ) + ) + ) + const readable = firstOfEachDocument(permitted) + const matching = readable.filter(({ document, documentId }) => + matchesLiveFilters(document, documentId, account.provider, filters) + ) + const undatedExcluded = readable.some(({ document }) => + lacksFilterDate(document, account.provider, filters) + ) + const undatedUnsorted = + dateSorted && matching.some(({ document }) => !sourceDate(document, account.provider)) + const moreUnsorted = dateSorted && Boolean(page.nextCursor || page.hasMore) + const moreUnreachable = Boolean(page.hasMore && !page.nextCursor && !matching.length) + const degraded = + unverified || + session.servicePartial || + page.partial || + undatedExcluded || + undatedUnsorted || + moreUnsorted || + moreUnreachable + return { + status: { + ...status, + status: degraded ? 'partial' : 'ok', + message: joinMessages([ + page.message, + session.servicePartial + ? 'Service account verification covered a bounded subset of the configured users. Narrow the source user list for complete coverage; external Drive users can only search files also visible to the source administrator.' + : undefined, + unverified + ? 'Some results could not be verified against the source settings and were omitted.' + : undefined, + undatedExcluded + ? 'Some results lacked date metadata and were excluded; date coverage is incomplete.' + : undefined, + dateSorted + ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.' + : undefined, + moreUnreachable + ? 'More matches exist than this search returned. Narrow the query or target one source.' + : undefined, + ]), + nextCursor: page.nextCursor, + }, + results: matching.map((candidate, index) => + resultFor( + candidate, + account, + index + 1, + native?.query || input.query, + input.resultSecretRegistry + ) + ), + } + } catch (error) { + input.signal?.throwIfAborted() + const failure = + error instanceof NativeSearchError + ? error + : signal.aborted + ? new NativeSearchError( + 'timeout', + 'Provider search timed out. Narrow the query or try again.' + ) + : new NativeSearchError( + 'unavailable', + 'This account could not be searched. Check the connection and try again.' + ) + return { + status: { + ...status, + status: failure.status, + message: failure.message, + retryAfterSeconds: failure.retryAfterSeconds, + }, + results: [], + } + } finally { + settled.abort() } } + let searched: Awaited>[] + try { + searched = await mapWithConcurrency(selected, ACCOUNT_CONCURRENCY, searchAccount) + } finally { + pool.destroy() + } + const seen = new Set() + const ranked = searched + .flatMap(({ results }, account) => results.map((result) => ({ account, result }))) + .sort(({ result: a }, { result: b }) => { + if (!dateSorted) return b.similarity - a.similarity + const left = Date.parse(a.sourceDate ?? '') + const right = Date.parse(b.sourceDate ?? '') + if (!Number.isFinite(left)) return Number.isFinite(right) ? 1 : b.similarity - a.similarity + if (!Number.isFinite(right)) return -1 + return (direction === 'asc' ? left - right : right - left) || b.similarity - a.similarity + }) + .filter(({ result }) => { + const key = result.sourceUrl || result.documentId + if (seen.has(key)) return false + seen.add(key) + return true + }) + /** + * An account's cursor continues after its own page, so it would skip that account's results + * cut from this merge. Those accounts drop the cursor and point to a targeted search instead. + */ + const truncated = new Set(ranked.slice(input.topK).map(({ account }) => account)) + const accounts: LiveSearchAccountStatus[] = searched.map(({ status }, index) => { + if (!truncated.has(index)) return status + const { nextCursor: _, ...rest } = status + return { + ...rest, + message: joinMessages([ + status.message, + 'More matches ranked below the returned results. Search this account alone to see them.', + ]), + } + }) for (const query of queries ?? []) { if ( !selected.some( @@ -433,34 +507,12 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ message: 'No connection with this provider is configured and approved in this scope.', }) } - const seen = new Set() - const ranked = results - .sort((a, b) => { - if (!input.filters?.sortBy || input.filters.sortBy === 'relevance') - return b.similarity - a.similarity - const left = Date.parse(a.sourceDate ?? '') - const right = Date.parse(b.sourceDate ?? '') - if (!Number.isFinite(left)) return Number.isFinite(right) ? 1 : b.similarity - a.similarity - if (!Number.isFinite(right)) return -1 - return ( - (input.filters.sortBy === 'oldest' ? left - right : right - left) || - b.similarity - a.similarity - ) - }) - .filter((item) => { - const key = item.sourceUrl || item.documentId - if (seen.has(key)) return false - seen.add(key) - return true - }) return { query: input.query, - results: ranked.slice(0, input.topK), + results: ranked.slice(0, input.topK).map(({ result }) => result), retrieval: { status: - accounts.some((account) => account.status !== 'ok') || - eligible.length > selected.length || - ranked.length > input.topK + accounts.some((account) => account.status !== 'ok') || eligible.length > selected.length ? 'partial' : 'complete', timedOutLegs: [], @@ -500,95 +552,40 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ (input.filters?.documentIds && !input.filters.documentIds.includes(input.documentId)) ) throw new OrchestrationError('not_found', 'Document is outside the selected search filters') - const resolved = await resolveLiveAccount(input, userId, reference.account) + const [resolved, policies] = await Promise.all([ + resolveLiveAccount(input, userId, reference.account), + loadLiveSearchPolicies(input), + ]) if (resolved.account.provider !== reference.provider) throw new OrchestrationError('not_found', 'Document account changed') const signal = input.signal ? AbortSignal.any([input.signal, AbortSignal.timeout(15_000)]) : AbortSignal.timeout(15_000) - const client = - resolved.account.type === 'managed_mcp' - ? null - : createNativeClient({ - origin: - 'origin' in resolved - ? resolved.origin - : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin, - accessToken: resolved.accessToken, - signal, - }) - let policy = livePolicyFor(await loadLiveSearchPolicies(input), reference.provider) - const admin = - 'adminSource' in resolved && client - ? await createAdminGitLabSession({ - owner: input, - userId, - source: resolved.adminSource, - token: resolved.accessToken, - client, - signal, - }) - : undefined - const mcp = client - ? undefined - : await createCodaMcpClient(input, userId, reference.account, signal) - const service = await createLiveServiceSession({ - owner: input, - userId, - provider: reference.provider, - policy, - member: client, - mcp, - signal, - }) - if (service) policy = service.policy - const verify = - service?.verify ?? - createPolicyVerifier( - reference.provider, - policy, - client, - 'origin' in resolved - ? resolved.origin - : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin, - mcp - ) - if (!(await verify(reference)) || (admin && !(await admin.verify(reference)))) - throw new OrchestrationError( - 'not_found', - 'Document is outside your organization’s search scope' - ) - const document = admin - ? await admin.read(reference) - : client - ? await readNativeProvider(reference.provider, client, reference, policy, input.filters) - : await readCodaMcp(mcp!, reference.id) - const currentPolicy = livePolicyFor(await loadLiveSearchPolicies(input), reference.provider) - const currentService = await createLiveServiceSession({ - owner: input, - userId, - provider: reference.provider, - policy: currentPolicy, - member: client, - mcp, - signal, - }) - const verifyCurrent = - currentService?.verify ?? - createPolicyVerifier( - reference.provider, - currentPolicy, - client, - 'origin' in resolved - ? resolved.origin - : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin, - mcp - ) - if (!(await verifyCurrent(document))) - throw new OrchestrationError( - 'not_found', - 'Document is outside your organization’s search scope' - ) + const pool = createPinnedConnectionPool() + let document: NativeDocument + try { + const session = await openLiveAccountSession({ + owner: input, + userId, + resolved, + policies, + signal, + pool, + }) + if (!(await session.verify(reference))) + throw new OrchestrationError( + 'not_found', + 'Document is outside your organization’s search scope' + ) + document = await measureSearchStage('live.read', () => session.read(reference, input.filters)) + if (!(await session.verifyCurrent(document))) + throw new OrchestrationError( + 'not_found', + 'Document is outside your organization’s search scope' + ) + } finally { + pool.destroy() + } if (!matchesLiveFilters(document, input.documentId, reference.provider, input.filters)) throw new OrchestrationError('not_found', 'Document is outside the selected search filters') const content = safeContent(document.content, input.resultSecretRegistry) @@ -603,7 +600,10 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ 'validation', 'This document changed. Read again from the beginning' ) - const end = Math.min(content.length, start + 8000) + const end = Math.min( + content.length, + start + READ_WINDOW_CHARACTERS * Math.ceil(input.limit / 3) + ) return { documentId: input.documentId, knowledgeBaseId: '', diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index d3a108df0fc..c5a610c6b45 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -1,4 +1,4 @@ -import { nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' +import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' import { array, NativeSearchError, @@ -7,6 +7,8 @@ import { string, textContent, } from '@/lib/sim-search/live/http' +import { interleaveByRank } from '@/lib/sim-search/live/pages' +import { providerText } from '@/lib/sim-search/live/text' import type { NativeClient, NativeDocument, @@ -73,15 +75,16 @@ function page(row: Record, cloudId: string, site: string): Nati title: string(content.title) || string(row.title), url: `${site}/wiki${string(links.webui) || `/pages/${segment(string(content.id))}`}`, content: - string(object(object(content.body).view).value).replace(/<[^>]*>/g, ' ') || - string(row.excerpt).replace(/<[^>]*>/g, ' ') || + providerText(string(object(object(content.body).view).value), 'html') || + providerText(string(row.excerpt).replace(/@@@(?:end)?hl@@@/g, ''), 'html') || string(content.title), modifiedAt: string(version.when) || string(row.lastModified), author: string(object(version.by).displayName), } } +/** The sites a grant can reach, requested once per client and shared with its verifier. */ async function sites(client: NativeClient) { - return array(await client.json('/oauth/token/accessible-resources')) + return array(await client.json('/oauth/token/accessible-resources', { memo: true })) } export async function searchAtlassian( @@ -102,56 +105,52 @@ export async function searchAtlassian( 'reconnect', 'No accessible Atlassian site matches this account or site ID.' ) - const documents: NativeDocument[] = [] - let partial = !input.native?.project && allSites.length > selected.length - let nextCursor: string | undefined - for (const site of selected) { - const cloudId = string(site.id) - const origin = string(site.url).replace(/\/$/, '') - const policyScope = - input.policy?.mode === 'selected' - ? `(${input.policy.included.map((id) => `${provider === 'jira' ? 'project' : 'space'} = ${JSON.stringify(id)}`).join(' OR ')})` - : undefined - const dates = nativeDateBounds(input) - const field = provider === 'jira' ? 'updated' : 'lastmodified' - const scope = - [ - policyScope, - dates.start - ? `${field} >= "${new Date(Date.parse(dates.start) - 86400000).toISOString().slice(0, 10)}"` - : '', - dates.end - ? `${field} <= "${new Date(Date.parse(dates.end) + 86400000).toISOString().slice(0, 10)}"` - : '', - ] - .filter(Boolean) - .join(' AND ') || undefined - const order = - input.filters?.sortBy && input.filters.sortBy !== 'relevance' - ? `${field} ${input.filters.sortBy === 'oldest' ? 'ASC' : 'DESC'}` - : undefined - const text = nativeText(input) - if (provider === 'jira') { - const data = object( - await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { - body: { - jql: scopeAtlassianQuery( - input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''), - scope, - order - ), - maxResults: input.limit, - fields: ['summary', 'description', 'updated', 'creator', 'status'], - ...(input.native?.cursor && selected.length === 1 - ? { nextPageToken: input.native.cursor } - : {}), - }, - }) - ) - documents.push(...array(data.issues).map((row) => issue(row, cloudId, origin))) - partial ||= Boolean(data.nextPageToken) - if (selected.length === 1) nextCursor = string(data.nextPageToken) || undefined - } else { + const single = selected.length === 1 + const policyScope = + input.policy?.mode === 'selected' + ? `(${input.policy.included.map((id) => `${provider === 'jira' ? 'project' : 'space'} = ${JSON.stringify(id)}`).join(' OR ')})` + : undefined + const dates = nativeDateBounds(input) + const field = provider === 'jira' ? 'updated' : 'lastmodified' + const scope = + [ + policyScope, + dates.start + ? `${field} >= "${new Date(Date.parse(dates.start) - 86400000).toISOString().slice(0, 10)}"` + : '', + dates.end + ? `${field} <= "${new Date(Date.parse(dates.end) + 86400000).toISOString().slice(0, 10)}"` + : '', + ] + .filter(Boolean) + .join(' AND ') || undefined + const direction = dateSortDirection(input.filters) + const order = direction ? `${field} ${direction.toUpperCase()}` : undefined + const text = nativeText(input) + const pages = await Promise.all( + selected.map(async (site) => { + const cloudId = string(site.id) + const origin = string(site.url).replace(/\/$/, '') + if (provider === 'jira') { + const data = object( + await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { + body: { + jql: scopeAtlassianQuery( + input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''), + scope, + order + ), + maxResults: input.limit, + fields: ['summary', 'description', 'updated', 'creator', 'status'], + ...(input.native?.cursor && single ? { nextPageToken: input.native.cursor } : {}), + }, + }) + ) + return { + documents: array(data.issues).map((row) => issue(row, cloudId, origin)), + next: string(data.nextPageToken) || undefined, + } + } const data = object( await client.json(`/ex/confluence/${segment(cloudId)}/wiki/rest/api/search`, { query: { @@ -163,24 +162,25 @@ export async function searchAtlassian( ), limit: String(input.limit), expand: 'content.version', - ...(input.native?.cursor && selected.length === 1 - ? { cursor: input.native.cursor } - : {}), + ...(input.native?.cursor && single ? { cursor: input.native.cursor } : {}), }, }) ) - documents.push(...array(data.results).map((row) => page(row, cloudId, origin))) const next = string(object(data._links).next) - partial ||= Boolean(next) - if (next && selected.length === 1) - nextCursor = - new URL(next, 'https://api.atlassian.com').searchParams.get('cursor') ?? undefined - } - } + return { + documents: array(data.results).map((row) => page(row, cloudId, origin)), + next: next + ? (new URL(next, 'https://api.atlassian.com').searchParams.get('cursor') ?? undefined) + : undefined, + } + }) + ) + const documents = interleaveByRank(pages.map((result) => result.documents)) return { documents, - partial, - nextCursor, + partial: !input.native?.project && allSites.length > selected.length, + hasMore: pages.some((result) => Boolean(result.next)), + nextCursor: single ? pages[0]?.next : undefined, message: 'Searches up to four accessible Atlassian sites. For a specific site and pagination, set project to its cloud ID.', } diff --git a/apps/sim/lib/sim-search/live/coda-mcp.test.ts b/apps/sim/lib/sim-search/live/coda-mcp.test.ts index 42bd317954a..26fc2df9add 100644 --- a/apps/sim/lib/sim-search/live/coda-mcp.test.ts +++ b/apps/sim/lib/sim-search/live/coda-mcp.test.ts @@ -85,7 +85,7 @@ describe('Coda MCP content search', () => { }) expect(page).toMatchObject({ nextCursor: 'next', - partial: true, + partial: false, documents: [{ id: 'coda://docs/doc/pages/page', kind: 'mcp', content: 'Release next week' }], }) }) diff --git a/apps/sim/lib/sim-search/live/coda-mcp.ts b/apps/sim/lib/sim-search/live/coda-mcp.ts index bf33e26ed82..085600c80de 100644 --- a/apps/sim/lib/sim-search/live/coda-mcp.ts +++ b/apps/sim/lib/sim-search/live/coda-mcp.ts @@ -44,11 +44,22 @@ export async function createCodaMcpClient( throw new NativeSearchError('reconnect', 'Coda connection changed. Search again.') return current } - const loadProvider = async () => createManagedMcpAuthProvider(await loadCurrent()) + /** + * The first provider load reuses the grant checked moments earlier in the same step; any later + * load, such as a token refresh inside the MCP client, reads the current grant again. + */ + const providerLoader = (checked: typeof initial) => { + let first: typeof initial | undefined = checked + return async () => { + const current = first ?? (await loadCurrent()) + first = undefined + return createManagedMcpAuthProvider(current) + } + } const tools = await mcpService.discoverManagedMcpTools( initial.mcpServerId, initial.scope, - { credentialId, loadProvider }, + { credentialId, loadProvider: providerLoader(initial) }, signal, { requireComplete: true } ) @@ -64,13 +75,13 @@ export async function createCodaMcpClient( `Coda no longer advertises ${name}. Reconnect or update the connector.` ) validateToolArguments(tool, args) - await loadCurrent() + const current = await loadCurrent() const result = await mcpService.executeManagedMcpTool({ connectionId: credentialId, serverId: initial.mcpServerId, scope: initial.scope, toolCall: { name, arguments: args }, - loadAuthProvider: loadProvider, + loadAuthProvider: providerLoader(current), signal, timeoutMs: 10_000, }) @@ -190,11 +201,8 @@ export async function searchCodaMcp( return { documents, nextCursor, - partial: - documents.length < rows.length || - Boolean(nextCursor) || - result.hasMore === true || - hasDateBounds(input.filters), + hasMore: result.hasMore === true, + partial: documents.length < rows.length || hasDateBounds(input.filters), message: (query ? 'Coda content search includes pages and table rows. Narrow the query or target a document for more results.' diff --git a/apps/sim/lib/sim-search/live/dates.test.ts b/apps/sim/lib/sim-search/live/dates.test.ts index 6a5acd34dbe..5599343dc2e 100644 --- a/apps/sim/lib/sim-search/live/dates.test.ts +++ b/apps/sim/lib/sim-search/live/dates.test.ts @@ -243,7 +243,7 @@ describe('generic live search dates', () => { }) expect(api.json).toHaveBeenCalledTimes(1) expect(api.json.mock.calls[0]?.[1].query).toMatchObject({ - q: expect.stringContaining('updated:>=2026-09-22T07:00:00.000Z'), + q: 'repo:team/repo is:pr updated:2026-09-22T07:00:00.000Z..2026-09-23T07:00:00.000Z', sort: 'updated', order: 'asc', }) diff --git a/apps/sim/lib/sim-search/live/dates.ts b/apps/sim/lib/sim-search/live/dates.ts index 992c07e9883..4e2cae914c1 100644 --- a/apps/sim/lib/sim-search/live/dates.ts +++ b/apps/sim/lib/sim-search/live/dates.ts @@ -24,6 +24,12 @@ export function hasDateBounds(filters?: WorkspaceSearchFilters): boolean { ) } +/** The provider-date order results must follow, or undefined when ranked by relevance. */ +export function dateSortDirection(filters?: WorkspaceSearchFilters): 'asc' | 'desc' | undefined { + if (!filters?.sortBy || filters.sortBy === 'relevance') return undefined + return filters.sortBy === 'oldest' ? 'asc' : 'desc' +} + /** Native bounds may be widened for provider precision; returned metadata is checked exactly. */ export function nativeDateBounds(input: NativeSearchInput): { start?: string; end?: string } { const filters = input.filters diff --git a/apps/sim/lib/sim-search/live/github-service.ts b/apps/sim/lib/sim-search/live/github-service.ts index 9b88313c352..7d2f2645e7d 100644 --- a/apps/sim/lib/sim-search/live/github-service.ts +++ b/apps/sim/lib/sim-search/live/github-service.ts @@ -1,4 +1,5 @@ import { decryptSecret } from '@/lib/core/security/encryption' +import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server' import { assertGitHubInstallationRepositoryActive, parseGitHubInstallationBinding, @@ -47,7 +48,8 @@ function codeAllowed(source: LiveGitHubSource, document: Pick() for (const source of sources) { @@ -107,6 +109,7 @@ export function createGitHubServiceVerifier( origin: 'https://api.github.com', accessToken, signal, + pool, }) const [appRepository, memberRepository] = await Promise.all([ app.json(path).then(object), diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts index d57e04c381c..fedbc860a8f 100644 --- a/apps/sim/lib/sim-search/live/github.ts +++ b/apps/sim/lib/sim-search/live/github.ts @@ -1,6 +1,11 @@ -import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' +import { + dateSortDirection, + hasDateBounds, + nativeDateBounds, + nativeText, +} from '@/lib/sim-search/live/dates' import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http' -import { collectNativePages } from '@/lib/sim-search/live/pages' +import { collectNativePages, joinMessages } from '@/lib/sim-search/live/pages' import type { NativeClient, NativeDocument, @@ -41,6 +46,69 @@ function githubDocument(row: Record, kind: string): NativeDocum } } +/** + * Code search rejects a `q` over 1,000 UTF-8 bytes with qualifiers counted, although the + * documentation cites only 256 characters of text. Batches keep a small margin under it. + */ +const GITHUB_CODE_QUERY_BYTES = 980 +/** + * Issue search limits only its free text to 256 characters and accepts roughly 4,000 bytes of + * repository qualifiers; batches leave room for the type and date qualifiers appended later. + */ +const GITHUB_ISSUE_QUERY_BYTES = 3800 +/** Batches per kind; each code batch spends one of GitHub's ten code searches per minute. */ +const GITHUB_MAX_REPOSITORY_BATCHES = 4 + +type GitHubKind = 'issues' | 'code' | 'repositories' +const CODE_EXCLUDED_BY_DATES = 'Code has no dates and is excluded from date-filtered searches.' + +/** Code has no file dates, so a date-bounded search covers issues and pull requests only. */ +function githubKinds(input: NativeSearchInput): GitHubKind[] { + const kind = input.native?.kind + if (kind === 'issues' || kind === 'code' || kind === 'repositories') return [kind] + if (kind) + throw new NativeSearchError( + 'unavailable', + 'GitHub search supports issues, code, or repositories.' + ) + return hasDateBounds(input.filters) ? ['issues'] : ['issues', 'code'] +} + +/** Splits repositories into qualifier batches that keep each query within `maxBytes`. */ +function repositoryBatches(query: string, names: readonly string[], maxBytes: number): string[][] { + const batches: string[][] = [] + let batch: string[] = [] + let bytes = Buffer.byteLength(query) + for (const name of names) { + const term = Buffer.byteLength(` repo:${name}`) + if (batch.length && bytes + term > maxBytes) { + if (batches.push(batch) === GITHUB_MAX_REPOSITORY_BATCHES) return batches + batch = [] + bytes = Buffer.byteLength(query) + } + batch.push(name) + bytes += term + } + if (batch.length) batches.push(batch) + return batches +} + +/** `key:value` or `key:"quoted value"`, excluding URLs such as `https://…`. */ +const GITHUB_QUALIFIER = /^-?[a-z][\w-]*:(?!\/\/)\S/i + +/** + * Groups free text so boolean operators cannot absorb appended qualifiers. GitHub treats a + * qualifier inside parentheses as search text, so qualifiers stay outside the group. A query + * that already uses parentheses is structured by its author and is left as written. + */ +function groupGitHubText(query: string): string { + if (!query || /[()]/.test(query)) return query + const tokens = query.match(/-?[\w-]+:"[^"]*"|-?"[^"]*"|\S+/g) ?? [] + const qualifiers = tokens.filter((token) => GITHUB_QUALIFIER.test(token)) + const text = tokens.filter((token) => !GITHUB_QUALIFIER.test(token)).join(' ') + return [text ? `(${text})` : '', ...qualifiers].filter(Boolean).join(' ') +} + export async function searchGitHub( client: NativeClient, input: NativeSearchInput @@ -64,50 +132,63 @@ export async function searchGitHub( documents: [], message: 'No repositories are accessible through this GitHub connection.', } - const batches: string[][] = [] - for (let offset = 0; offset < names.length; offset += 25) - batches.push(names.slice(offset, offset + 25)) - const pages: Promise[] = [] - for (const names of batches) { - pages.push( - searchGitHub(client, { - ...input, - native: { - provider: 'github', - ...input.native, - query: `${query} ${names.map((name) => `repo:${name}`).join(' ')}`, - }, - }) + const kinds = githubKinds(input) + const batches = kinds.map((kind) => ({ + kind, + repositories: repositoryBatches( + query, + names, + kind === 'issues' ? GITHUB_ISSUE_QUERY_BYTES : GITHUB_CODE_QUERY_BYTES + ), + })) + const searched = Math.min( + ...batches.map(({ repositories }) => + repositories.reduce((count, batch) => count + batch.length, 0) ) - } + ) const result = await collectNativePages( - pages, + batches.flatMap(({ kind, repositories }) => + repositories.map((batch) => + searchGitHub(client, { + ...input, + native: { + provider: 'github', + ...input.native, + kind, + query: [query, ...batch.map((name) => `repo:${name}`)].filter(Boolean).join(' '), + }, + }) + ) + ), 'Searched repositories you own, collaborate on, or access through organization membership. Use a repo: qualifier to narrow results.' ) + const capped = repositories.length === 100 || searched < names.length return { ...result, - partial: result.partial || repositories.length === 100, - message: - repositories.length === 100 - ? `${result.message} Only the 100 most recently pushed repositories were searched; target a repository for broader coverage.` - : result.message, + partial: result.partial || capped, + message: joinMessages([ + result.message, + !input.native?.kind && !kinds.includes('code') ? CODE_EXCLUDED_BY_DATES : undefined, + capped + ? `Only the ${searched} most recently pushed repositories were searched; target a repository for broader coverage.` + : undefined, + ]), } } - if (!input.native?.kind) + if (!input.native?.kind) { + const kinds = githubKinds(input) return collectNativePages( - ['issues', 'code'].map((kind) => + kinds.map((kind) => searchGitHub(client, { ...input, - native: { - provider: 'github', - ...input.native, - query, - kind: kind === 'code' ? 'code' : 'issues', - }, + native: { provider: 'github', ...input.native, query, kind }, }) ), - 'Searched GitHub issues, pull requests, and code.' + kinds.includes('code') + ? 'Searched GitHub issues, pull requests, and code.' + : `Searched GitHub issues and pull requests. ${CODE_EXCLUDED_BY_DATES}` ) + } if ( input.native.kind === 'issues' && !/(?:^|\s)(?:is|type):(?:issue|pr|pull-request)(?:\s|$)/i.test(query) @@ -121,12 +202,7 @@ export async function searchGitHub( ), 'Searched issues and pull requests separately.' ) - const kind = input.native?.kind ?? 'issues' - if (!['issues', 'code', 'repositories'].includes(kind)) - throw new NativeSearchError( - 'unavailable', - 'GitHub search supports issues, code, or repositories.' - ) + const [kind] = githubKinds(input) if (kind === 'code' && hasDateBounds(input.filters)) throw new NativeSearchError( 'unavailable', @@ -134,16 +210,17 @@ export async function searchGitHub( ) const dates = nativeDateBounds(input) const text = nativeText(input) + /** GitHub ORs repeated qualifiers, so both bounds must share one `updated:` range. */ + const updated = + dates.start && dates.end + ? `updated:${dates.start}..${dates.end}` + : dates.start + ? `updated:>=${dates.start}` + : dates.end + ? `updated:<=${dates.end}` + : '' const datedQuery = - kind === 'issues' - ? [ - text ? `(${text})` : '', - dates.start ? `updated:>=${dates.start}` : '', - dates.end ? `updated:<=${dates.end}` : '', - ] - .filter(Boolean) - .join(' ') - : text + kind === 'issues' ? [groupGitHubText(text), updated].filter(Boolean).join(' ') : text const page = input.native?.cursor ?? '1' if (!/^\d{1,3}$/.test(page) || Number(page) < 1) throw new NativeSearchError('unavailable', 'Invalid GitHub page.') @@ -154,8 +231,8 @@ export async function searchGitHub( q: hasDateBounds(input.filters) ? datedQuery : text, per_page: String(input.limit), page, - ...(kind === 'issues' && input.filters?.sortBy && input.filters.sortBy !== 'relevance' - ? { sort: 'updated', order: input.filters.sortBy === 'oldest' ? 'asc' : 'desc' } + ...(kind === 'issues' && dateSortDirection(input.filters) + ? { sort: 'updated', order: dateSortDirection(input.filters) } : {}), }, }) @@ -175,7 +252,8 @@ export async function searchGitHub( return { documents: array(data.items).map((row) => githubDocument(row, kind)), nextCursor, - partial: data.incomplete_results === true || total > 1000, + hasMore: total > 1000, + partial: data.incomplete_results === true, message: kind === 'code' ? 'GitHub REST code search covers the default branch and files below 384 KB; code queries have a separate rate limit. Read results for file contents.' diff --git a/apps/sim/lib/sim-search/live/gitlab.ts b/apps/sim/lib/sim-search/live/gitlab.ts index ce430e41d5a..67523fda03f 100644 --- a/apps/sim/lib/sim-search/live/gitlab.ts +++ b/apps/sim/lib/sim-search/live/gitlab.ts @@ -1,4 +1,9 @@ -import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' +import { + dateSortDirection, + hasDateBounds, + nativeDateBounds, + nativeText, +} from '@/lib/sim-search/live/dates' import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http' import { collectNativePages } from '@/lib/sim-search/live/pages' import type { @@ -13,7 +18,10 @@ function document(row: Record, kind: string): NativeDocument { accessMetadata: { confidential: row.confidential, authorId: object(row.author).id, - assigneeIds: array(row.assignees).map((person) => person.id), + /** Older merge request payloads carry only the single, deprecated `assignee`. */ + assigneeIds: Array.isArray(row.assignees) + ? array(row.assignees).map((person) => person.id) + : [object(row.assignee).id].filter((id) => id !== undefined), state: row.state, labels: row.labels, milestone: row.milestone, @@ -109,8 +117,7 @@ export async function searchGitLab( const listing = Boolean( input.native?.project && ['issues', 'merge_requests'].includes(kind) && - (hasDateBounds(input.filters) || - (input.filters?.sortBy && input.filters.sortBy !== 'relevance')) + (hasDateBounds(input.filters) || Boolean(dateSortDirection(input.filters))) ) let response: unknown try { @@ -127,7 +134,7 @@ export async function searchGitLab( ...(listing && dates.start ? { updated_after: dates.start } : {}), ...(listing && dates.end ? { updated_before: dates.end } : {}), ...(listing - ? { order_by: 'updated_at', sort: input.filters?.sortBy === 'oldest' ? 'asc' : 'desc' } + ? { order_by: 'updated_at', sort: dateSortDirection(input.filters) ?? 'desc' } : {}), per_page: String(input.limit), page, diff --git a/apps/sim/lib/sim-search/live/google-service.test.ts b/apps/sim/lib/sim-search/live/google-service.test.ts index 49f2c14a9a9..46ed322e59c 100644 --- a/apps/sim/lib/sim-search/live/google-service.test.ts +++ b/apps/sim/lib/sim-search/live/google-service.test.ts @@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ConnectorAccessToken } from '@/lib/knowledge/connectors/access-token' import { searchCalendar } from '@/lib/sim-search/live/google' import { createGoogleServiceVerifier } from '@/lib/sim-search/live/google-service' -import { NativeSearchError } from '@/lib/sim-search/live/http' +import { NativeSearchError, withJsonMemo } from '@/lib/sim-search/live/http' import { liveSourcePolicy } from '@/lib/sim-search/live/source-policy' import type { NativeClient, NativeSearchInput } from '@/lib/sim-search/live/types' @@ -41,7 +41,7 @@ beforeEach(() => { vi.setSystemTime(new Date('2026-09-22T12:00:00Z')) member.json.mockResolvedValue({ emailAddress: 'reader@example.com' }) mocks.directory.mockImplementation(async (_token, email: string) => person(email)) - mocks.createClient.mockReturnValue(delegated) + mocks.createClient.mockImplementation(() => withJsonMemo(delegated)) mint.mockResolvedValue('delegated-token') }) afterEach(() => vi.useRealTimers()) @@ -428,4 +428,12 @@ describe('Google service search scope and continuation', () => { native, }) }) + it('caps Drive pages to what source verification can check within its request budget', async () => { + member.json.mockResolvedValue({ user: { emailAddress: 'reader@example.com' } }) + const search = { query: 'launch', limit: 50, scopes: [] } + const scoped = await create('google_drive', { folderId: 'folder' }) + expect(scoped.scopeSearch(search)).toMatchObject({ limit: 20 }) + expect(scoped.scopeSearch({ ...search, limit: 10 })).toMatchObject({ limit: 10 }) + expect((await create('google_drive')).scopeSearch(search)).toMatchObject({ limit: 29 }) + }) }) diff --git a/apps/sim/lib/sim-search/live/google-service.ts b/apps/sim/lib/sim-search/live/google-service.ts index 6c3b09fe90d..67f398d2ace 100644 --- a/apps/sim/lib/sim-search/live/google-service.ts +++ b/apps/sim/lib/sim-search/live/google-service.ts @@ -1,4 +1,5 @@ import { normalizeEmail } from '@sim/utils/string' +import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server' import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' import type { ConnectorAccessToken } from '@/lib/knowledge/connectors/access-token' import { @@ -20,6 +21,8 @@ import { type GoogleProvider = 'google_drive' | 'gmail' | 'google_calendar' type Reference = Pick const SOURCE_REQUEST_BUDGET = 30 +/** Distinct ancestor folders a restricted Drive page may read while verifying its files. */ +const DRIVE_FOLDER_RESERVE = 9 const DAY_MS = 86400000 interface GoogleServiceSession { @@ -43,8 +46,9 @@ export async function createGoogleServiceVerifier(input: { config: Record policy: LiveSearchPolicy signal: AbortSignal + pool?: PinnedConnectionPool }): Promise { - const { provider, token, member, config, policy, signal } = input + const { provider, token, member, config, policy, signal, pool } = input if (!token.getDelegatedAccessToken) throw new NativeSearchError( 'unavailable', @@ -99,7 +103,12 @@ export async function createGoogleServiceVerifier(input: { ) continue const accessToken = await token.getDelegatedAccessToken(person.email, signal) - const client = createNativeClient({ origin: 'https://www.googleapis.com', accessToken, signal }) + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken, + signal, + pool, + }) clients.push({ email: person.email, client, @@ -110,7 +119,12 @@ export async function createGoogleServiceVerifier(input: { if (provider === 'google_drive' && !selected.length && !clients.length) { signal.throwIfAborted() const accessToken = await token.getDelegatedAccessToken(administrator.email, signal) - const client = createNativeClient({ origin: 'https://www.googleapis.com', accessToken, signal }) + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken, + signal, + pool, + }) clients.push({ email: administrator.email, client, @@ -261,6 +275,15 @@ export async function createGoogleServiceVerifier(input: { partial, scopeSearch(search) { if (!clients.length) return null + if (provider === 'google_drive') { + /** Each file costs one read; a restricted source also reads folders shared across files. */ + const folderReserve = + policy.mode === 'selected' || policy.excluded.length ? DRIVE_FOLDER_RESERVE : 0 + return { + ...search, + limit: Math.min(search.limit, SOURCE_REQUEST_BUDGET - 1 - folderReserve), + } + } if (provider === 'gmail') { const requestsPerMessage = string(config.query).trim() ? 2 : 1 return { diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index e2eb88bc5e1..6c97f02c56c 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -1,7 +1,15 @@ +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' -import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' +import { + dateSortDirection, + hasDateBounds, + nativeDateBounds, + nativeText, +} from '@/lib/sim-search/live/dates' import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http' +import { interleaveByRank } from '@/lib/sim-search/live/pages' import { permitsResources } from '@/lib/sim-search/live/policy' +import { providerText } from '@/lib/sim-search/live/text' import type { NativeClient, NativeDocument, @@ -46,8 +54,10 @@ export async function searchDrive( ...(dates.start ? [`modifiedTime >= '${dates.start}'`] : []), ...(dates.end ? [`modifiedTime <= '${dates.end}'`] : []), ].join(' and '), - ...(input.filters?.sortBy && input.filters.sortBy !== 'relevance' - ? { orderBy: `modifiedTime${input.filters.sortBy === 'oldest' ? '' : ' desc'}` } + ...(dateSortDirection(input.filters) + ? { + orderBy: `modifiedTime${dateSortDirection(input.filters) === 'asc' ? '' : ' desc'}`, + } : {}), fields: `nextPageToken,incompleteSearch,files(${DRIVE_FIELDS})`, pageSize: String(input.limit), @@ -117,6 +127,11 @@ export async function readDrive(client: NativeClient, id: string): Promise): NativeDocument { const payload = object(row.payload) const headers = array(payload.headers) @@ -124,10 +139,13 @@ function gmailDocument(row: Record): NativeDocument { string(headers.find((h) => string(h.name).toLowerCase() === name)?.value) const timestamp = Number(row.internalDate) return { + ...(Array.isArray(row.labelIds) + ? { accessMetadata: { id: row.id, labelIds: row.labelIds } } + : {}), id: string(row.id), title: header('subject') || '(No subject)', url: `https://mail.google.com/mail/u/0/#all/${segment(string(row.id))}`, - content: string(row.snippet), + content: providerText(string(row.snippet), 'escaped'), author: header('from'), ...(Number.isFinite(timestamp) && timestamp > 0 ? { modifiedAt: new Date(timestamp).toISOString() } @@ -157,24 +175,22 @@ export async function searchGmail( }, }) ) - const documents: NativeDocument[] = [] - // Bound fanout while retaining provider rank. One metadata call per matching message. - const messages = array(data.messages) - for (let offset = 0; offset < messages.length; offset += 5) { - documents.push( - ...(await Promise.all( - messages.slice(offset, offset + 5).map(async (message) => - gmailDocument( - object( - await client.json(`/gmail/v1/users/me/messages/${segment(string(message.id))}`, { - query: { format: 'metadata' }, - }) - ) - ) + const documents = await mapWithConcurrency( + array(data.messages), + GMAIL_METADATA_CONCURRENCY, + async (message) => + gmailDocument( + object( + await client.json(`/gmail/v1/users/me/messages/${segment(string(message.id))}`, { + query: { + format: 'metadata', + metadataHeaders: ['Subject', 'From'], + fields: GMAIL_METADATA_FIELDS, + }, + }) ) - )) - ) - } + ) + ) return { documents, nextCursor: string(data.nextPageToken) || undefined, @@ -188,13 +204,11 @@ function mailText(value: unknown): string { const children = array(part.parts).map(mailText).filter(Boolean) const encoded = string(object(part.body).data) if (string(part.mimeType) === 'text/plain' && encoded) - return Buffer.from(encoded, 'base64url').toString('utf8') + return providerText(Buffer.from(encoded, 'base64url').toString('utf8')) if (children.length) return children.join('\n') // HTML-only messages remain text, never rendered as markup. if (string(part.mimeType) === 'text/html' && encoded) - return Buffer.from(encoded, 'base64url') - .toString('utf8') - .replace(/<[^>]*>/g, ' ') + return providerText(Buffer.from(encoded, 'base64url').toString('utf8'), 'html') return '' } @@ -226,7 +240,7 @@ function eventDocument( url: string(row.htmlLink), content: [ string(row.summary), - string(row.description), + providerText(string(row.description), 'auto'), string(row.location), `Start: ${string(object(row.start).dateTime) || string(object(row.start).date)}`, `End: ${string(object(row.end).dateTime) || string(object(row.end).date)}`, @@ -242,6 +256,27 @@ function eventDocument( } } +/** + * A shared meeting appears once in every attendee calendar the member can see. Its iCalendar + * UID and start instant identify the occurrence across calendars, which may report the start in + * their own time zones; a modified recurring instance stays distinct from its series. + */ +function calendarOccurrenceKey(row: Record): string { + const start = object(row.originalStartTime ?? row.start) + const instant = Date.parse(string(start.dateTime)) + return JSON.stringify([ + string(row.iCalUID) || string(row.id), + row.recurringEventId ? 'instance' : 'event', + Number.isFinite(instant) ? instant : string(start.date), + ]) +} + +/** Events without a resolvable start sort after every scheduled one. */ +function eventStartTime(document: NativeDocument): number { + const time = Date.parse(document.eventStartAt ?? '') + return Number.isFinite(time) ? time : Number.MAX_SAFE_INTEGER +} + export async function searchCalendar( client: NativeClient, input: NativeSearchInput @@ -262,76 +297,68 @@ export async function searchCalendar( ]) ) .slice(0, 20) - const documents: NativeDocument[] = [] - let partial = Boolean(calendars.nextPageToken) - let nextCursor: string | undefined - let failedCalendars = 0 - for (let offset = 0; offset < rows.length; offset += 4) { - const pages = await Promise.allSettled( - rows.slice(offset, offset + 4).map(async (row) => { - const calendarId = string(row.id) - const data = object( - await client.json(`/calendar/v3/calendars/${segment(calendarId)}/events`, { - query: { - ...(nativeText(input) ? { q: nativeText(input) } : {}), - ...(input.filters?.startDate - ? { - timeMin: new Date( - Math.floor(Date.parse(input.filters.startDate) / 1000) * 1000 - 1000 - ).toISOString(), - } - : {}), - ...(input.filters?.endDate - ? { - timeMax: new Date( - Math.ceil(Date.parse(input.filters.endDate) / 1000) * 1000 - ).toISOString(), - } - : {}), - ...(input.filters?.modifiedAfter ? { updatedMin: input.filters.modifiedAfter } : {}), - ...(hasDateBounds(input.filters) || - (input.filters?.sortBy && input.filters.sortBy !== 'relevance') - ? { singleEvents: 'true', orderBy: 'startTime' } - : {}), - maxResults: String(input.limit), - showDeleted: 'false', - ...(input.native?.cursor && input.native.project - ? { pageToken: input.native.cursor } - : {}), - }, - }) - ) - return { data, calendarId, timeZone: string(data.timeZone) || string(row.timeZone) } - }) - ) - for (const result of pages) { - if (result.status === 'rejected') { - if (input.native?.project) throw result.reason - partial = true - failedCalendars++ - continue - } - const { data, calendarId, timeZone } = result.value - documents.push( - ...array(data.items) - .filter((event) => event.status !== 'cancelled') - .map((event) => - eventDocument(event, calendarId, input.policy?.includeAttendees, timeZone) - ) + .sort((left, right) => Number(right.primary === true) - Number(left.primary === true)) + const chronological = hasDateBounds(input.filters) || Boolean(dateSortDirection(input.filters)) + const pages = await mapWithConcurrency(rows, 4, async (row) => { + const calendarId = string(row.id) + try { + const data = object( + await client.json(`/calendar/v3/calendars/${segment(calendarId)}/events`, { + query: { + ...(nativeText(input) ? { q: nativeText(input) } : {}), + ...(input.filters?.startDate + ? { + timeMin: new Date( + Math.floor(Date.parse(input.filters.startDate) / 1000) * 1000 - 1000 + ).toISOString(), + } + : {}), + ...(input.filters?.endDate + ? { + timeMax: new Date( + Math.ceil(Date.parse(input.filters.endDate) / 1000) * 1000 + ).toISOString(), + } + : {}), + ...(input.filters?.modifiedAfter ? { updatedMin: input.filters.modifiedAfter } : {}), + ...(chronological ? { singleEvents: 'true', orderBy: 'startTime' } : {}), + maxResults: String(input.limit), + showDeleted: 'false', + ...(input.native?.cursor && input.native.project + ? { pageToken: input.native.cursor } + : {}), + }, + }) ) - partial ||= Boolean(data.nextPageToken) - if (input.native?.project) nextCursor = string(data.nextPageToken) || undefined + return { data, calendarId, timeZone: string(data.timeZone) || string(row.timeZone) } + } catch (error) { + if (input.native?.project) throw error + return null } - } - if (rows.length > 0 && failedCalendars === rows.length) + }) + const searched = pages.filter((page) => page !== null) + if (rows.length > 0 && searched.length === 0) throw new NativeSearchError( 'unavailable', 'None of the calendars could be searched. Check calendar permissions.' ) + const perCalendar = searched.map(({ data, calendarId, timeZone }) => + array(data.items) + .filter((event) => event.status !== 'cancelled') + .map((event) => ({ + ...eventDocument(event, calendarId, input.policy?.includeAttendees, timeZone), + dedupeKey: calendarOccurrenceKey(event), + })) + ) + const documents = chronological + ? perCalendar.flat().sort((left, right) => eventStartTime(left) - eventStartTime(right)) + : interleaveByRank(perCalendar) + const single = input.native?.project ? searched[0] : undefined return { documents, - partial, - nextCursor, + partial: Boolean(calendars.nextPageToken) || searched.length < rows.length, + hasMore: searched.some(({ data }) => Boolean(data.nextPageToken)), + nextCursor: single ? string(single.data.nextPageToken) || undefined : undefined, message: 'Calendar supports text and date-only searches across up to 20 calendars. startDate/endDate filter scheduled starts; recurring events expand within the window. sourceDate is scheduled start and sourceModifiedAt is last edit. Use project = calendar ID to paginate; reverse ordering is limited to the fetched page.', } diff --git a/apps/sim/lib/sim-search/live/http.test.ts b/apps/sim/lib/sim-search/live/http.test.ts index 61ba4c935bf..bde4d4e0645 100644 --- a/apps/sim/lib/sim-search/live/http.test.ts +++ b/apps/sim/lib/sim-search/live/http.test.ts @@ -95,6 +95,52 @@ describe('native search network boundary', () => { message: expect.stringContaining('read permissions for Contents, Issues, and Pull requests'), }) }) + it('reports a Google quota 403 as a rate limit rather than a reconnect', async () => { + mocks.fetch.mockResolvedValue( + new Response( + JSON.stringify({ + error: { code: 403, errors: [{ reason: 'userRateLimitExceeded' }], message: 'private' }, + }), + { status: 403 } + ) + ) + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken: 'private', + signal: new AbortController().signal, + }) + await expect(client.json('/drive/v3/files')).rejects.toMatchObject({ + status: 'rate_limited', + message: 'Provider rate limit reached. Try again later.', + }) + }) + it('still asks for a reconnect when Google denies access', async () => { + mocks.fetch.mockResolvedValue( + new Response(JSON.stringify({ error: { errors: [{ reason: 'insufficientPermissions' }] } }), { + status: 403, + }) + ) + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken: 'private', + signal: new AbortController().signal, + }) + await expect(client.json('/drive/v3/files')).rejects.toMatchObject({ status: 'reconnect' }) + }) + it('requests compressed bodies and passes the caller connection pool through', async () => { + const pool = { agent: vi.fn(), destroy: vi.fn() } + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken: 'private', + signal: new AbortController().signal, + pool, + }) + await client.json('/drive/v3/files') + expect(mocks.fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ acceptCompressed: true, connectionPool: pool }) + ) + }) it('does not issue any request after cancellation', async () => { const controller = new AbortController() controller.abort() diff --git a/apps/sim/lib/sim-search/live/http.ts b/apps/sim/lib/sim-search/live/http.ts index c4282b043a1..e3f8789973e 100644 --- a/apps/sim/lib/sim-search/live/http.ts +++ b/apps/sim/lib/sim-search/live/http.ts @@ -1,5 +1,9 @@ -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { + type PinnedConnectionPool, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import type { NativeClient } from '@/lib/sim-search/live/types' +import { isGoogleQuotaReason } from '@/connectors/google-workspace/api-errors' export class NativeSearchError extends Error { constructor( @@ -16,6 +20,8 @@ export function createNativeClient(input: { origin: string accessToken: string signal: AbortSignal + /** Reuses connections across this client's requests; the caller owns its lifetime. */ + pool?: PinnedConnectionPool }): NativeClient { let requests = 0 async function request( @@ -57,10 +63,18 @@ export function createNativeClient(input: { timeout: 10_000, maxResponseBytes: 4 * 1024 * 1024, maxRedirects: 0, + acceptCompressed: true, + connectionPool: input.pool, }) if (!response.ok) { + /** Reading the small error body also returns a pooled connection for reuse. */ + let quotaExceeded = false + if (response.status === 403 && url.hostname.endsWith('.googleapis.com')) + quotaExceeded = isGoogleQuotaError(await response.json().catch(() => null)) + else await response.text().catch(() => '') if ( response.status === 429 || + quotaExceeded || (response.status === 403 && (response.headers.get('retry-after') !== null || response.headers.get('x-ratelimit-remaining') === '0')) @@ -90,16 +104,53 @@ export function createNativeClient(input: { } return response } - return { + return withJsonMemo({ async json(path, options) { return (await request(path, options)).json() }, async text(path, query) { return (await request(path, { query })).text() }, + }) +} + +/** + * Answers a repeated `memo` GET from the client's earlier response. A client lives for one + * search or read, so a memoized response never outlives the request that authorized it. + */ +export function withJsonMemo(client: NativeClient): NativeClient { + const memo = new Map>() + return { + ...client, + json(...request) { + const [path, options] = request + if (!options?.memo || options.body) return client.json(...request) + const key = JSON.stringify([path, options.query, options.googleService]) + let pending = memo.get(key) + if (!pending) { + pending = client.json(...request) + memo.set(key, pending) + pending.catch(() => memo.delete(key)) + } + return pending + }, } } +/** + * Google reports quota exhaustion as a 403 whose body names the reason, without the rate-limit + * headers other providers send. Only the reason codes are read; the body is never surfaced. + */ +function isGoogleQuotaError(body: unknown): boolean { + const error = object(object(body).error) + return ( + error.status === 'RESOURCE_EXHAUSTED' || + [...array(error.errors), ...array(error.details)].some((entry) => + isGoogleQuotaReason(string(entry.reason)) + ) + ) +} + /** Only extract typed fields from untrusted provider JSON. */ export function object(value: unknown): Record { return value !== null && typeof value === 'object' && !Array.isArray(value) diff --git a/apps/sim/lib/sim-search/live/pages.test.ts b/apps/sim/lib/sim-search/live/pages.test.ts new file mode 100644 index 00000000000..b7a23e03038 --- /dev/null +++ b/apps/sim/lib/sim-search/live/pages.test.ts @@ -0,0 +1,39 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { NativeSearchError } from '@/lib/sim-search/live/http' +import { collectNativePages, interleaveByRank, joinMessages } from '@/lib/sim-search/live/pages' + +const doc = (id: string) => ({ id, title: id, url: '', content: '' }) + +describe('native page merging', () => { + it('interleaves ranked lists of different lengths', () => { + expect(interleaveByRank([[1, 2, 3], [4], [5, 6]])).toEqual([1, 4, 5, 2, 6, 3]) + }) + it('joins each message once', () => { + expect(joinMessages(['a.', undefined, 'b.', 'a.'])).toBe('a. b.') + expect(joinMessages([undefined])).toBeUndefined() + }) + it('reports collection cursors as more results rather than degraded coverage', async () => { + const page = await collectNativePages( + [ + Promise.resolve({ documents: [doc('a')], nextCursor: '2', message: 'Searched.' }), + Promise.resolve({ documents: [doc('b')], message: 'Searched.' }), + ], + 'Guidance.' + ) + expect(page).toMatchObject({ partial: false, hasMore: true, message: 'Guidance.' }) + expect(page.documents.map((item) => item.id)).toEqual(['a', 'b']) + }) + it('reports a failed collection once, keeping successful evidence', async () => { + const failure = new NativeSearchError('unavailable', 'Code search failed.') + const page = await collectNativePages( + [ + Promise.resolve({ documents: [doc('a')] }), + Promise.reject(failure), + Promise.reject(failure), + ], + 'Guidance.' + ) + expect(page).toMatchObject({ partial: true, message: 'Guidance. Code search failed.' }) + }) +}) diff --git a/apps/sim/lib/sim-search/live/pages.ts b/apps/sim/lib/sim-search/live/pages.ts index 67aa635603a..539feec0406 100644 --- a/apps/sim/lib/sim-search/live/pages.ts +++ b/apps/sim/lib/sim-search/live/pages.ts @@ -1,6 +1,27 @@ import { NativeSearchError } from '@/lib/sim-search/live/http' import type { NativePage } from '@/lib/sim-search/live/types' +/** + * Merges ranked lists by position: every list's first item, then every second item, and so on. + * A populous list therefore cannot consume all of the previews ahead of the others. + */ +export function interleaveByRank(lists: readonly (readonly T[])[]): T[] { + const merged: T[] = [] + const longest = Math.max(0, ...lists.map((list) => list.length)) + for (let rank = 0; rank < longest; rank++) { + for (const list of lists) if (rank < list.length) merged.push(list[rank]!) + } + return merged +} + +/** Joins status sentences once each, in first-seen order. */ +export function joinMessages(messages: readonly (string | undefined)[]): string | undefined { + return ( + [...new Set(messages.filter((message): message is string => Boolean(message)))].join(' ') || + undefined + ) +} + /** Independent collections retain their successful evidence if another collection fails. */ export async function collectNativePages( requests: Promise[], @@ -12,20 +33,16 @@ export async function collectNativePages( result.status === 'rejected' ? [result.reason] : [] ) if (!pages.length && failures.length) throw failures[0] - const documents = [] - // Interleave ranks so a populous collection cannot consume all cross-collection previews. - for (let rank = 0; rank < Math.max(0, ...pages.map((page) => page.documents.length)); rank++) { - for (const page of pages) if (page.documents[rank]) documents.push(page.documents[rank]) - } return { - documents, - partial: failures.length > 0 || pages.some((page) => page.partial || page.nextCursor), - message: [ + documents: interleaveByRank(pages.map((page) => page.documents)), + partial: failures.length > 0 || pages.some((page) => page.partial), + hasMore: pages.some((page) => page.hasMore || page.nextCursor), + message: joinMessages([ guidance, - ...pages.filter((page) => page.partial && page.message).map((page) => page.message), + ...pages.filter((page) => page.partial).map((page) => page.message), ...failures.map((error) => error instanceof NativeSearchError ? error.message : 'One collection could not be searched.' ), - ].join(' '), + ]), } } diff --git a/apps/sim/lib/sim-search/live/policy.ts b/apps/sim/lib/sim-search/live/policy.ts index a8a5ce2a143..d2e41252059 100644 --- a/apps/sim/lib/sim-search/live/policy.ts +++ b/apps/sim/lib/sim-search/live/policy.ts @@ -30,7 +30,8 @@ export function permitsPath(policy: LiveSearchPolicy, path: string) { } /** - * One request-local metadata cache; neither tokens nor permissions survive the request. + * Metadata reads are memoized on the request-scoped client, so neither tokens nor permissions + * survive the request. * Optional document metadata must come from a current read using this verifier's client. */ export function createPolicyVerifier( @@ -41,17 +42,10 @@ export function createPolicyVerifier( mcp?: CodaMcpClient ): PolicyVerifier { if (!requiresScopedRetrieval(provider, policy)) return async () => true - const calls = new Map>() const json = (path: string, query?: Record) => { if (!client) throw new NativeSearchError('unavailable', 'This connection cannot verify the search scope.') - const key = JSON.stringify([path, query]) - let result = calls.get(key) - if (!result) { - result = client.json(path, query ? { query } : undefined) - calls.set(key, result) - } - return result + return client.json(path, { query, memo: true }) } const restricted = policy.mode === 'selected' || policy.excluded.length > 0 const siteAllowed = (value: string) => { diff --git a/apps/sim/lib/sim-search/live/providers.test.ts b/apps/sim/lib/sim-search/live/providers.test.ts index 4a19ae95e83..63047f8a8b8 100644 --- a/apps/sim/lib/sim-search/live/providers.test.ts +++ b/apps/sim/lib/sim-search/live/providers.test.ts @@ -5,6 +5,7 @@ import { searchCoda } from '@/lib/sim-search/live/coda' import { searchGitHub } from '@/lib/sim-search/live/github' import { readGitLab, searchGitLab } from '@/lib/sim-search/live/gitlab' import { readDrive, searchCalendar, searchDrive, searchGmail } from '@/lib/sim-search/live/google' +import { withJsonMemo } from '@/lib/sim-search/live/http' import { readSlack, searchSlack } from '@/lib/sim-search/live/slack' import type { NativeClient } from '@/lib/sim-search/live/types' @@ -105,13 +106,190 @@ describe('native search endpoints', () => { const api = client() api.json .mockResolvedValueOnce({ items: [{ id: 'primary' }, { id: 'team@example.com' }] }) - .mockResolvedValue({ - items: [{ id: 'e', summary: 'Launch', htmlLink: 'https://calendar.google.com/event' }], - }) + .mockResolvedValueOnce({ items: [{ id: 'mine', summary: 'Launch' }] }) + .mockResolvedValueOnce({ items: [{ id: 'team', summary: 'Launch review' }] }) const result = await searchCalendar(api, input) expect(result.documents.map((item) => item.container)).toEqual(['primary', 'team@example.com']) expect(api.json.mock.calls[2][0]).toContain('team%40example.com') }) + it('marks a meeting shared across calendars as one item, primary calendar first', async () => { + const api = client() + const shared = (calendar: string, dateTime: string) => ({ + id: 'meeting', + iCalUID: 'meeting@google.com', + summary: 'Pilot planning', + htmlLink: `https://calendar.google.com/${calendar}`, + start: { dateTime }, + }) + api.json + .mockResolvedValueOnce({ + items: [{ id: 'teammate@example.com' }, { id: 'member@example.com', primary: true }], + }) + .mockResolvedValueOnce({ items: [shared('me', '2026-09-24T09:00:00-07:00')] }) + .mockResolvedValueOnce({ items: [shared('emir', '2026-09-24T12:00:00-04:00')] }) + const result = await searchCalendar(api, input) + expect(result.documents.map((item) => item.container)).toEqual([ + 'member@example.com', + 'teammate@example.com', + ]) + expect(new Set(result.documents.map((item) => item.dedupeKey)).size).toBe(1) + expect(api.json.mock.calls[1][0]).toContain('member%40example.com') + }) + it('keeps recurring occurrences that share one iCalendar UID', async () => { + const api = client() + const occurrence = (day: string) => ({ + id: `standup_${day}`, + iCalUID: 'standup@google.com', + summary: 'Standup', + start: { dateTime: `2026-09-${day}T09:00:00Z` }, + }) + api.json + .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }] }) + .mockResolvedValueOnce({ items: [occurrence('24'), occurrence('25')] }) + const result = await searchCalendar(api, { + ...input, + query: '', + filters: { startDate: '2026-09-24T00:00:00Z', endDate: '2026-09-26T00:00:00Z' }, + }) + expect(result.documents.map((item) => item.id)).toEqual(['standup_24', 'standup_25']) + expect(result.documents[0]?.dedupeKey).not.toBe(result.documents[1]?.dedupeKey) + }) + it('orders a dated agenda by start time across calendars', async () => { + const api = client() + const event = (id: string, hour: string) => ({ + id, + summary: id, + start: { dateTime: `2026-09-24T${hour}:00:00Z` }, + }) + api.json + .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }, { id: 'team' }] }) + .mockResolvedValueOnce({ items: [event('late', '17'), event('later', '18')] }) + .mockResolvedValueOnce({ items: [event('early', '08')], nextPageToken: 'more' }) + const result = await searchCalendar(api, { + ...input, + query: '', + filters: { startDate: '2026-09-24T00:00:00Z', endDate: '2026-09-25T00:00:00Z' }, + }) + expect(result.documents.map((item) => item.id)).toEqual(['early', 'late', 'later']) + expect(result).toMatchObject({ hasMore: true, partial: false }) + expect(api.json.mock.calls[1][1]?.query).toMatchObject({ + singleEvents: 'true', + orderBy: 'startTime', + }) + }) + it('turns HTML event descriptions into readable text', async () => { + const api = client() + api.json + .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }] }) + .mockResolvedValueOnce({ + items: [ + { + id: 'e', + summary: 'Demo', + description: + 'Who:
host@example.com
Where: https://meet.google.com/abc', + }, + ], + }) + const [event] = (await searchCalendar(api, input)).documents + expect(event.content).toContain('Who:\nhost@example.com\nWhere: https://meet.google.com/abc') + expect(event.content).not.toContain(' { + const api = client() + api.json.mockResolvedValueOnce({ messages: [{ id: 'm' }] }).mockResolvedValueOnce({ + id: 'm', + labelIds: ['INBOX'], + snippet: 'Let's ship it \u034f \u034f \u200c\u200b\ufeff \u034f', + internalDate: '1700000000000', + payload: { headers: [{ name: 'Subject', value: 'Launch' }] }, + }) + const [message] = (await searchGmail(api, input)).documents + expect(message.content).toBe("Let's ship it") + expect(message.accessMetadata).toEqual({ id: 'm', labelIds: ['INBOX'] }) + expect(api.json.mock.calls[1][1]?.query).toMatchObject({ + format: 'metadata', + metadataHeaders: ['Subject', 'From'], + fields: 'id,labelIds,snippet,internalDate,payload/headers', + }) + }) + it('batches code search by bytes and searches issues in fewer, larger batches', async () => { + const api = client() + const repositories = Array.from({ length: 100 }, (_, index) => ({ + full_name: `simstudioai/repository-with-a-long-name-${index}`, + })) + api.json.mockImplementation(async (path) => + path === '/user/repos' ? repositories : { items: [], total_count: 0 } + ) + const result = await searchGitHub(api, { ...input, query: 'déploiement 検索' }) + const queries = (path: string) => + api.json.mock.calls + .filter(([called]) => called === path) + .map(([, options]) => Buffer.byteLength(String(options?.query?.q))) + expect(Math.max(...queries('/search/code'))).toBeLessThanOrEqual(1000) + expect(queries('/search/code').length).toBeLessThanOrEqual(4) + expect(queries('/search/issues').length).toBeLessThan(queries('/search/code').length * 2) + expect(result.partial).toBe(true) + expect(result.message).toMatch(/Only the \d+ most recently pushed repositories were searched/) + }) + it('keeps GitHub qualifiers outside the grouped text of a dated search', async () => { + const api = client() + api.json.mockImplementation(async (path) => + path === '/user/repos' ? [{ full_name: 'org/repo' }] : { items: [], total_count: 0 } + ) + await searchGitHub(api, { + ...input, + filters: { startDate: '2026-09-20T00:00:00Z', endDate: '2026-09-24T00:00:00Z' }, + }) + const searches = api.json.mock.calls.filter(([path]) => path.startsWith('/search/')) + expect(searches.map(([path]) => path)).toEqual(['/search/issues', '/search/issues']) + expect(searches.map(([, options]) => options?.query?.q)).toEqual([ + '(launch) repo:org/repo is:issue updated:2026-09-20T00:00:00.000Z..2026-09-24T00:00:00.000Z', + '(launch) repo:org/repo is:pull-request updated:2026-09-20T00:00:00.000Z..2026-09-24T00:00:00.000Z', + ]) + }) + it('lists dated GitHub issues without grouping an empty query', async () => { + const api = client() + api.json.mockResolvedValue({ items: [], total_count: 0 }) + await searchGitHub(api, { + ...input, + query: '', + native: { provider: 'github', query: 'repo:org/repo is:issue', kind: 'issues' }, + filters: { startDate: '2026-09-20T00:00:00Z' }, + }) + expect(api.json.mock.calls[0][1]?.query?.q).toBe( + 'repo:org/repo is:issue updated:>=2026-09-20T00:00:00.000Z' + ) + }) + it('searches Atlassian sites in parallel and reads accessible sites once per client', async () => { + const api = client() + api.json.mockImplementation(async (path) => { + if (path === '/oauth/token/accessible-resources') + return [ + { id: 'one', url: 'https://one.atlassian.net' }, + { id: 'two', url: 'https://two.atlassian.net' }, + ] + const site = path.includes('/one/') ? 'one' : 'two' + return { + results: [ + { + content: { id: `${site}-1`, title: 'A' }, + excerpt: '@@@hl@@@Launch@@@endhl@@@ & plan', + }, + { content: { id: `${site}-2`, title: 'B' } }, + ], + } + }) + const session = withJsonMemo(api) + const result = await searchAtlassian(session, 'confluence', input) + await searchAtlassian(session, 'confluence', input) + expect(result.documents.map((item) => item.id)).toEqual(['one-1', 'two-1', 'one-2', 'two-2']) + expect(result.documents[0].content).toBe('Launch & plan') + expect( + api.json.mock.calls.filter(([path]) => path === '/oauth/token/accessible-resources') + ).toHaveLength(1) + }) it('calls Slack RTS with only channel types granted by the user token', async () => { const api = client() api.json.mockResolvedValue({ @@ -365,7 +543,7 @@ describe('native search endpoints', () => { ...input, native: { provider: 'github', query: 'repo:org/repo auth', kind: 'code' }, }) - expect(result).toMatchObject({ partial: true, nextCursor: '2' }) + expect(result).toMatchObject({ partial: true, hasMore: true, nextCursor: '2' }) expect(api.json.mock.calls[0][0]).toBe('/search/code') }) }) diff --git a/apps/sim/lib/sim-search/live/service-session.test.ts b/apps/sim/lib/sim-search/live/service-session.test.ts index f08bf2118e3..d752a131f30 100644 --- a/apps/sim/lib/sim-search/live/service-session.test.ts +++ b/apps/sim/lib/sim-search/live/service-session.test.ts @@ -137,7 +137,12 @@ describe('service credential isolation', () => { }) expect(session).toBeTruthy() expect(mocks.githubSources).toHaveBeenCalledWith(base.owner) - expect(mocks.github).toHaveBeenCalledWith([{ id: 'repository-source' }], api, base.signal) + expect(mocks.github).toHaveBeenCalledWith( + [{ id: 'repository-source' }], + api, + base.signal, + undefined + ) expect(mocks.source).not.toHaveBeenCalled() await expect( createLiveServiceSession({ ...base, member: null, provider: 'github', policy: configured }) diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts index a901dd92a28..9006ff86227 100644 --- a/apps/sim/lib/sim-search/live/service-session.ts +++ b/apps/sim/lib/sim-search/live/service-session.ts @@ -1,5 +1,6 @@ import type { LiveSearchProvider } from '@/lib/api/contracts/mothership-assistant-tools' import type { ResourceOwner } from '@/lib/core/resource-scope' +import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server' import { resolveConnectorAccessToken, resolveConnectorTokenUserId, @@ -41,12 +42,19 @@ export async function createLiveServiceSession(input: { member: NativeClient | null mcp?: CodaMcpClient signal: AbortSignal + /** Reuses connections for the source credential's requests; owned by the caller. */ + pool?: PinnedConnectionPool }): Promise { - const { provider, policy, member, signal } = input + const { provider, policy, member, signal, pool } = input if (policy.accessMode !== 'service_account' || provider === 'gitlab') return undefined if (provider === 'github') { if (!member) throw new NativeSearchError('reconnect', 'Connect your personal GitHub account.') - return createGitHubServiceVerifier(await loadLiveGitHubSources(input.owner), member, signal) + return createGitHubServiceVerifier( + await loadLiveGitHubSources(input.owner), + member, + signal, + pool + ) } if (!policy.sourceId) throw new NativeSearchError('unavailable', 'Ask an admin to select a service account source.') @@ -86,6 +94,7 @@ export async function createLiveServiceSession(input: { config: source.config, policy: sourcePolicy, signal, + pool, })), } } @@ -93,6 +102,7 @@ export async function createLiveServiceSession(input: { origin: LIVE_SEARCH_PROVIDER_CATALOG[provider].origin, accessToken: token.accessToken, signal, + pool, }) if (provider === 'coda') return { diff --git a/apps/sim/lib/sim-search/live/slack.ts b/apps/sim/lib/sim-search/live/slack.ts index b222ec42c7f..9897f72653b 100644 --- a/apps/sim/lib/sim-search/live/slack.ts +++ b/apps/sim/lib/sim-search/live/slack.ts @@ -1,4 +1,4 @@ -import { nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' +import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' import { array, NativeSearchError, object, string } from '@/lib/sim-search/live/http' import { slackConversationName, @@ -78,11 +78,8 @@ export async function searchSlack( limit: Math.min(input.limit, 20), ...(dates.start ? { after: Math.floor(Date.parse(dates.start) / 1000) - 1 } : {}), ...(dates.end ? { before: Math.ceil(Date.parse(dates.end) / 1000) } : {}), - ...(input.filters?.sortBy && input.filters.sortBy !== 'relevance' - ? { - sort: 'timestamp', - sort_dir: input.filters.sortBy === 'oldest' ? 'asc' : 'desc', - } + ...(dateSortDirection(input.filters) + ? { sort: 'timestamp', sort_dir: dateSortDirection(input.filters) } : {}), ...(input.native?.cursor ? { cursor: input.native.cursor } : {}), ...(input.native?.termClauses ? { term_clauses: input.native.termClauses } : {}), diff --git a/apps/sim/lib/sim-search/live/text.test.ts b/apps/sim/lib/sim-search/live/text.test.ts new file mode 100644 index 00000000000..9ec218da297 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text.test.ts @@ -0,0 +1,32 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { providerText } from '@/lib/sim-search/live/text' + +describe('providerText', () => { + it('decodes an escaped snippet and drops preheader padding', () => { + const padding = String.fromCodePoint(0x034f, 0x20, 0x200c, 0x200b, 0x200d, 0x200e, 0xfeff, 0xad) + expect(providerText(`Receipt from Exa 'Labs' ${padding.repeat(3)}`, 'escaped')).toBe( + "Receipt from Exa 'Labs'" + ) + }) + it('keeps a zero-width joiner that composes an emoji', () => { + const family = String.fromCodePoint(0x1f468, 0x200d, 0x1f469, 0x200d, 0x1f467) + expect(providerText(`Team ${family}`)).toBe(`Team ${family}`) + }) + it('keeps line structure when converting markup to text', () => { + expect( + providerText( + '

Who:

a@b.co
Notes & links

', + 'html' + ) + ).toBe('Who:\n\na@b.co\nNotes & links') + }) + it('leaves plain text with angle brackets untouched in auto mode', () => { + expect(providerText('Reply from John ', 'auto')).toBe( + 'Reply from John ' + ) + }) + it('does not decode entities in plain text', () => { + expect(providerText('Use <b> for bold')).toBe('Use <b> for bold') + }) +}) diff --git a/apps/sim/lib/sim-search/live/text.ts b/apps/sim/lib/sim-search/live/text.ts new file mode 100644 index 00000000000..dfe6ee87344 --- /dev/null +++ b/apps/sim/lib/sim-search/live/text.ts @@ -0,0 +1,57 @@ +import { convert, type HtmlToTextOptions } from 'html-to-text' +import { decodeHtmlEntities, looksLikeHtml } from '@/connectors/utils' + +/** + * How a provider encodes a text field: `plain` is used as-is, `escaped` is HTML-escaped text + * without markup (Gmail snippets), `html` is markup, and `auto` treats the value as markup only + * when it carries real HTML tags (Calendar descriptions may be either). + */ +type ProviderTextFormat = 'plain' | 'escaped' | 'html' | 'auto' + +/** + * Invisible format characters that marketing mail pads preheaders with. A zero-width joiner + * followed by a pictograph is kept because it composes a single emoji. The combining grapheme + * joiner is its own alternative because it cannot share a character class with base characters. + */ +const INVISIBLE_CHARACTERS = new RegExp( + [ + '[\\u00AD\\u061C\\u115F\\u1160\\u17B4\\u17B5\\u180E\\u200B\\u200C\\u200E\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\u3164\\uFEFF\\uFFA0]', + '\\u034F', + '\\u200D(?!\\p{Extended_Pictographic})', + ].join('|'), + 'gu' +) + +/** Markup becomes text only: links keep their visible text, and non-text elements are dropped. */ +const HTML_TO_TEXT: HtmlToTextOptions = { + wordwrap: false, + selectors: [ + { selector: 'a', options: { ignoreHref: true } }, + { selector: 'img', format: 'skip' }, + { selector: 'script', format: 'skip' }, + { selector: 'style', format: 'skip' }, + ...(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] as const).map((selector) => ({ + selector, + options: { uppercase: false }, + })), + ], +} + +/** + * Normalizes provider text into readable plain text for previews and document reads. Markup is + * never rendered: tags are dropped and entities decoded into literal characters. + */ +export function providerText(value: string, format: ProviderTextFormat = 'plain'): string { + let text = value + if (format === 'html' || (format === 'auto' && looksLikeHtml(text))) { + text = convert(text, HTML_TO_TEXT) + } else if (format === 'escaped') { + text = decodeHtmlEntities(text) + } + return text + .replace(INVISIBLE_CHARACTERS, '') + .replace(/[^\S\n]+/g, ' ') + .replace(/ *\n */g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim() +} diff --git a/apps/sim/lib/sim-search/live/types.ts b/apps/sim/lib/sim-search/live/types.ts index f0b38cfd7e2..f146546bf6b 100644 --- a/apps/sim/lib/sim-search/live/types.ts +++ b/apps/sim/lib/sim-search/live/types.ts @@ -21,8 +21,16 @@ export interface LiveAccount { } export interface NativeDocument { - /** Server-only GitLab permission evidence from the same response as the content. */ + /** + * Server-only permission evidence from the same response as the content. Only a verifier + * bound to the client that produced it may consume it; it is never projected to callers. + */ accessMetadata?: Record + /** + * Identifies one item a provider returns from several collections of the same account, such + * as a meeting on each attendee's calendar. Only the first verified copy is kept. + */ + dedupeKey?: string id: string container?: string containerName?: string @@ -40,7 +48,11 @@ export interface NativeDocument { export interface NativePage { documents: NativeDocument[] + /** Continues this exact query and account; implies more results exist. */ nextCursor?: string + /** More matches exist beyond this page but cannot be continued through it. */ + hasMore?: boolean + /** Coverage is degraded: a collection failed, a cap applied, or evidence was dropped. */ partial?: boolean message?: string } @@ -52,6 +64,8 @@ export interface NativeClient { query?: Record body?: unknown googleService?: 'sheets' + /** Reuse this client's earlier response to the same GET instead of requesting it again. */ + memo?: boolean } ): Promise text(path: string, query?: Record): Promise From 459e93cdedc1425c4fc5d731205f2cb3a67999b4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 21:38:57 -0700 Subject: [PATCH 2/6] fix(search): keep document spacing when cleaning provider text --- apps/sim/lib/sim-search/live/text.test.ts | 8 ++++++++ apps/sim/lib/sim-search/live/text.ts | 21 +++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/sim/lib/sim-search/live/text.test.ts b/apps/sim/lib/sim-search/live/text.test.ts index 9ec218da297..e203e0e16e7 100644 --- a/apps/sim/lib/sim-search/live/text.test.ts +++ b/apps/sim/lib/sim-search/live/text.test.ts @@ -26,6 +26,14 @@ describe('providerText', () => { 'Reply from John ' ) }) + it('keeps the spacing of plain text documents', () => { + const table = ' Name Qty\n Widget 2\n\n> quoted reply\n indented code' + expect(providerText(`${table} \n\n\n`)).toBe(table) + }) + it('removes invisible characters inside a word without splitting it', () => { + const softHyphen = String.fromCodePoint(0xad) + expect(providerText(`hyphen${softHyphen}ation`)).toBe('hyphenation') + }) it('does not decode entities in plain text', () => { expect(providerText('Use <b> for bold')).toBe('Use <b> for bold') }) diff --git a/apps/sim/lib/sim-search/live/text.ts b/apps/sim/lib/sim-search/live/text.ts index dfe6ee87344..2c21b4ac158 100644 --- a/apps/sim/lib/sim-search/live/text.ts +++ b/apps/sim/lib/sim-search/live/text.ts @@ -13,14 +13,10 @@ type ProviderTextFormat = 'plain' | 'escaped' | 'html' | 'auto' * followed by a pictograph is kept because it composes a single emoji. The combining grapheme * joiner is its own alternative because it cannot share a character class with base characters. */ -const INVISIBLE_CHARACTERS = new RegExp( - [ - '[\\u00AD\\u061C\\u115F\\u1160\\u17B4\\u17B5\\u180E\\u200B\\u200C\\u200E\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\u3164\\uFEFF\\uFFA0]', - '\\u034F', - '\\u200D(?!\\p{Extended_Pictographic})', - ].join('|'), - 'gu' -) +const INVISIBLE_CHARACTER = + '[\\u00AD\\u061C\\u115F\\u1160\\u17B4\\u17B5\\u180E\\u200B\\u200C\\u200E\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\u3164\\uFEFF\\uFFA0]|\\u034F|\\u200D(?!\\p{Extended_Pictographic})' +/** A run of invisible characters with the spaces between them, as preheader padding is built. */ +const INVISIBLE_RUN = new RegExp(`(?:[^\\S\\n]*(?:${INVISIBLE_CHARACTER}))+[^\\S\\n]*`, 'gu') /** Markup becomes text only: links keep their visible text, and non-text elements are dropped. */ const HTML_TO_TEXT: HtmlToTextOptions = { @@ -48,10 +44,11 @@ export function providerText(value: string, format: ProviderTextFormat = 'plain' } else if (format === 'escaped') { text = decodeHtmlEntities(text) } + /** Padding collapses to one space, or to nothing inside a word; other spacing is kept. */ + text = text.replace(INVISIBLE_RUN, (run) => (/\s/.test(run) ? ' ' : '')) + if (format === 'escaped') return text.replace(/\s+/g, ' ').trim() return text - .replace(INVISIBLE_CHARACTERS, '') - .replace(/[^\S\n]+/g, ' ') - .replace(/ *\n */g, '\n') + .replace(/[^\S\n]+$/gm, '') .replace(/\n{3,}/g, '\n\n') - .trim() + .replace(/^\n+|\n+$/g, '') } From af2a9a3e9f643f0b7e4a0f98731bd4fcc68cb574 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 22:01:04 -0700 Subject: [PATCH 3/6] fix(search): close audit findings on text, status, GitHub fan-out, and MCP chat - Keep invisible-character cleanup linear, table rows and definition lists readable, meaningful joiners intact, and deeply nested markup readable - Strip interactive Chat tags from MCP chat answers with the helper the Slack assistant shares - Report empty continuable pages and short dated listings as partial; dedupe after date filtering - Merge GitHub batches within each kind, give repository search the larger query budget, keep the 1,000-result hint, and skip code search when the query leaves no room for a repository - Keep memoized metadata failures for the request, re-read metadata after a document read, reload the Coda grant on every provider load, and request only gzip or brotli --- .../core/security/input-validation.server.ts | 9 ++- ...ecure-fetch-connection-pool.server.test.ts | 28 ++++++- .../lib/knowledge/application/chat.test.ts | 17 ++++ apps/sim/lib/knowledge/application/chat.ts | 9 ++- .../lib/mothership/chat/interactive-tags.ts | 8 ++ .../lib/sim-search/live/account-session.ts | 9 ++- .../lib/sim-search/live/application.test.ts | 80 ++++++++++++++++++- apps/sim/lib/sim-search/live/application.ts | 24 ++++-- apps/sim/lib/sim-search/live/coda-mcp.test.ts | 13 +++ apps/sim/lib/sim-search/live/coda-mcp.ts | 19 +---- apps/sim/lib/sim-search/live/github.ts | 60 +++++++++----- apps/sim/lib/sim-search/live/http.test.ts | 52 +++++++++++- apps/sim/lib/sim-search/live/http.ts | 6 +- apps/sim/lib/sim-search/live/pages.ts | 2 +- apps/sim/lib/sim-search/live/policy.test.ts | 16 ++++ apps/sim/lib/sim-search/live/policy.ts | 6 +- .../sim/lib/sim-search/live/providers.test.ts | 66 +++++++++++++++ .../sim-search/live/service-session.test.ts | 10 +++ apps/sim/lib/sim-search/live/text.test.ts | 26 ++++++ apps/sim/lib/sim-search/live/text.ts | 56 +++++++++++-- apps/sim/lib/slack-search/assistant-stream.ts | 6 +- 21 files changed, 451 insertions(+), 71 deletions(-) create mode 100644 apps/sim/lib/mothership/chat/interactive-tags.ts diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 8c89ed838c9..6e8bc5289b6 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -357,14 +357,16 @@ export interface SecureFetchOptions { /** Hide credential-derived URL details from validation logs. */ logUrlValidationDetails?: boolean /** - * Ask for a gzip, deflate, or brotli body. The body is decoded before it is returned, and + * Ask for a gzip or brotli body. The body is decoded before it is returned, and * `maxResponseBytes` bounds the decoded bytes, so a compression bomb still stops at the cap. */ acceptCompressed?: boolean /** * Reuses keep-alive connections to the same pinned address across requests. A connection is * only ever reused for the IP it was opened to, so every request keeps its DNS pinning. The - * owner must call {@link PinnedConnectionPool.destroy} once its requests have finished. + * owner must call {@link PinnedConnectionPool.destroy} once its requests have finished. Bun + * keeps sockets in its own per-address pool, so there reuse spans pools and `destroy` releases + * only the agents. */ connectionPool?: PinnedConnectionPool /** @@ -1158,7 +1160,8 @@ export async function secureFetchWithPinnedIP( } const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} - if (options.acceptCompressed) sanitizedHeaders['accept-encoding'] = 'gzip, deflate, br' + /** Raw deflate streams are ambiguous to decode, so only gzip and brotli are requested. */ + if (options.acceptCompressed) sanitizedHeaders['accept-encoding'] = 'gzip, br' if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) { sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT } diff --git a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts index 97d67cbc308..482a1ed64d4 100644 --- a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts @@ -86,9 +86,35 @@ describe('secureFetchWithPinnedIP connection reuse', () => { pool.destroy() } }) + + it('falls back to a single-use agent for a request after the pool is destroyed', async () => { + const server = await startServer((_req, res) => res.end('ok')) + const pool = createPinnedConnectionPool() + pool.destroy() + expect(pool.agent(false, '127.0.0.1', 80, '127.0.0.1')).toBeUndefined() + const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + connectionPool: pool, + }) + await expect(response.text()).resolves.toBe('ok') + }) }) describe('secureFetchWithPinnedIP compressed responses', () => { + it('caps the decoded size of a compressed body it asked for', async () => { + const bomb = gzipSync(Buffer.alloc(64 * 1024, 0x41)) + const server = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(bomb) + }) + const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + acceptCompressed: true, + maxResponseBytes: 1024, + }) + expect(bomb.length).toBeLessThan(1024) + await expect(response.text()).rejects.toThrow(/response body/i) + }) it('asks for compression only when requested and returns the decoded body', async () => { const encodings: (string | undefined)[] = [] const server = await startServer((req, res) => { @@ -109,7 +135,7 @@ describe('secureFetchWithPinnedIP compressed responses', () => { profile: 'configuredEndpoint', }) await expect(plain.json()).resolves.toEqual({ ok: true }) - expect(encodings).toEqual(['gzip, deflate, br', undefined]) + expect(encodings).toEqual(['gzip, br', undefined]) }) }) diff --git a/apps/sim/lib/knowledge/application/chat.test.ts b/apps/sim/lib/knowledge/application/chat.test.ts index 2ea5c6de390..d14f5084ab3 100644 --- a/apps/sim/lib/knowledge/application/chat.test.ts +++ b/apps/sim/lib/knowledge/application/chat.test.ts @@ -303,6 +303,23 @@ describe('organization Search Assistant chat', () => { expect(dbChainMockFns.update).toHaveBeenCalledOnce() }) + it('drops interactive Chat tags from the MCP answer but keeps them in the transcript', async () => { + const tags = '{"1":"Open it"}Which kit?' + mocks.lifecycle.mockResolvedValue(createResult({ content: `Violet suitcase.${tags}` })) + const result = await execute() + expect(result.content).toBe('Violet suitcase.') + const [, messages] = mocks.persist.mock.calls[0] + expect(JSON.stringify(messages)).toContain('') + expect(JSON.stringify(messages)).toContain('') + }) + + it('refuses an answer that is only interactive Chat tags', async () => { + mocks.lifecycle.mockResolvedValue( + createResult({ content: '{"1":"Open it"}' }) + ) + await expect(execute()).rejects.toThrow('The assistant returned no answer') + }) + it('fails closed when retrieval makes provenance incomplete', async () => { const registry = new ResolvedSecretTraceRegistry() mocks.lifecycle.mockImplementation(async () => { diff --git a/apps/sim/lib/knowledge/application/chat.ts b/apps/sim/lib/knowledge/application/chat.ts index dc0ccdb0498..53778efa041 100644 --- a/apps/sim/lib/knowledge/application/chat.ts +++ b/apps/sim/lib/knowledge/application/chat.ts @@ -22,6 +22,7 @@ import { } from '@/lib/knowledge/application/chat-citations' import { organizationSearchChatOperation } from '@/lib/knowledge/application/chat-operations' import { loadCopilotSearchIntegrations } from '@/lib/mothership/application/load-search-integrations' +import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' import { persistCopilotChatTurn } from '@/lib/mothership/chat/messages-store' import { buildCopilotRequestPayload } from '@/lib/mothership/chat/payload' import { @@ -201,7 +202,13 @@ export const organizationSearchChat: OperationUseCase< buildPersistedAssistantMessage(result, messageId, 'assistant'), registry ) - const answer = resolveSearchChatCitations(assistantMessage.content, result.toolCalls) + /** MCP clients receive text; interactive Chat cards stay only in the saved transcript. */ + const answer = resolveSearchChatCitations( + stripInteractiveTags(assistantMessage.content), + result.toolCalls + ) + if (!answer.content.trim()) + throw new OrchestrationError('internal', 'The assistant returned no answer. Try again.') if (!isResolvedSecretModelContentUnchanged(answer, registry)) { throw new OrchestrationError('internal', 'Search answer could not be returned safely') } diff --git a/apps/sim/lib/mothership/chat/interactive-tags.ts b/apps/sim/lib/mothership/chat/interactive-tags.ts new file mode 100644 index 00000000000..b149eaaf7c5 --- /dev/null +++ b/apps/sim/lib/mothership/chat/interactive-tags.ts @@ -0,0 +1,8 @@ +/** Answer tags the Chat UI renders as interactive cards; text-only surfaces drop them. */ +const INTERACTIVE_TAGS = + /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?(?:<\/\1>|$)/g + +/** Removes interactive Chat UI tags, including one still open at the end of a partial stream. */ +export function stripInteractiveTags(text: string): string { + return text.replace(INTERACTIVE_TAGS, '') +} diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index b84e7cc452a..60df3975c8d 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -73,7 +73,7 @@ export async function openLiveAccountSession( const mcp = client ? undefined : await createCodaMcpClient(owner, userId, account.id, signal) /** A service source replaces the member policy and verifies with its own credential. */ - const sourceBoundary = async (memberPolicy: LiveSearchPolicy) => { + const sourceBoundary = async (memberPolicy: LiveSearchPolicy, fresh = false) => { const service = await createLiveServiceSession({ owner, userId, @@ -85,7 +85,9 @@ export async function openLiveAccountSession( pool: input.pool, }) if (service) return service - const verifyPolicy = createPolicyVerifier(provider, memberPolicy, client, origin, mcp) + const verifyPolicy = createPolicyVerifier(provider, memberPolicy, client, origin, mcp, { + fresh, + }) return { policy: memberPolicy, partial: false, @@ -117,7 +119,8 @@ export async function openLiveAccountSession( }, async verifyCurrent(document) { const current = await sourceBoundary( - livePolicyFor(await loadLiveSearchPolicies(owner), provider) + livePolicyFor(await loadLiveSearchPolicies(owner), provider), + true ) return current.verify(document) }, diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index cc51ac8c910..6dbeab6a755 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -16,6 +16,10 @@ const mocks = vi.hoisted(() => ({ adminVerify: vi.fn(), json: vi.fn(), service: vi.fn(), + destroyPool: vi.fn(), +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + createPinnedConnectionPool: () => ({ agent: vi.fn(), destroy: mocks.destroyPool }), })) vi.mock('@/lib/sim-search/live/service-session', () => ({ createLiveServiceSession: mocks.service, @@ -401,11 +405,85 @@ describe('authorized live retrieval', () => { expect(status.message).toContain('Search this account alone') } }) + it('drops the cursor only for the account whose results were cut from the merge', async () => { + const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' } + mocks.accounts.mockResolvedValue([account, gmail]) + mocks.search.mockImplementation(async (provider: string) => ({ + documents: (provider === 'gmail' ? [1, 2] : [1]).map((rank) => ({ + ...document, + id: `${provider}-${rank}`, + url: `https://docs.google.com/${provider}-${rank}`, + })), + nextCursor: 'next', + })) + const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 2 } }) + const statusOf = (id: string) => result.live?.accounts.find((row) => row.accountId === id) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).not.toContain( + 'gmail-2' + ) + expect(statusOf('mail')?.nextCursor).toBeUndefined() + expect(statusOf('account')).toMatchObject({ nextCursor: 'next' }) + }) + it('reports a dated search partial when the provider has more it cannot continue', async () => { + mocks.search.mockResolvedValue({ documents: [document], hasMore: true }) + const result = await searchLiveKnowledge.execute({ + principal, + input: { + ...input, + filters: { startDate: '2026-08-01T00:00:00Z', endDate: '2026-10-01T00:00:00Z' }, + }, + }) + expect(result.results).toHaveLength(1) + expect(result.live?.accounts[0]).toMatchObject({ + status: 'partial', + message: expect.stringContaining('date range'), + }) + }) + it('applies date filters before spending provider verification', async () => { + const verify = vi.fn(async () => true) + mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false }) + const outside = { + ...document, + id: 'outside', + url: 'https://docs.google.com/outside', + modifiedAt: '2026-01-01T00:00:00Z', + } + mocks.search.mockResolvedValue({ documents: [document, outside] }) + const result = await searchLiveKnowledge.execute({ + principal, + input: { + ...input, + filters: { startDate: '2026-08-01T00:00:00Z', endDate: '2026-10-01T00:00:00Z' }, + }, + }) + expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual(['doc']) + expect(verify).toHaveBeenCalledExactlyOnceWith(document) + }) + it('releases the connection pool once, even when the search is cancelled mid-flight', async () => { + await searchLiveKnowledge.execute({ principal, input }) + expect(mocks.destroyPool).toHaveBeenCalledOnce() + mocks.destroyPool.mockClear() + const controller = new AbortController() + mocks.search.mockImplementation(async () => { + controller.abort() + throw new NativeSearchError('unavailable', 'Cancelled') + }) + await expect( + searchLiveKnowledge.execute({ principal, input: { ...input, signal: controller.signal } }) + ).rejects.toThrow() + expect(mocks.destroyPool).toHaveBeenCalledOnce() + }) it('reports partial coverage when more matches exist but none could be returned', async () => { mocks.search.mockResolvedValue({ documents: [], hasMore: true }) const result = await searchLiveKnowledge.execute({ principal, input }) expect(result.retrieval.status).toBe('partial') - expect(result.live?.accounts[0]?.message).toContain('More matches exist') + expect(result.live?.accounts[0]?.message).toContain('More matches may exist') + }) + it('reports partial coverage when a continuable page returned nothing readable', async () => { + mocks.search.mockResolvedValue({ documents: [], nextCursor: 'next' }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.retrieval.status).toBe('partial') + expect(result.live?.accounts[0]).toMatchObject({ status: 'partial', nextCursor: 'next' }) }) it('rejects invalid dates before resolving provider credentials', async () => { await expect( diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 692a8faf887..0da9ea55dd7 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -372,17 +372,21 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ ) ) ) - const readable = firstOfEachDocument(permitted) - const matching = readable.filter(({ document, documentId }) => - matchesLiveFilters(document, documentId, account.provider, filters) + const matching = firstOfEachDocument( + permitted.filter(({ document, documentId }) => + matchesLiveFilters(document, documentId, account.provider, filters) + ) ) - const undatedExcluded = readable.some(({ document }) => + const undatedExcluded = permitted.some(({ document }) => lacksFilterDate(document, account.provider, filters) ) const undatedUnsorted = dateSorted && matching.some(({ document }) => !sourceDate(document, account.provider)) const moreUnsorted = dateSorted && Boolean(page.nextCursor || page.hasMore) - const moreUnreachable = Boolean(page.hasMore && !page.nextCursor && !matching.length) + /** Nothing readable came back, yet the provider has more: an empty page proves nothing. */ + const moreWithoutResults = Boolean((page.hasMore || page.nextCursor) && !matching.length) + /** A dated listing should cover its window; uncontinuable extra matches leave it short. */ + const moreInDateRange = Boolean(hasDateBounds(filters) && page.hasMore && !page.nextCursor) const degraded = unverified || session.servicePartial || @@ -390,7 +394,8 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ undatedExcluded || undatedUnsorted || moreUnsorted || - moreUnreachable + moreWithoutResults || + moreInDateRange return { status: { ...status, @@ -409,8 +414,11 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ dateSorted ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.' : undefined, - moreUnreachable - ? 'More matches exist than this search returned. Narrow the query or target one source.' + moreWithoutResults + ? 'More matches may exist beyond this page. Continue with nextCursor when present, narrow the query, or target one source.' + : undefined, + moreInDateRange && !moreWithoutResults + ? 'More matches exist in this date range than were returned. Narrow the range or target one source.' : undefined, ]), nextCursor: page.nextCursor, diff --git a/apps/sim/lib/sim-search/live/coda-mcp.test.ts b/apps/sim/lib/sim-search/live/coda-mcp.test.ts index 26fc2df9add..742f675ce45 100644 --- a/apps/sim/lib/sim-search/live/coda-mcp.test.ts +++ b/apps/sim/lib/sim-search/live/coda-mcp.test.ts @@ -126,6 +126,19 @@ describe('Coda MCP content search', () => { expect(page).toMatchObject({ partial: true, documents: [{ id: 'coda://docs/doc' }] }) expect(page.message).toContain('Date filters apply to returned timestamps in Sim') }) + it('reports more results without claiming degraded coverage', async () => { + const call = vi.fn().mockResolvedValue({ + results: [{ uri: 'coda://docs/doc', title: 'Launch' }], + hasMore: true, + }) + expect(await searchCodaMcp({ call }, input)).toMatchObject({ hasMore: true, partial: false }) + expect( + await searchCodaMcp( + { call }, + { ...input, filters: { startDate: '2026-09-01', endDate: '2026-09-30' } } + ) + ).toMatchObject({ hasMore: true, partial: true }) + }) it('rejects page-scoped document filters before calling the provider', async () => { const call = vi.fn() await expect( diff --git a/apps/sim/lib/sim-search/live/coda-mcp.ts b/apps/sim/lib/sim-search/live/coda-mcp.ts index 085600c80de..37520c4617c 100644 --- a/apps/sim/lib/sim-search/live/coda-mcp.ts +++ b/apps/sim/lib/sim-search/live/coda-mcp.ts @@ -44,22 +44,11 @@ export async function createCodaMcpClient( throw new NativeSearchError('reconnect', 'Coda connection changed. Search again.') return current } - /** - * The first provider load reuses the grant checked moments earlier in the same step; any later - * load, such as a token refresh inside the MCP client, reads the current grant again. - */ - const providerLoader = (checked: typeof initial) => { - let first: typeof initial | undefined = checked - return async () => { - const current = first ?? (await loadCurrent()) - first = undefined - return createManagedMcpAuthProvider(current) - } - } + const loadProvider = async () => createManagedMcpAuthProvider(await loadCurrent()) const tools = await mcpService.discoverManagedMcpTools( initial.mcpServerId, initial.scope, - { credentialId, loadProvider: providerLoader(initial) }, + { credentialId, loadProvider }, signal, { requireComplete: true } ) @@ -75,13 +64,13 @@ export async function createCodaMcpClient( `Coda no longer advertises ${name}. Reconnect or update the connector.` ) validateToolArguments(tool, args) - const current = await loadCurrent() + await loadCurrent() const result = await mcpService.executeManagedMcpTool({ connectionId: credentialId, serverId: initial.mcpServerId, scope: initial.scope, toolCall: { name, arguments: args }, - loadAuthProvider: providerLoader(current), + loadAuthProvider: loadProvider, signal, timeoutMs: 10_000, }) diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts index fedbc860a8f..1dc73a365d4 100644 --- a/apps/sim/lib/sim-search/live/github.ts +++ b/apps/sim/lib/sim-search/live/github.ts @@ -52,8 +52,8 @@ function githubDocument(row: Record, kind: string): NativeDocum */ const GITHUB_CODE_QUERY_BYTES = 980 /** - * Issue search limits only its free text to 256 characters and accepts roughly 4,000 bytes of - * repository qualifiers; batches leave room for the type and date qualifiers appended later. + * Issue and repository search limit only free text to 256 characters and accept roughly 4,000 + * bytes of repository qualifiers; batches leave room for type and date qualifiers appended later. */ const GITHUB_ISSUE_QUERY_BYTES = 3800 /** Batches per kind; each code batch spends one of GitHub's ten code searches per minute. */ @@ -74,17 +74,22 @@ function githubKinds(input: NativeSearchInput): GitHubKind[] { return hasDateBounds(input.filters) ? ['issues'] : ['issues', 'code'] } -/** Splits repositories into qualifier batches that keep each query within `maxBytes`. */ +/** + * Splits repositories into qualifier batches that keep each query within `maxBytes`. A + * repository whose qualifier cannot fit beside the query at all is left out. + */ function repositoryBatches(query: string, names: readonly string[], maxBytes: number): string[][] { + const base = Buffer.byteLength(query) const batches: string[][] = [] let batch: string[] = [] - let bytes = Buffer.byteLength(query) + let bytes = base for (const name of names) { const term = Buffer.byteLength(` repo:${name}`) - if (batch.length && bytes + term > maxBytes) { + if (base + term > maxBytes) continue + if (bytes + term > maxBytes) { if (batches.push(batch) === GITHUB_MAX_REPOSITORY_BATCHES) return batches batch = [] - bytes = Buffer.byteLength(query) + bytes = base } batch.push(name) bytes += term @@ -133,31 +138,42 @@ export async function searchGitHub( message: 'No repositories are accessible through this GitHub connection.', } const kinds = githubKinds(input) - const batches = kinds.map((kind) => ({ + const planned = kinds.map((kind) => ({ kind, repositories: repositoryBatches( query, names, - kind === 'issues' ? GITHUB_ISSUE_QUERY_BYTES : GITHUB_CODE_QUERY_BYTES + kind === 'code' ? GITHUB_CODE_QUERY_BYTES : GITHUB_ISSUE_QUERY_BYTES ), })) + const batches = planned.filter(({ repositories }) => repositories.length) + const skipped = planned.filter(({ repositories }) => !repositories.length) + if (!batches.length) + throw new NativeSearchError( + 'unavailable', + 'This query is too long to scope to your GitHub repositories. Shorten it or add a repo: qualifier.' + ) const searched = Math.min( ...batches.map(({ repositories }) => repositories.reduce((count, batch) => count + batch.length, 0) ) ) + /** Batches merge within their kind first, so a kind with more batches cannot crowd out another. */ const result = await collectNativePages( - batches.flatMap(({ kind, repositories }) => - repositories.map((batch) => - searchGitHub(client, { - ...input, - native: { - provider: 'github', - ...input.native, - kind, - query: [query, ...batch.map((name) => `repo:${name}`)].filter(Boolean).join(' '), - }, - }) + batches.map(({ kind, repositories }) => + collectNativePages( + repositories.map((batch) => + searchGitHub(client, { + ...input, + native: { + provider: 'github', + ...input.native, + kind, + query: [query, ...batch.map((name) => `repo:${name}`)].filter(Boolean).join(' '), + }, + }) + ), + '' ) ), 'Searched repositories you own, collaborate on, or access through organization membership. Use a repo: qualifier to narrow results.' @@ -165,9 +181,13 @@ export async function searchGitHub( const capped = repositories.length === 100 || searched < names.length return { ...result, - partial: result.partial || capped, + partial: result.partial || capped || skipped.length > 0, message: joinMessages([ result.message, + ...skipped.map( + ({ kind }) => + `GitHub ${kind} search was skipped because the query is too long to scope to repositories.` + ), !input.native?.kind && !kinds.includes('code') ? CODE_EXCLUDED_BY_DATES : undefined, capped ? `Only the ${searched} most recently pushed repositories were searched; target a repository for broader coverage.` diff --git a/apps/sim/lib/sim-search/live/http.test.ts b/apps/sim/lib/sim-search/live/http.test.ts index bde4d4e0645..0f07ecccb07 100644 --- a/apps/sim/lib/sim-search/live/http.test.ts +++ b/apps/sim/lib/sim-search/live/http.test.ts @@ -6,7 +6,7 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({ secureFetchWithValidation: mocks.fetch, })) -import { createNativeClient } from '@/lib/sim-search/live/http' +import { createNativeClient, NativeSearchError, withJsonMemo } from '@/lib/sim-search/live/http' describe('native search network boundary', () => { beforeEach(() => { @@ -141,6 +141,18 @@ describe('native search network boundary', () => { expect.objectContaining({ acceptCompressed: true, connectionPool: pool }) ) }) + it('reuses a memoized response, including a failure, only when asked to', async () => { + const json = vi + .fn() + .mockRejectedValueOnce(new NativeSearchError('rate_limited', 'Later')) + .mockResolvedValue({ ok: true }) + const client = withJsonMemo({ json, text: vi.fn() }) + await expect(client.json('/labels', { memo: true })).rejects.toThrow('Later') + await expect(client.json('/labels', { memo: true })).rejects.toThrow('Later') + expect(json).toHaveBeenCalledTimes(1) + await expect(client.json('/labels')).resolves.toEqual({ ok: true }) + expect(json).toHaveBeenCalledTimes(2) + }) it('does not issue any request after cancellation', async () => { const controller = new AbortController() controller.abort() @@ -152,4 +164,42 @@ describe('native search network boundary', () => { await expect(client.json('/api/assistant.search.context')).rejects.toThrow() expect(mocks.fetch).not.toHaveBeenCalled() }) + it('memoizes a repeated GET by path and query, never a request with a body', async () => { + mocks.fetch.mockImplementation(async () => new Response('{"ok":true}', { status: 200 })) + const client = createNativeClient({ + origin: 'https://slack.com', + accessToken: 'private', + signal: new AbortController().signal, + }) + await client.json('/api/team.info', { query: { team: 'a' }, memo: true }) + await client.json('/api/team.info', { query: { team: 'a' }, memo: true }) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + await client.json('/api/team.info', { query: { team: 'b' }, memo: true }) + expect(mocks.fetch).toHaveBeenCalledTimes(2) + await client.json('/api/search', { body: { query: 'launch' }, memo: true }) + await client.json('/api/search', { body: { query: 'launch' }, memo: true }) + expect(mocks.fetch).toHaveBeenCalledTimes(4) + }) + it('reports a Google RESOURCE_EXHAUSTED 403 as a rate limit', async () => { + mocks.fetch.mockResolvedValue( + new Response(JSON.stringify({ error: { status: 'RESOURCE_EXHAUSTED' } }), { status: 403 }) + ) + const client = createNativeClient({ + origin: 'https://www.googleapis.com', + accessToken: 'private', + signal: new AbortController().signal, + }) + await expect(client.json('/drive/v3/files')).rejects.toMatchObject({ status: 'rate_limited' }) + }) + it('reads a quota-shaped 403 body only from Google origins', async () => { + mocks.fetch.mockResolvedValue( + new Response(JSON.stringify({ error: { status: 'RESOURCE_EXHAUSTED' } }), { status: 403 }) + ) + const client = createNativeClient({ + origin: 'https://slack.com', + accessToken: 'private', + signal: new AbortController().signal, + }) + await expect(client.json('/api/search.messages')).rejects.toMatchObject({ status: 'reconnect' }) + }) }) diff --git a/apps/sim/lib/sim-search/live/http.ts b/apps/sim/lib/sim-search/live/http.ts index e3f8789973e..b284f283ba7 100644 --- a/apps/sim/lib/sim-search/live/http.ts +++ b/apps/sim/lib/sim-search/live/http.ts @@ -115,8 +115,9 @@ export function createNativeClient(input: { } /** - * Answers a repeated `memo` GET from the client's earlier response. A client lives for one - * search or read, so a memoized response never outlives the request that authorized it. + * Answers a repeated `memo` GET from the client's earlier response, including a failure, so a + * rate-limited endpoint is not requested again. A client lives for one search or read, so a + * memoized response never outlives the request that authorized it. */ export function withJsonMemo(client: NativeClient): NativeClient { const memo = new Map>() @@ -130,7 +131,6 @@ export function withJsonMemo(client: NativeClient): NativeClient { if (!pending) { pending = client.json(...request) memo.set(key, pending) - pending.catch(() => memo.delete(key)) } return pending }, diff --git a/apps/sim/lib/sim-search/live/pages.ts b/apps/sim/lib/sim-search/live/pages.ts index 539feec0406..a2f30959c16 100644 --- a/apps/sim/lib/sim-search/live/pages.ts +++ b/apps/sim/lib/sim-search/live/pages.ts @@ -39,7 +39,7 @@ export async function collectNativePages( hasMore: pages.some((page) => page.hasMore || page.nextCursor), message: joinMessages([ guidance, - ...pages.filter((page) => page.partial).map((page) => page.message), + ...pages.filter((page) => page.partial || page.hasMore).map((page) => page.message), ...failures.map((error) => error instanceof NativeSearchError ? error.message : 'One collection could not be searched.' ), diff --git a/apps/sim/lib/sim-search/live/policy.test.ts b/apps/sim/lib/sim-search/live/policy.test.ts index 7aa5b44f554..41eee09e123 100644 --- a/apps/sim/lib/sim-search/live/policy.test.ts +++ b/apps/sim/lib/sim-search/live/policy.test.ts @@ -1,5 +1,6 @@ /** @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' +import { withJsonMemo } from '@/lib/sim-search/live/http' import { createPolicyVerifier } from '@/lib/sim-search/live/policy' import { defaultLiveSearchPolicy, @@ -243,4 +244,19 @@ describe('organization search scope enforcement', () => { expect(await verify({ id: 'https://coda.io:444/d/doc' })).toBe(false) expect(await verify({ id: 'https://evil.test/d/doc' })).toBe(false) }) + it('shares metadata within a client unless the verifier must read it fresh', async () => { + const api = client({ + '/drive/v3/files/doc': { id: 'doc', parents: ['root'] }, + '/drive/v3/files/root': { id: 'root', parents: [] }, + }) + const session = withJsonMemo(api) + const verify = createPolicyVerifier('google_drive', selected(['root']), session, '') + await verify({ id: 'doc' }) + await verify({ id: 'doc' }) + expect(api.json).toHaveBeenCalledTimes(2) + await createPolicyVerifier('google_drive', selected(['root']), session, '', undefined, { + fresh: true, + })({ id: 'doc' }) + expect(api.json).toHaveBeenCalledTimes(4) + }) }) diff --git a/apps/sim/lib/sim-search/live/policy.ts b/apps/sim/lib/sim-search/live/policy.ts index d2e41252059..43ec5a3c339 100644 --- a/apps/sim/lib/sim-search/live/policy.ts +++ b/apps/sim/lib/sim-search/live/policy.ts @@ -39,13 +39,15 @@ export function createPolicyVerifier( policy: LiveSearchPolicy, client: NativeClient | null, origin: string, - mcp?: CodaMcpClient + mcp?: CodaMcpClient, + /** A fresh verifier reads provider metadata again instead of reusing this client's responses. */ + options: { fresh?: boolean } = {} ): PolicyVerifier { if (!requiresScopedRetrieval(provider, policy)) return async () => true const json = (path: string, query?: Record) => { if (!client) throw new NativeSearchError('unavailable', 'This connection cannot verify the search scope.') - return client.json(path, { query, memo: true }) + return client.json(path, { query, memo: !options.fresh }) } const restricted = policy.mode === 'selected' || policy.excluded.length > 0 const siteAllowed = (value: string) => { diff --git a/apps/sim/lib/sim-search/live/providers.test.ts b/apps/sim/lib/sim-search/live/providers.test.ts index 63047f8a8b8..e12d7d9af4e 100644 --- a/apps/sim/lib/sim-search/live/providers.test.ts +++ b/apps/sim/lib/sim-search/live/providers.test.ts @@ -233,6 +233,51 @@ describe('native search endpoints', () => { expect(result.partial).toBe(true) expect(result.message).toMatch(/Only the \d+ most recently pushed repositories were searched/) }) + it('interleaves GitHub kinds before batches so one kind cannot crowd out another', async () => { + const api = client() + const repositories = Array.from({ length: 100 }, (_, index) => ({ + full_name: `simstudioai/repository-with-a-long-name-${index}`, + })) + api.json.mockImplementation(async (path) => { + if (path === '/user/repos') return repositories + return path === '/search/code' + ? { items: [{ path: 'src/a.ts', repository: { full_name: 'org/repo' } }], total_count: 1 } + : { items: [{ number: 1, title: 'Issue' }], total_count: 1 } + }) + const result = await searchGitHub(api, input) + const count = (path: string) => api.json.mock.calls.filter(([called]) => called === path).length + expect(count('/search/code')).toBeGreaterThan(count('/search/issues') / 2) + expect(result.documents.slice(0, 2).map(({ kind }) => kind)).toEqual(['issues', 'code']) + }) + it('batches GitHub repository search like issue search, not by the code query limit', async () => { + const api = client() + const repositories = Array.from({ length: 100 }, (_, index) => ({ + full_name: `simstudioai/repository-with-a-long-name-${index}`, + })) + api.json.mockImplementation(async (path) => + path === '/user/repos' ? repositories : { items: [], total_count: 0 } + ) + await searchGitHub(api, { + ...input, + native: { provider: 'github', query: 'launch', kind: 'repositories' }, + }) + const searches = api.json.mock.calls.filter(([path]) => path === '/search/repositories') + expect(searches.length).toBeGreaterThan(0) + expect(searches.length).toBeLessThanOrEqual(2) + }) + it('skips code search when the query leaves no room for a repository qualifier', async () => { + const api = client() + api.json.mockImplementation(async (path) => + path === '/user/repos' ? [{ full_name: 'org/repo' }] : { items: [], total_count: 0 } + ) + const result = await searchGitHub(api, { ...input, query: 'x'.repeat(975) }) + const searches = api.json.mock.calls.filter(([path]) => path.startsWith('/search/')) + expect(searches.map(([path]) => path)).toEqual(['/search/issues', '/search/issues']) + expect(result).toMatchObject({ + partial: true, + message: expect.stringContaining('code search was skipped'), + }) + }) it('keeps GitHub qualifiers outside the grouped text of a dated search', async () => { const api = client() api.json.mockImplementation(async (path) => @@ -456,6 +501,27 @@ describe('native search endpoints', () => { ).toHaveLength(3) expect(api.json.mock.calls.some(([path]) => path === '/api/v4/search')).toBe(false) }) + it('reads GitLab merge request assignees from the deprecated single assignee', async () => { + const api = client() + const search = (row: Record) => { + api.json.mockResolvedValueOnce([ + { + iid: 7, + project_id: 4, + web_url: 'https://gitlab.com/org/repo/-/merge_requests/7', + ...row, + }, + ]) + return searchGitLab(api, { + ...input, + native: { provider: 'gitlab', query: 'launch', kind: 'merge_requests', project: '4' }, + }) + } + expect(await search({ assignee: { id: 3 } })).toMatchObject({ + documents: [{ accessMetadata: { assigneeIds: [3] } }], + }) + expect(await search({})).toMatchObject({ documents: [{ accessMetadata: { assigneeIds: [] } }] }) + }) it('binds GitLab code references and links to the searched revision', async () => { const api = client() api.json diff --git a/apps/sim/lib/sim-search/live/service-session.test.ts b/apps/sim/lib/sim-search/live/service-session.test.ts index d752a131f30..8a131c5ae85 100644 --- a/apps/sim/lib/sim-search/live/service-session.test.ts +++ b/apps/sim/lib/sim-search/live/service-session.test.ts @@ -148,6 +148,16 @@ describe('service credential isolation', () => { createLiveServiceSession({ ...base, member: null, provider: 'github', policy: configured }) ).rejects.toThrow('personal GitHub account') }) + it('passes the caller connection pool to the GitHub verifier', async () => { + const pool = { agent: vi.fn(), destroy: vi.fn() } + await createLiveServiceSession({ + ...base, + provider: 'github', + policy: { ...policy, sourceId: undefined }, + pool, + }) + expect(mocks.github).toHaveBeenCalledWith(expect.anything(), api, base.signal, pool) + }) }) describe('Confluence source namespace', () => { diff --git a/apps/sim/lib/sim-search/live/text.test.ts b/apps/sim/lib/sim-search/live/text.test.ts index e203e0e16e7..f22dff4287f 100644 --- a/apps/sim/lib/sim-search/live/text.test.ts +++ b/apps/sim/lib/sim-search/live/text.test.ts @@ -37,4 +37,30 @@ describe('providerText', () => { it('does not decode entities in plain text', () => { expect(providerText('Use <b> for bold')).toBe('Use <b> for bold') }) + it('keeps each table row on one line', () => { + expect(providerText('
Widget2
', 'html')).toContain( + 'Widget 2' + ) + }) + it('separates a definition term from its definition', () => { + expect(providerText('
Term
Def
', 'html')).toMatch(/Term\s+Def/) + }) + it('does not throw on deeply nested markup', () => { + expect(() => providerText(`${''.repeat(20000)}x`, 'html')).not.toThrow() + }) + it.each(['plain', 'escaped'] as const)( + 'cleans a long run of spaces in linear time (%s)', + (format) => { + const started = performance.now() + expect(providerText(`${' '.repeat(1_000_000)}x`, format)).toContain('x') + expect(performance.now() - started).toBeLessThan(1000) + } + ) + it('keeps a zero-width non-joiner that shapes a Persian word', () => { + const word = String.fromCodePoint(0x0645, 0x06cc, 0x200c, 0x062e, 0x0648, 0x0627, 0x0645) + expect(providerText(word)).toBe(word) + }) + it('removes a stray byte-order mark inside a word without adding a space', () => { + expect(providerText(`doc${String.fromCodePoint(0xfeff)}ument`)).toBe('document') + }) }) diff --git a/apps/sim/lib/sim-search/live/text.ts b/apps/sim/lib/sim-search/live/text.ts index 2c21b4ac158..2fa4ebfa691 100644 --- a/apps/sim/lib/sim-search/live/text.ts +++ b/apps/sim/lib/sim-search/live/text.ts @@ -1,5 +1,5 @@ import { convert, type HtmlToTextOptions } from 'html-to-text' -import { decodeHtmlEntities, looksLikeHtml } from '@/connectors/utils' +import { decodeHtmlEntities, htmlToPlainText, looksLikeHtml } from '@/connectors/utils' /** * How a provider encodes a text field: `plain` is used as-is, `escaped` is HTML-escaped text @@ -15,17 +15,46 @@ type ProviderTextFormat = 'plain' | 'escaped' | 'html' | 'auto' */ const INVISIBLE_CHARACTER = '[\\u00AD\\u061C\\u115F\\u1160\\u17B4\\u17B5\\u180E\\u200B\\u200C\\u200E\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\u3164\\uFEFF\\uFFA0]|\\u034F|\\u200D(?!\\p{Extended_Pictographic})' -/** A run of invisible characters with the spaces between them, as preheader padding is built. */ -const INVISIBLE_RUN = new RegExp(`(?:[^\\S\\n]*(?:${INVISIBLE_CHARACTER}))+[^\\S\\n]*`, 'gu') +/** + * A run of invisible characters with the spaces between them, as preheader padding is built. + * The lookbehind starts a match only at the beginning of a space run, which keeps long runs of + * spaces linear instead of rescanning them from every position. + */ +const INVISIBLE_RUN = new RegExp( + `(? ({ selector, options: { uppercase: false }, @@ -33,6 +62,15 @@ const HTML_TO_TEXT: HtmlToTextOptions = { ], } +/** Markup the converter cannot walk keeps its text through the flat tag-stripping path. */ +function markupText(html: string): string { + try { + return convert(html, HTML_TO_TEXT) + } catch { + return htmlToPlainText(html) + } +} + /** * Normalizes provider text into readable plain text for previews and document reads. Markup is * never rendered: tags are dropped and entities decoded into literal characters. @@ -40,15 +78,17 @@ const HTML_TO_TEXT: HtmlToTextOptions = { export function providerText(value: string, format: ProviderTextFormat = 'plain'): string { let text = value if (format === 'html' || (format === 'auto' && looksLikeHtml(text))) { - text = convert(text, HTML_TO_TEXT) + text = markupText(text) } else if (format === 'escaped') { text = decodeHtmlEntities(text) } - /** Padding collapses to one space, or to nothing inside a word; other spacing is kept. */ - text = text.replace(INVISIBLE_RUN, (run) => (/\s/.test(run) ? ' ' : '')) + /** Padding collapses to one space; a stray mark inside a word is dropped unless it shapes it. */ + text = text.replace(INVISIBLE_RUN, (run) => + SPACING.test(run) ? ' ' : MEANINGFUL_MARK.test(run) ? run : '' + ) if (format === 'escaped') return text.replace(/\s+/g, ' ').trim() return text - .replace(/[^\S\n]+$/gm, '') + .replace(TRAILING_SPACES, '') .replace(/\n{3,}/g, '\n\n') .replace(/^\n+|\n+$/g, '') } diff --git a/apps/sim/lib/slack-search/assistant-stream.ts b/apps/sim/lib/slack-search/assistant-stream.ts index 06f3125b206..66444866c1d 100644 --- a/apps/sim/lib/slack-search/assistant-stream.ts +++ b/apps/sim/lib/slack-search/assistant-stream.ts @@ -10,6 +10,7 @@ import { parseCitationRecord, type RetrievalCitationBlock, } from '@/lib/mothership/chat/citation-evidence' +import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' import { redactSensitiveContent } from '@/lib/mothership/chat/sim-key-redaction' import type { StreamEvent, @@ -35,10 +36,7 @@ export function publicSlackAnswer( sources: ReadonlyMap = new Map(), integrationsUrl?: string ): string { - let value = text.replace( - /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?(?:<\/\1>|$)/g, - '' - ) + let value = stripInteractiveTags(text) if (!complete) { let end = Math.max(value.lastIndexOf(' '), value.lastIndexOf('\n')) + 1 const sourceStart = value.lastIndexOf('<') From 0a4813b37aac3c9fbf95b879f53d9c8348e85c1c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 22:16:14 -0700 Subject: [PATCH 4/6] fix(search): address review round on status, read budget, GitHub queries, and answer tags - Report any page with more matches no cursor can reach as partial, and an empty continuable page as partial - Keep live reads on the 8,000-character page budget indexed reads use - Leave boolean GitHub queries as written and explain the 256-character search text limit - Strip only closed interactive tags from complete MCP answers so prose after an unclosed opener survives - Skip the wire-size Content-Length precheck for encoded bodies; the decoded stream stays capped - Build the meaningful-mark pattern without a joiner inside a character class --- .../core/security/input-validation.server.ts | 8 ++++- ...ecure-fetch-connection-pool.server.test.ts | 18 +++++++++++ apps/sim/lib/knowledge/application/chat.ts | 2 +- .../mothership/chat/interactive-tags.test.ts | 19 ++++++++++++ .../lib/mothership/chat/interactive-tags.ts | 17 +++++++---- .../lib/sim-search/live/application.test.ts | 14 ++++++--- apps/sim/lib/sim-search/live/application.ts | 30 ++++++++----------- apps/sim/lib/sim-search/live/github.ts | 25 ++++++++++++---- .../sim/lib/sim-search/live/providers.test.ts | 28 ++++++++++++++++- apps/sim/lib/sim-search/live/text.ts | 4 +-- apps/sim/lib/slack-search/assistant-stream.ts | 2 +- 11 files changed, 129 insertions(+), 38 deletions(-) create mode 100644 apps/sim/lib/mothership/chat/interactive-tags.test.ts diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 6e8bc5289b6..0bf028fa4ba 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -1326,7 +1326,13 @@ export async function secureFetchWithPinnedIP( statusCode === 204 || statusCode === 205 || statusCode === 304 - const contentLength = headersRecord['content-length'] + /** + * An encoded body's Content-Length is its wire size, not the decoded size the cap bounds, + * so only an identity body is rejected up front; the decoded stream is capped as it reads. + */ + const contentLength = headersRecord['content-encoding'] + ? undefined + : headersRecord['content-length'] if (contentLength && !isBodylessResponse) { const parsedLength = Number.parseInt(contentLength, 10) if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) { diff --git a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts index 482a1ed64d4..adfdd76d20b 100644 --- a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts @@ -101,6 +101,24 @@ describe('secureFetchWithPinnedIP connection reuse', () => { }) describe('secureFetchWithPinnedIP compressed responses', () => { + it('does not reject an encoded body by its wire size when the decoded body fits', async () => { + const payload = Buffer.from(JSON.stringify({ ok: true })) + const encoded = gzipSync(payload) + const server = await startServer((_req, res) => { + res.writeHead(200, { + 'Content-Encoding': 'gzip', + 'Content-Length': String(encoded.length), + }) + res.end(encoded) + }) + const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', { + profile: 'configuredEndpoint', + acceptCompressed: true, + maxResponseBytes: payload.length, + }) + expect(encoded.length).toBeGreaterThan(payload.length) + await expect(response.json()).resolves.toEqual({ ok: true }) + }) it('caps the decoded size of a compressed body it asked for', async () => { const bomb = gzipSync(Buffer.alloc(64 * 1024, 0x41)) const server = await startServer((_req, res) => { diff --git a/apps/sim/lib/knowledge/application/chat.ts b/apps/sim/lib/knowledge/application/chat.ts index 53778efa041..b6fb5c2c56e 100644 --- a/apps/sim/lib/knowledge/application/chat.ts +++ b/apps/sim/lib/knowledge/application/chat.ts @@ -204,7 +204,7 @@ export const organizationSearchChat: OperationUseCase< ) /** MCP clients receive text; interactive Chat cards stay only in the saved transcript. */ const answer = resolveSearchChatCitations( - stripInteractiveTags(assistantMessage.content), + stripInteractiveTags(assistantMessage.content, { complete: true }), result.toolCalls ) if (!answer.content.trim()) diff --git a/apps/sim/lib/mothership/chat/interactive-tags.test.ts b/apps/sim/lib/mothership/chat/interactive-tags.test.ts new file mode 100644 index 00000000000..41a54600cd5 --- /dev/null +++ b/apps/sim/lib/mothership/chat/interactive-tags.test.ts @@ -0,0 +1,19 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' + +describe('stripInteractiveTags', () => { + it('removes closed interactive tags', () => { + expect(stripInteractiveTags('Answer {"a":1} end', { complete: true })).toBe( + 'Answer end' + ) + }) + it('keeps prose after an unclosed opener in a complete answer', () => { + expect(stripInteractiveTags('Use to ask a follow-up.', { complete: true })).toBe( + 'Use to ask a follow-up.' + ) + }) + it('withholds a tag still open at the end of a streaming answer', () => { + expect(stripInteractiveTags('Answer {"a"', { complete: false })).toBe('Answer ') + }) +}) diff --git a/apps/sim/lib/mothership/chat/interactive-tags.ts b/apps/sim/lib/mothership/chat/interactive-tags.ts index b149eaaf7c5..959b3b30a80 100644 --- a/apps/sim/lib/mothership/chat/interactive-tags.ts +++ b/apps/sim/lib/mothership/chat/interactive-tags.ts @@ -1,8 +1,15 @@ /** Answer tags the Chat UI renders as interactive cards; text-only surfaces drop them. */ -const INTERACTIVE_TAGS = - /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?(?:<\/\1>|$)/g +const CLOSED_TAGS = + /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?<\/\1>/g +/** An interactive tag still open at the end of a partial stream. */ +const TRAILING_OPEN_TAG = + /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>(?![\s\S]*<\/\1>)[\s\S]*$/ -/** Removes interactive Chat UI tags, including one still open at the end of a partial stream. */ -export function stripInteractiveTags(text: string): string { - return text.replace(INTERACTIVE_TAGS, '') +/** + * Removes interactive Chat UI tags. A streaming answer also withholds a tag that has not closed + * yet; a complete answer keeps text after an unclosed opener, which is prose rather than a card. + */ +export function stripInteractiveTags(text: string, options: { complete: boolean }): string { + const closed = text.replace(CLOSED_TAGS, '') + return options.complete ? closed : closed.replace(TRAILING_OPEN_TAG, '') } diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 6dbeab6a755..407e0ed791d 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -321,7 +321,13 @@ describe('authorized live retrieval', () => { const order = (data: typeof result) => data.results.map((row) => row.connectorType) expect(order(reversed)).toEqual(order(result)) }) - it('scales the read window with the requested chunk limit', async () => { + it('reports partial coverage when more matches exist that no cursor reaches', async () => { + mocks.search.mockResolvedValue({ documents: [document], hasMore: true }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results).toHaveLength(1) + expect(result.live?.accounts[0]).toMatchObject({ status: 'partial' }) + }) + it('keeps the read page budget of indexed reads at any chunk limit', async () => { mocks.read.mockResolvedValue({ ...document, content: 'x'.repeat(30_000) }) const search = await searchLiveKnowledge.execute({ principal, input }) const read = (limit: number) => @@ -335,7 +341,7 @@ describe('authorized live retrieval', () => { }, }) expect((await read(3)).chunks[0]?.content).toHaveLength(8000) - expect((await read(8)).chunks[0]?.content).toHaveLength(24_000) + expect((await read(8)).chunks[0]?.content).toHaveLength(8000) }) it('centers the preview on the query match', async () => { mocks.search.mockResolvedValue({ @@ -436,7 +442,7 @@ describe('authorized live retrieval', () => { expect(result.results).toHaveLength(1) expect(result.live?.accounts[0]).toMatchObject({ status: 'partial', - message: expect.stringContaining('date range'), + message: expect.stringContaining('could return'), }) }) it('applies date filters before spending provider verification', async () => { @@ -477,7 +483,7 @@ describe('authorized live retrieval', () => { mocks.search.mockResolvedValue({ documents: [], hasMore: true }) const result = await searchLiveKnowledge.execute({ principal, input }) expect(result.retrieval.status).toBe('partial') - expect(result.live?.accounts[0]?.message).toContain('More matches may exist') + expect(result.live?.accounts[0]?.message).toContain('could return') }) it('reports partial coverage when a continuable page returned nothing readable', async () => { mocks.search.mockResolvedValue({ documents: [], nextCursor: 'next' }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 0da9ea55dd7..953c028f7f7 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -208,10 +208,7 @@ const VERIFY_CONCURRENCY = 5 const MAX_ACCOUNTS = 20 /** Previews center on the query's longest matching term, like indexed passages. */ const PREVIEW_CHARACTERS = 1800 -/** - * Characters a read returns per three requested chunks. `limit` counts index chunks elsewhere, - * so the default of three keeps one window and the maximum of eight returns three. - */ +/** Characters one read returns, the same page budget indexed document reads use. */ const READ_WINDOW_CHARACTERS = 8000 /** @@ -383,10 +380,10 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ const undatedUnsorted = dateSorted && matching.some(({ document }) => !sourceDate(document, account.provider)) const moreUnsorted = dateSorted && Boolean(page.nextCursor || page.hasMore) - /** Nothing readable came back, yet the provider has more: an empty page proves nothing. */ - const moreWithoutResults = Boolean((page.hasMore || page.nextCursor) && !matching.length) - /** A dated listing should cover its window; uncontinuable extra matches leave it short. */ - const moreInDateRange = Boolean(hasDateBounds(filters) && page.hasMore && !page.nextCursor) + /** More matches exist that no cursor can reach, so coverage is short. */ + const moreUnreachable = Boolean(page.hasMore && !page.nextCursor) + /** A continuable page with nothing readable proves nothing about the pages after it. */ + const emptyContinuable = Boolean(page.nextCursor && !matching.length) const degraded = unverified || session.servicePartial || @@ -394,8 +391,8 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ undatedExcluded || undatedUnsorted || moreUnsorted || - moreWithoutResults || - moreInDateRange + moreUnreachable || + emptyContinuable return { status: { ...status, @@ -414,11 +411,11 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ dateSorted ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.' : undefined, - moreWithoutResults - ? 'More matches may exist beyond this page. Continue with nextCursor when present, narrow the query, or target one source.' + moreUnreachable + ? 'More matches exist than this search could return. Narrow the query or target one source.' : undefined, - moreInDateRange && !moreWithoutResults - ? 'More matches exist in this date range than were returned. Narrow the range or target one source.' + emptyContinuable + ? 'No readable matches on this page. Continue with nextCursor for more.' : undefined, ]), nextCursor: page.nextCursor, @@ -608,10 +605,7 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ 'validation', 'This document changed. Read again from the beginning' ) - const end = Math.min( - content.length, - start + READ_WINDOW_CHARACTERS * Math.ceil(input.limit / 3) - ) + const end = Math.min(content.length, start + READ_WINDOW_CHARACTERS) return { documentId: input.documentId, knowledgeBaseId: '', diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts index 1dc73a365d4..e1968ec42fd 100644 --- a/apps/sim/lib/sim-search/live/github.ts +++ b/apps/sim/lib/sim-search/live/github.ts @@ -101,19 +101,29 @@ function repositoryBatches(query: string, names: readonly string[], maxBytes: nu /** `key:value` or `key:"quoted value"`, excluding URLs such as `https://…`. */ const GITHUB_QUALIFIER = /^-?[a-z][\w-]*:(?!\/\/)\S/i +/** Search terms, quoted phrases, and `key:"quoted value"` qualifiers as GitHub tokenizes them. */ +const githubTokens = (query: string) => query.match(/-?[\w-]+:"[^"]*"|-?"[^"]*"|\S+/g) ?? [] + /** - * Groups free text so boolean operators cannot absorb appended qualifiers. GitHub treats a - * qualifier inside parentheses as search text, so qualifiers stay outside the group. A query - * that already uses parentheses is structured by its author and is left as written. + * Groups free text so appended qualifiers apply to all of it. GitHub treats a qualifier inside + * parentheses as search text, so qualifiers stay outside the group. A query that already uses + * parentheses or boolean operators is structured by its author and is left as written. */ function groupGitHubText(query: string): string { - if (!query || /[()]/.test(query)) return query - const tokens = query.match(/-?[\w-]+:"[^"]*"|-?"[^"]*"|\S+/g) ?? [] + if (!query || /[()]/.test(query) || /(?:^|\s)(?:AND|OR|NOT)(?=\s|$)/.test(query)) return query + const tokens = githubTokens(query) const qualifiers = tokens.filter((token) => GITHUB_QUALIFIER.test(token)) const text = tokens.filter((token) => !GITHUB_QUALIFIER.test(token)).join(' ') return [text ? `(${text})` : '', ...qualifiers].filter(Boolean).join(' ') } +/** GitHub rejects more than 256 characters of search text; qualifiers do not count toward it. */ +const GITHUB_TEXT_CHARACTERS = 256 +const githubTextLength = (query: string) => + githubTokens(query) + .filter((token) => !GITHUB_QUALIFIER.test(token)) + .join(' ').length + export async function searchGitHub( client: NativeClient, input: NativeSearchInput @@ -241,6 +251,11 @@ export async function searchGitHub( : '' const datedQuery = kind === 'issues' ? [groupGitHubText(text), updated].filter(Boolean).join(' ') : text + if (githubTextLength(text) > GITHUB_TEXT_CHARACTERS) + throw new NativeSearchError( + 'unavailable', + 'GitHub search text is limited to 256 characters. Shorten the query.' + ) const page = input.native?.cursor ?? '1' if (!/^\d{1,3}$/.test(page) || Number(page) < 1) throw new NativeSearchError('unavailable', 'Invalid GitHub page.') diff --git a/apps/sim/lib/sim-search/live/providers.test.ts b/apps/sim/lib/sim-search/live/providers.test.ts index e12d7d9af4e..73aa4b46600 100644 --- a/apps/sim/lib/sim-search/live/providers.test.ts +++ b/apps/sim/lib/sim-search/live/providers.test.ts @@ -270,7 +270,7 @@ describe('native search endpoints', () => { api.json.mockImplementation(async (path) => path === '/user/repos' ? [{ full_name: 'org/repo' }] : { items: [], total_count: 0 } ) - const result = await searchGitHub(api, { ...input, query: 'x'.repeat(975) }) + const result = await searchGitHub(api, { ...input, query: `launch label:"${'x'.repeat(960)}"` }) const searches = api.json.mock.calls.filter(([path]) => path.startsWith('/search/')) expect(searches.map(([path]) => path)).toEqual(['/search/issues', '/search/issues']) expect(result).toMatchObject({ @@ -278,6 +278,32 @@ describe('native search endpoints', () => { message: expect.stringContaining('code search was skipped'), }) }) + it('leaves a boolean GitHub query as written when dates are appended', async () => { + const api = client() + api.json.mockResolvedValue({ items: [], total_count: 0 }) + await searchGitHub(api, { + ...input, + native: { + provider: 'github', + query: 'repo:org/repo is:issue label:bug OR label:feature', + kind: 'issues', + }, + filters: { startDate: '2026-09-20T00:00:00Z' }, + }) + expect(api.json.mock.calls[0][1]?.query?.q).toBe( + 'repo:org/repo is:issue label:bug OR label:feature updated:>=2026-09-20T00:00:00.000Z' + ) + }) + it("explains GitHub's search text limit instead of sending an oversized query", async () => { + const api = client() + await expect( + searchGitHub(api, { + ...input, + native: { provider: 'github', query: `repo:org/repo ${'word '.repeat(60)}`, kind: 'code' }, + }) + ).rejects.toThrow('limited to 256 characters') + expect(api.json).not.toHaveBeenCalled() + }) it('keeps GitHub qualifiers outside the grouped text of a dated search', async () => { const api = client() api.json.mockImplementation(async (path) => diff --git a/apps/sim/lib/sim-search/live/text.ts b/apps/sim/lib/sim-search/live/text.ts index 2fa4ebfa691..049da163f93 100644 --- a/apps/sim/lib/sim-search/live/text.ts +++ b/apps/sim/lib/sim-search/live/text.ts @@ -27,9 +27,9 @@ const INVISIBLE_RUN = new RegExp( /** Trailing spaces on a line, matched only from the start of their run for the same reason. */ const TRAILING_SPACES = /(? = new Map(), integrationsUrl?: string ): string { - let value = stripInteractiveTags(text) + let value = stripInteractiveTags(text, { complete: false }) if (!complete) { let end = Math.max(value.lastIndexOf(' '), value.lastIndexOf('\n')) + 1 const sourceStart = value.lastIndexOf('<') From 45522d549719fd34e327421841e40e6e74d352db Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 22:30:16 -0700 Subject: [PATCH 5/6] fix(search): strip only JSON cards from MCP answers and keep tag-shaped prose - Remove interactive cards by their JSON payload so closing markers inside strings cannot end a card and tag-shaped prose stays - Keep the Slack stream's existing tag withholding unchanged - Require valid element syntax in looksLikeHtml so an address like stays text - Keep verified results when verification is rate-limited and name the rate limit --- apps/sim/connectors/utils.test.ts | 12 ++++++++++ apps/sim/connectors/utils.ts | 4 ++-- .../application/chat-citations.test.ts | 21 ++++++++++++++++++ .../knowledge/application/chat-citations.ts | 22 +++++++++++++++++++ .../lib/knowledge/application/chat.test.ts | 3 ++- apps/sim/lib/knowledge/application/chat.ts | 4 ++-- .../mothership/chat/interactive-tags.test.ts | 19 ---------------- .../lib/mothership/chat/interactive-tags.ts | 15 ------------- .../lib/sim-search/live/application.test.ts | 19 ++++++++++++++++ apps/sim/lib/sim-search/live/application.ts | 13 +++++++---- apps/sim/lib/sim-search/live/text.test.ts | 3 +++ apps/sim/lib/slack-search/assistant-stream.ts | 6 +++-- 12 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 apps/sim/lib/knowledge/application/chat-citations.test.ts delete mode 100644 apps/sim/lib/mothership/chat/interactive-tags.test.ts delete mode 100644 apps/sim/lib/mothership/chat/interactive-tags.ts diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index b8640d29a75..4325ac4fde4 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -72,6 +72,7 @@ import { isIndexableConnectorFile, isSkippableMicrosoftGraphFolderError, isSkippedDocument, + looksLikeHtml, MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES, MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, @@ -1754,3 +1755,14 @@ describe('BoundedLines', () => { }) }) }) + +describe('looksLikeHtml', () => { + it('recognizes elements with valid tag syntax', () => { + for (const markup of ['

', '

', '
', '
', '', '']) + expect(looksLikeHtml(`text ${markup} text`)).toBe(true) + }) + it('does not mistake an address whose name starts like a tag for markup', () => { + for (const text of ['Invite ', 'From ', '']) + expect(looksLikeHtml(text)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index a85d26048e2..7688e920b1c 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -339,11 +339,11 @@ function decodeCharacterReference(raw: string, code: number): string { * A false positive therefore does not merely pass text through untouched — it * deletes the bracketed span and flattens the document's line structure. Plain * text routinely contains angle brackets that are not markup: an email address - * (`Reply from John `), a markdown autolink + * (`Reply from John `, or ``, whose name is a tag's), a markdown autolink * (``), or a placeholder (``). */ const HTML_TAG_PATTERN = - /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)\b[^>]*>/i + /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)(?=[\s/>])[^>]*>/i /** * Reports whether a value carries real HTML markup and is therefore worth routing diff --git a/apps/sim/lib/knowledge/application/chat-citations.test.ts b/apps/sim/lib/knowledge/application/chat-citations.test.ts new file mode 100644 index 00000000000..35068980229 --- /dev/null +++ b/apps/sim/lib/knowledge/application/chat-citations.test.ts @@ -0,0 +1,21 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { stripInteractiveCards } from '@/lib/knowledge/application/chat-citations' + +describe('stripInteractiveCards', () => { + it('removes a JSON card, including closing markers inside its strings', () => { + expect( + stripInteractiveCards('Answer {"1":{"title":"Say x"}} end') + ).toBe('Answer end') + }) + it('removes thinking blocks', () => { + expect(stripInteractiveCards('planAnswer')).toBe('Answer') + }) + it('keeps tag-shaped prose', () => { + const prose = 'Wrap choices in your list tags, or use to ask.' + expect(stripInteractiveCards(prose)).toBe(prose) + }) + it('drops a card left open before its JSON payload', () => { + expect(stripInteractiveCards('Connect here {"type":"link"')).toBe('Connect here ') + }) +}) diff --git a/apps/sim/lib/knowledge/application/chat-citations.ts b/apps/sim/lib/knowledge/application/chat-citations.ts index 3e72158c715..9dba5bb846b 100644 --- a/apps/sim/lib/knowledge/application/chat-citations.ts +++ b/apps/sim/lib/knowledge/application/chat-citations.ts @@ -8,6 +8,28 @@ export interface SearchChatCitation { url: string } +/** Cards the Chat UI renders from a JSON payload; an MCP text answer drops them. */ +const CARD_TAGS = 'options|question|usage_upgrade|credential|workspace_resource' +/** A JSON string owns its escaped quotes and any tag-shaped text inside it. */ +const JSON_STRING = '"(?:\\\\.|[^"\\\\\\r\\n])*"' +/** + * A card whose body is a JSON value, a thinking block, or a card opener left unclosed before a + * JSON payload. A tag-shaped pair or opener in ordinary prose is not a card and stays. + */ +const INTERACTIVE_CARD = new RegExp( + [ + `<(${CARD_TAGS})>\\s*[[{](?:${JSON_STRING}|[^"<])*?[\\]}]\\s*`, + '[\\s\\S]*?', + `<(?:${CARD_TAGS})>\\s*[[{][\\s\\S]*$`, + ].join('|'), + 'g' +) + +/** Removes interactive Chat cards so a text-only surface receives only the answer prose. */ +export function stripInteractiveCards(content: string): string { + return content.replace(INTERACTIVE_CARD, '') +} + /** Resolves Assistant source tags only against successful, bounded retrieval evidence. */ export function resolveSearchChatCitations(content: string, toolCalls: ToolCallSummary[]) { const evidence = new Map() diff --git a/apps/sim/lib/knowledge/application/chat.test.ts b/apps/sim/lib/knowledge/application/chat.test.ts index d14f5084ab3..40936b631f6 100644 --- a/apps/sim/lib/knowledge/application/chat.test.ts +++ b/apps/sim/lib/knowledge/application/chat.test.ts @@ -304,7 +304,8 @@ describe('organization Search Assistant chat', () => { }) it('drops interactive Chat tags from the MCP answer but keeps them in the transcript', async () => { - const tags = '{"1":"Open it"}Which kit?' + const tags = + '{"1":"Open it"}{"prompt":"Which kit?"}' mocks.lifecycle.mockResolvedValue(createResult({ content: `Violet suitcase.${tags}` })) const result = await execute() expect(result.content).toBe('Violet suitcase.') diff --git a/apps/sim/lib/knowledge/application/chat.ts b/apps/sim/lib/knowledge/application/chat.ts index b6fb5c2c56e..9d4d85fbdf5 100644 --- a/apps/sim/lib/knowledge/application/chat.ts +++ b/apps/sim/lib/knowledge/application/chat.ts @@ -19,10 +19,10 @@ import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/avail import { resolveSearchChatCitations, type SearchChatCitation, + stripInteractiveCards, } from '@/lib/knowledge/application/chat-citations' import { organizationSearchChatOperation } from '@/lib/knowledge/application/chat-operations' import { loadCopilotSearchIntegrations } from '@/lib/mothership/application/load-search-integrations' -import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' import { persistCopilotChatTurn } from '@/lib/mothership/chat/messages-store' import { buildCopilotRequestPayload } from '@/lib/mothership/chat/payload' import { @@ -204,7 +204,7 @@ export const organizationSearchChat: OperationUseCase< ) /** MCP clients receive text; interactive Chat cards stay only in the saved transcript. */ const answer = resolveSearchChatCitations( - stripInteractiveTags(assistantMessage.content, { complete: true }), + stripInteractiveCards(assistantMessage.content), result.toolCalls ) if (!answer.content.trim()) diff --git a/apps/sim/lib/mothership/chat/interactive-tags.test.ts b/apps/sim/lib/mothership/chat/interactive-tags.test.ts deleted file mode 100644 index 41a54600cd5..00000000000 --- a/apps/sim/lib/mothership/chat/interactive-tags.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' - -describe('stripInteractiveTags', () => { - it('removes closed interactive tags', () => { - expect(stripInteractiveTags('Answer {"a":1} end', { complete: true })).toBe( - 'Answer end' - ) - }) - it('keeps prose after an unclosed opener in a complete answer', () => { - expect(stripInteractiveTags('Use to ask a follow-up.', { complete: true })).toBe( - 'Use to ask a follow-up.' - ) - }) - it('withholds a tag still open at the end of a streaming answer', () => { - expect(stripInteractiveTags('Answer {"a"', { complete: false })).toBe('Answer ') - }) -}) diff --git a/apps/sim/lib/mothership/chat/interactive-tags.ts b/apps/sim/lib/mothership/chat/interactive-tags.ts deleted file mode 100644 index 959b3b30a80..00000000000 --- a/apps/sim/lib/mothership/chat/interactive-tags.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** Answer tags the Chat UI renders as interactive cards; text-only surfaces drop them. */ -const CLOSED_TAGS = - /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?<\/\1>/g -/** An interactive tag still open at the end of a partial stream. */ -const TRAILING_OPEN_TAG = - /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>(?![\s\S]*<\/\1>)[\s\S]*$/ - -/** - * Removes interactive Chat UI tags. A streaming answer also withholds a tag that has not closed - * yet; a complete answer keeps text after an unclosed opener, which is prose rather than a card. - */ -export function stripInteractiveTags(text: string, options: { complete: boolean }): string { - const closed = text.replace(CLOSED_TAGS, '') - return options.complete ? closed : closed.replace(TRAILING_OPEN_TAG, '') -} diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 407e0ed791d..0f733d3889c 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -274,6 +274,25 @@ describe('authorized live retrieval', () => { message: expect.stringContaining('could not be verified'), }) }) + it('keeps verified results and names a rate limit hit during verification', async () => { + mocks.search.mockResolvedValue({ + documents: [document, { ...document, id: 'other', url: 'https://docs.google.com/other' }], + }) + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async ({ id }: { id: string }) => { + if (id === 'other') throw new NativeSearchError('rate_limited', 'Later', 30) + return true + }, + }) + const result = await searchLiveKnowledge.execute({ principal, input }) + expect(result.results).toHaveLength(1) + expect(result.live?.accounts[0]).toMatchObject({ + status: 'partial', + message: expect.stringContaining('rate-limited verification'), + }) + }) it('fails the account when a grant is revoked during verification', async () => { mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 953c028f7f7..1841c2185dd 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -224,13 +224,16 @@ async function verifyCandidates(session: LiveAccountSession, candidates: LiveCan return (await session.verify(document)) ? ('permitted' as const) : ('denied' as const) } catch (error) { if (error instanceof NativeSearchError && error.status === 'reconnect') throw error - return 'unverified' as const + return error instanceof NativeSearchError && error.status === 'rate_limited' + ? ('rate_limited' as const) + : ('unverified' as const) } } ) return { permitted: candidates.filter((_, index) => outcomes[index] === 'permitted'), - unverified: outcomes.includes('unverified'), + unverified: outcomes.some((outcome) => outcome === 'unverified' || outcome === 'rate_limited'), + rateLimited: outcomes.includes('rate_limited'), } } @@ -359,7 +362,7 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ * Local filters run first so provider verification is spent only on eligible results. * Undated results are verified too, so their exclusion is reported only when readable. */ - const { permitted, unverified } = await measureSearchStage('live.verify', () => + const { permitted, unverified, rateLimited } = await measureSearchStage('live.verify', () => verifyCandidates( session, candidates.filter( @@ -403,7 +406,9 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ ? 'Service account verification covered a bounded subset of the configured users. Narrow the source user list for complete coverage; external Drive users can only search files also visible to the source administrator.' : undefined, unverified - ? 'Some results could not be verified against the source settings and were omitted.' + ? rateLimited + ? 'The provider rate-limited verification, so some results were omitted. Try again later.' + : 'Some results could not be verified against the source settings and were omitted.' : undefined, undatedExcluded ? 'Some results lacked date metadata and were excluded; date coverage is incomplete.' diff --git a/apps/sim/lib/sim-search/live/text.test.ts b/apps/sim/lib/sim-search/live/text.test.ts index f22dff4287f..5318f47e551 100644 --- a/apps/sim/lib/sim-search/live/text.test.ts +++ b/apps/sim/lib/sim-search/live/text.test.ts @@ -21,6 +21,9 @@ describe('providerText', () => { ) ).toBe('Who:\n\na@b.co\nNotes & links') }) + it('keeps an address whose name starts like a tag in auto mode', () => { + expect(providerText('Organizer ', 'auto')).toBe('Organizer ') + }) it('leaves plain text with angle brackets untouched in auto mode', () => { expect(providerText('Reply from John ', 'auto')).toBe( 'Reply from John ' diff --git a/apps/sim/lib/slack-search/assistant-stream.ts b/apps/sim/lib/slack-search/assistant-stream.ts index cd674c15599..06f3125b206 100644 --- a/apps/sim/lib/slack-search/assistant-stream.ts +++ b/apps/sim/lib/slack-search/assistant-stream.ts @@ -10,7 +10,6 @@ import { parseCitationRecord, type RetrievalCitationBlock, } from '@/lib/mothership/chat/citation-evidence' -import { stripInteractiveTags } from '@/lib/mothership/chat/interactive-tags' import { redactSensitiveContent } from '@/lib/mothership/chat/sim-key-redaction' import type { StreamEvent, @@ -36,7 +35,10 @@ export function publicSlackAnswer( sources: ReadonlyMap = new Map(), integrationsUrl?: string ): string { - let value = stripInteractiveTags(text, { complete: false }) + let value = text.replace( + /<(options|question|thinking|usage_upgrade|credential|workspace_resource)>[\s\S]*?(?:<\/\1>|$)/g, + '' + ) if (!complete) { let end = Math.max(value.lastIndexOf(' '), value.lastIndexOf('\n')) + 1 const sourceStart = value.lastIndexOf('<') From ea8616e909d0edb3fdd3d1e566fb7d3d4cf0094e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 23 Sep 2026 22:40:30 -0700 Subject: [PATCH 6/6] fix(search): strip an MCP answer card only when its body parses as JSON --- .../application/chat-citations.test.ts | 9 +++++ .../knowledge/application/chat-citations.ts | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/apps/sim/lib/knowledge/application/chat-citations.test.ts b/apps/sim/lib/knowledge/application/chat-citations.test.ts index 35068980229..0fe6c9a8647 100644 --- a/apps/sim/lib/knowledge/application/chat-citations.test.ts +++ b/apps/sim/lib/knowledge/application/chat-citations.test.ts @@ -15,6 +15,15 @@ describe('stripInteractiveCards', () => { const prose = 'Wrap choices in your list tags, or use to ask.' expect(stripInteractiveCards(prose)).toBe(prose) }) + it('keeps braces in prose that are not a JSON payload', () => { + const prose = + 'See {the Q4 report}, and {bold} styling applies.' + expect(stripInteractiveCards(prose)).toBe(prose) + }) + it('keeps a closed card whose payload is not valid JSON, and the text after it', () => { + const text = 'Before {"a": oops} after.' + expect(stripInteractiveCards(text)).toBe(text) + }) it('drops a card left open before its JSON payload', () => { expect(stripInteractiveCards('Connect here {"type":"link"')).toBe('Connect here ') }) diff --git a/apps/sim/lib/knowledge/application/chat-citations.ts b/apps/sim/lib/knowledge/application/chat-citations.ts index 9dba5bb846b..1129c6dc892 100644 --- a/apps/sim/lib/knowledge/application/chat-citations.ts +++ b/apps/sim/lib/knowledge/application/chat-citations.ts @@ -12,22 +12,35 @@ export interface SearchChatCitation { const CARD_TAGS = 'options|question|usage_upgrade|credential|workspace_resource' /** A JSON string owns its escaped quotes and any tag-shaped text inside it. */ const JSON_STRING = '"(?:\\\\.|[^"\\\\\\r\\n])*"' -/** - * A card whose body is a JSON value, a thinking block, or a card opener left unclosed before a - * JSON payload. A tag-shaped pair or opener in ordinary prose is not a card and stays. - */ -const INTERACTIVE_CARD = new RegExp( - [ - `<(${CARD_TAGS})>\\s*[[{](?:${JSON_STRING}|[^"<])*?[\\]}]\\s*`, - '[\\s\\S]*?', - `<(?:${CARD_TAGS})>\\s*[[{][\\s\\S]*$`, - ].join('|'), +/** A closed card candidate; its body is removed only when it parses as a JSON payload. */ +const CLOSED_CARD = new RegExp( + `<(${CARD_TAGS})>(\\s*[[{](?:${JSON_STRING}|[^"<])*?[\\]}]\\s*)`, 'g' ) +/** A card the model left open, followed by the start of a JSON object with a quoted key. */ +const UNCLOSED_CARD = new RegExp( + `<(${CARD_TAGS})>(?![\\s\\S]*)\\s*(?:\\{\\s*"|\\[\\s*[{"])[\\s\\S]*$` +) +const THINKING = /[\s\S]*?<\/thinking>/g + +function isJsonPayload(body: string): boolean { + try { + const value: unknown = JSON.parse(body) + return typeof value === 'object' && value !== null + } catch { + return false + } +} -/** Removes interactive Chat cards so a text-only surface receives only the answer prose. */ +/** + * Removes interactive Chat cards so a text-only surface receives only the answer prose. A tag + * pair whose body is not a JSON payload is prose that happens to look like a card, and stays. + */ export function stripInteractiveCards(content: string): string { - return content.replace(INTERACTIVE_CARD, '') + return content + .replace(CLOSED_CARD, (card, _tag: string, body: string) => (isJsonPayload(body) ? '' : card)) + .replace(THINKING, '') + .replace(UNCLOSED_CARD, '') } /** Resolves Assistant source tags only against successful, bounded retrieval evidence. */