diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index efb2effe479..a3d121e59a6 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -147,6 +147,24 @@ jobs: if-no-files-found: ignore retention-days: 7 + - name: Verify Lucid MCP search and complete diagram reads over real HTTP + if: matrix.provision == 'push' + working-directory: apps/sim + env: + NEXT_PUBLIC_APP_URL: http://127.0.0.1:3040 + NEXT_PUBLIC_FORCE_HOSTED: 'false' + SEARCH_LUCID_REPORT_PATH: ${{ runner.temp }}/search-lucid.json + run: bun scripts/test-search-lucid-e2e.ts + + - name: Upload Lucid acceptance report + if: failure() && matrix.provision == 'push' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: search-lucid + path: ${{ runner.temp }}/search-lucid.json + if-no-files-found: ignore + retention-days: 7 + - name: Verify SCIM and administration over real HTTP working-directory: apps/sim env: diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index 01ba721eec7..fad7cd5d558 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -1,6 +1,22 @@ import type { SVGProps } from 'react' import { useId } from 'react' +interface LucidIconProps extends SVGProps {} + +export function LucidIcon(props: LucidIconProps) { + return ( + + + + + ) +} + export function EnrichmentIcon(props: SVGProps) { return ( {} + +export function LucidIcon(props: LucidIconProps) { + return ( + + + + + ) +} + export function EnrichmentIcon(props: SVGProps) { return ( -export const NOTION_SEARCH_TERMS_REQUIRED = - 'Notion requires search terms. Add keywords or a concise question.' +export const SEARCH_TERMS_REQUIRED = { + notion: 'Notion requires search terms. Add keywords or a concise question.', + lucid: 'Lucid requires search terms. Add document-title keywords or a literal shape-text query.', +} as const /** * Native queries one call may send to the same provider account. Alternatives run as separate @@ -24,6 +26,7 @@ const PROVIDER_KIND_SCHEMAS = { github: z.enum(['issues', 'code', 'repositories', 'commits']), gitlab: z.enum(['issues', 'code', 'merge_requests', 'wiki']), hubspot: z.enum(['contacts', 'companies', 'deals', 'tickets']), + lucid: z.enum(['lucidchart', 'lucidspark']), } as const function hasSearchKinds( @@ -36,6 +39,7 @@ const nativeSearchKindSchema = z.enum([ ...PROVIDER_KIND_SCHEMAS.github.options, ...PROVIDER_KIND_SCHEMAS.gitlab.options, ...PROVIDER_KIND_SCHEMAS.hubspot.options, + ...PROVIDER_KIND_SCHEMAS.lucid.options, ]) /** Queries are data for fixed read-only provider endpoints, never URLs or credentials. */ @@ -62,11 +66,11 @@ export const nativeSearchQuerySchema = z message: `${input.provider} kind must be one of: ${kinds.options.join(', ')}.`, }) } - if (input.provider === 'notion' && !input.query) + if ((input.provider === 'notion' || input.provider === 'lucid') && !input.query) context.addIssue({ code: 'custom', path: ['query'], - message: NOTION_SEARCH_TERMS_REQUIRED, + message: SEARCH_TERMS_REQUIRED[input.provider], }) }) export type NativeSearchQuery = z.output @@ -106,7 +110,7 @@ export const nativeSearchQueriesSchema = z earlier.some((previous) => !previous.kind || !query.kind) ) addIssue( - 'A GitHub, GitLab, or HubSpot query without a kind already searches every kind; give each query on this account a kind.' + 'A GitHub, GitLab, HubSpot, or Lucid query without a kind already searches its default kinds; give each query on this account a kind.' ) else if (busiestAccountLoad(earlier) >= MAX_NATIVE_QUERIES_PER_ACCOUNT) addIssue( @@ -188,7 +192,7 @@ export const searchWorkspaceInputSchema = workspaceSearchFiltersSchema nativeQueries: nativeSearchQueriesSchema .optional() .describe( - `Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion always requires search terms. Up to ${MAX_NATIVE_QUERIES_PER_ACCOUNT} per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; ownership filters are unsupported. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.` + `Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot/Lucid terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion and Lucid always require search terms. Up to ${MAX_NATIVE_QUERIES_PER_ACCOUNT} per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; Lucid kinds are lucidchart and lucidspark. Both reject ownership filters. Lucid searches titles with no search continuation; project can scope a literal shape-text query to one known document UUID or Lucid URL. Read for structured diagram evidence. Dates and sorting cover only retrieved candidates, not globally newest/oldest matches. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.` ), query: z .string() diff --git a/apps/sim/lib/credential-groups/managed-mcp-connector-icons.ts b/apps/sim/lib/credential-groups/managed-mcp-connector-icons.ts index ed3ce05a2b8..11c2456fcaf 100644 --- a/apps/sim/lib/credential-groups/managed-mcp-connector-icons.ts +++ b/apps/sim/lib/credential-groups/managed-mcp-connector-icons.ts @@ -4,6 +4,7 @@ import { FirefliesIcon, GranolaIcon, HubspotIcon, + LucidIcon, NotionIcon, } from '@/components/icons' import type { ManagedMcpConnectorId } from '@/lib/credential-groups/managed-mcp-connectors' @@ -12,6 +13,7 @@ export const MANAGED_MCP_CONNECTOR_ICONS = { fireflies: FirefliesIcon, granola: GranolaIcon, hubspot: HubspotIcon, + lucid: LucidIcon, coda: CodaIcon, notion: NotionIcon, databricks: DatabricksIcon, diff --git a/apps/sim/lib/credential-groups/managed-mcp-connectors.ts b/apps/sim/lib/credential-groups/managed-mcp-connectors.ts index 3c6f71632d2..a4868f80b72 100644 --- a/apps/sim/lib/credential-groups/managed-mcp-connectors.ts +++ b/apps/sim/lib/credential-groups/managed-mcp-connectors.ts @@ -5,6 +5,7 @@ export const MANAGED_MCP_CONNECTOR_IDS = [ 'coda', 'notion', 'hubspot', + 'lucid', ] as const export type ManagedMcpConnectorId = (typeof MANAGED_MCP_CONNECTOR_IDS)[number] @@ -38,6 +39,13 @@ export type ManagedMcpConnector = | HubSpotManagedMcpConnector export const MANAGED_MCP_CONNECTORS = { + lucid: { + id: 'lucid', + name: 'Lucid', + description: 'Search Lucidchart diagrams and Lucidspark boards using your Lucid account', + url: 'https://mcp.lucid.app/mcp/readonly', + oauthClientRegistration: 'dynamic', + }, hubspot: { id: 'hubspot', name: 'HubSpot', diff --git a/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts b/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts index 73a895d6345..a97bdcfd82e 100644 --- a/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts @@ -30,7 +30,11 @@ describe('atomic organization live Search MCP setup', () => { beforeAll(() => { vi.spyOn(dns, 'resolveHostAddresses').mockImplementation(async (hostname) => { - if (!['api.fireflies.ai', 'mcp.granola.ai', 'mcp.notion.com'].includes(hostname)) + if ( + !['api.fireflies.ai', 'mcp.granola.ai', 'mcp.notion.com', 'mcp.lucid.app'].includes( + hostname + ) + ) throw new Error(`Unexpected DNS lookup in setup fixture: ${hostname}`) return { addresses: ['93.184.216.34'], preferred: '93.184.216.34' } }) @@ -105,6 +109,7 @@ describe('atomic organization live Search MCP setup', () => { ['fireflies', 'https://api.fireflies.ai/mcp'], ['granola', 'https://mcp.granola.ai/mcp'], ['notion', 'https://mcp.notion.com/mcp'], + ['lucid', 'https://mcp.lucid.app/mcp/readonly'], ])( 'approves %s with an organization-owned sign-in server and access policy', async (provider, url) => { diff --git a/apps/sim/lib/mothership/generated/docs-manifest.ts b/apps/sim/lib/mothership/generated/docs-manifest.ts index 0e07fb694a8..549d62106e4 100644 --- a/apps/sim/lib/mothership/generated/docs-manifest.ts +++ b/apps/sim/lib/mothership/generated/docs-manifest.ts @@ -440,6 +440,7 @@ export const DOCS_MANIFEST: readonly string[] = [ 'search/hubspot.mdx', 'search/jira.mdx', 'search/linear.mdx', + 'search/lucid.mdx', 'search/mcp.mdx', 'search/notion.mdx', 'search/slack.mdx', diff --git a/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts b/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts index e9f86f3e8a2..61a8393e234 100644 --- a/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts +++ b/apps/sim/lib/mothership/generated/sim-assistant-tools.generated.ts @@ -13,6 +13,7 @@ export const liveSearchProviderSchema = z.enum([ 'github', 'gitlab', 'linear', + 'lucid', 'hubspot', 'fireflies', 'granola', @@ -21,8 +22,10 @@ export const liveSearchProviderSchema = z.enum([ ]) export type LiveSearchProvider = z.output -export const NOTION_SEARCH_TERMS_REQUIRED = - 'Notion requires search terms. Add keywords or a concise question.' +export const SEARCH_TERMS_REQUIRED = { + notion: 'Notion requires search terms. Add keywords or a concise question.', + lucid: 'Lucid requires search terms. Add document-title keywords or a literal shape-text query.', +} as const /** * Native queries one call may send to the same provider account. Alternatives run as separate @@ -41,6 +44,7 @@ const PROVIDER_KIND_SCHEMAS = { github: z.enum(['issues', 'code', 'repositories', 'commits']), gitlab: z.enum(['issues', 'code', 'merge_requests', 'wiki']), hubspot: z.enum(['contacts', 'companies', 'deals', 'tickets']), + lucid: z.enum(['lucidchart', 'lucidspark']), } as const function hasSearchKinds( @@ -53,6 +57,7 @@ const nativeSearchKindSchema = z.enum([ ...PROVIDER_KIND_SCHEMAS.github.options, ...PROVIDER_KIND_SCHEMAS.gitlab.options, ...PROVIDER_KIND_SCHEMAS.hubspot.options, + ...PROVIDER_KIND_SCHEMAS.lucid.options, ]) /** Queries are data for fixed read-only provider endpoints, never URLs or credentials. */ @@ -79,11 +84,11 @@ export const nativeSearchQuerySchema = z message: `${input.provider} kind must be one of: ${kinds.options.join(', ')}.`, }) } - if (input.provider === 'notion' && !input.query) + if ((input.provider === 'notion' || input.provider === 'lucid') && !input.query) context.addIssue({ code: 'custom', path: ['query'], - message: NOTION_SEARCH_TERMS_REQUIRED, + message: SEARCH_TERMS_REQUIRED[input.provider], }) }) export type NativeSearchQuery = z.output @@ -123,7 +128,7 @@ export const nativeSearchQueriesSchema = z earlier.some((previous) => !previous.kind || !query.kind) ) addIssue( - 'A GitHub, GitLab, or HubSpot query without a kind already searches every kind; give each query on this account a kind.' + 'A GitHub, GitLab, HubSpot, or Lucid query without a kind already searches its default kinds; give each query on this account a kind.' ) else if (busiestAccountLoad(earlier) >= MAX_NATIVE_QUERIES_PER_ACCOUNT) addIssue( @@ -205,7 +210,7 @@ export const searchWorkspaceInputSchema = workspaceSearchFiltersSchema nativeQueries: nativeSearchQueriesSchema .optional() .describe( - `Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion always requires search terms. Up to ${MAX_NATIVE_QUERIES_PER_ACCOUNT} per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; ownership filters are unsupported. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.` + `Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot/Lucid terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion and Lucid always require search terms. Up to ${MAX_NATIVE_QUERIES_PER_ACCOUNT} per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; Lucid kinds are lucidchart and lucidspark. Both reject ownership filters. Lucid searches titles with no search continuation; project can scope a literal shape-text query to one known document UUID or Lucid URL. Read for structured diagram evidence. Dates and sorting cover only retrieved candidates, not globally newest/oldest matches. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.` ), query: z .string() diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index 35b58f4b7d5..f50870f44d2 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -6095,7 +6095,7 @@ export const SearchWorkspace: ToolCatalogEntry = { }, nativeQueries: { description: - "Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion always requires search terms. Up to 4 per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; ownership filters are unsupported. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.", + "Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot/Lucid terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion and Lucid always require search terms. Up to 4 per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; Lucid kinds are lucidchart and lucidspark. Both reject ownership filters. Lucid searches titles with no search continuation; project can scope a literal shape-text query to one known document UUID or Lucid URL. Read for structured diagram evidence. Dates and sorting cover only retrieved candidates, not globally newest/oldest matches. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.", minItems: 1, maxItems: 9, type: 'array', @@ -6114,6 +6114,7 @@ export const SearchWorkspace: ToolCatalogEntry = { 'github', 'gitlab', 'linear', + 'lucid', 'hubspot', 'fireflies', 'granola', @@ -6136,6 +6137,8 @@ export const SearchWorkspace: ToolCatalogEntry = { 'companies', 'deals', 'tickets', + 'lucidchart', + 'lucidspark', ], }, project: { type: 'string', minLength: 1, maxLength: 300 }, diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index 51b94ea62ce..dd414882e30 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -6043,7 +6043,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, nativeQueries: { description: - "Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion always requires search terms. Up to 4 per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; ownership filters are unsupported. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.", + "Live search only: queries in a provider's own language (Drive q, Gmail operators, JQL, CQL, GitHub qualifiers, Slack RTS, plain Linear/Fireflies/HubSpot/Lucid terms, Granola natural-language questions, Notion keywords or AI questions when available). Blank queries require a date bound or sortBy newest/oldest; Notion and Lucid always require search terms. Up to 4 per account run separately and merge; one GitHub, GitLab, or HubSpot query without a kind searches GitHub issues (plus code when the query has no date bound or boolean operators, as its status message says), GitLab issues, merge requests, and code, or every HubSpot CRM kind; other collections, and multiple queries on one account, each need a kind, which may repeat. HubSpot kinds are contacts, companies, deals, and tickets; Lucid kinds are lucidchart and lucidspark. Both reject ownership filters. Lucid searches titles with no search continuation; project can scope a literal shape-text query to one known document UUID or Lucid URL. Read for structured diagram evidence. Dates and sorting cover only retrieved candidates, not globally newest/oldest matches. Write queries from the returned live guidance and account IDs; each account status names the queryIndex its cursor belongs to. Omit for simple cross-provider terms.", minItems: 1, maxItems: 9, type: 'array', @@ -6062,6 +6062,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'github', 'gitlab', 'linear', + 'lucid', 'hubspot', 'fireflies', 'granola', @@ -6091,6 +6092,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { 'companies', 'deals', 'tickets', + 'lucidchart', + 'lucidspark', ], }, project: { diff --git a/apps/sim/lib/sim-search/live/README.md b/apps/sim/lib/sim-search/live/README.md index 6dee2df8372..18baee8477f 100644 --- a/apps/sim/lib/sim-search/live/README.md +++ b/apps/sim/lib/sim-search/live/README.md @@ -94,7 +94,7 @@ Content types, code branch/tag, path prefix, file extensions, and issue state/la ## Adding a live Search connector -A workspace KB connector and a live Search provider are different runtime integrations. `listDocuments`/`getDocument`, hashes, chunks, and embeddings remain the KB contract; adding those functions alone does not implement live Search. There are currently thirteen live providers, while the broader KB registry contains additional providers that are not advertised for Search. +A workspace KB connector and a live Search provider are different runtime integrations. `listDocuments`/`getDocument`, hashes, chunks, and embeddings remain the KB contract; adding those functions alone does not implement live Search. The live provider catalog is independent of the broader KB registry, which contains additional providers that are not advertised for Search. ### Registration and ownership @@ -130,6 +130,7 @@ Self-managed GitLab is resolved from the saved source's validated host/project i | Confluence | `/ex/confluence/{cloudId}/wiki/rest/api/search` with CQL | v2 `/wiki/api/v2/pages/{id}` or `/blogposts/{id}` (`body-format=view`); a space reads as its homepage | Same site, spaces, current type/status/labels, source readability | | GitHub | `/search/issues`, `/search/code`, `/search/repositories`, `/search/commits` | Issue, repository, commit, or contents endpoint for returned kind | Added repositories; installation coverage/stable IDs and code filters | | GitLab | Configured `/api/v4/projects/{project}/search`, or supported date listing | Project issue/MR/wiki/file endpoint | Current request-local admin ACL evidence or saved CSV grants, plus content filters | +| Lucid | MCP `search` for titles; `lucid_search_document` within a known document | Bounded complete `fetch` pages/regions | Member only; official read-only MCP, current grant and stable document version | | Linear | GraphQL `searchIssues` including comments/archived, or dated `issues` listing | Issue description and paginated comments | Member only; current OAuth grant | | Fireflies | MCP `fireflies_get_transcripts` with `scope: all` | Transcript sentences plus summary | Member only; fixed official OAuth server | | Granola | MCP meeting query or date listing, hydrated cited meetings | Notes and transcript when available | Member only; source evidence and explicit bounded coverage | @@ -138,6 +139,16 @@ Self-managed GitLab is resolved from the saved source's validated host/project i GitHub members use App user tokens. The deployment App needs read permissions for Contents, Issues, and Pull requests for full supported search/read coverage, plus Metadata, organization Members, and user Email addresses for existing setup/identity checks. Installation tokens used to prove repository coverage stay narrowed to contents/metadata; do not use them to replace the member's content grant. +### Lucid + +Uses the official `https://mcp.lucid.app/mcp/readonly` server with dynamic OAuth registration through the existing managed-MCP member flow. No custom OAuth client, new env variables, email-identity exception or schema change is required. Only search, document-text search, metadata and content fetch are allowlisted; feedback submission is excluded. Lucid enforces an account boundary and does not expose externally owned documents even when shared. + +Document search is title-oriented, relevance-ranked, capped at 200 provider candidates and 10 current metadata reads, with no continuation. A known document UUID or Lucid URL in `project` enables literal case-insensitive shape-text search. Modification-date filters and sorting cover the retrieved candidates; status and guidance disclose that limitation. Metadata previews are not diagram evidence. + +Reads preserve provider page/region JSON, including node and edge properties, without interpreting layout as connectivity or fetching image/link references. The adapter validates all declared page regions, current document identity and revision, and rejects incomplete, changed or oversized reads. It permits at most 8 region calls plus a manifest and two metadata calls within the shared 12-call budget; output is capped at 512 KiB of UTF-8 JSON. Signed revisions bind read continuations to the original version. Comments, rendered images and Lucidscale are outside this integration. + +`test-search-lucid-e2e.ts` exercises the production MCP transport, payload parser and adapter over loopback HTTP with synthetic provider responses and writes `SEARCH_LUCID_REPORT_PATH`. It is separate from real-account acceptance; do not present deterministic fixtures as live Lucid evidence. + ### Shared invariants - Keep provider parsing isolated from authorization. The application operation owns current membership, policy loading, active account resolution, scoped references, and result projection. diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index c4c7b034956..d442af87e58 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -12,6 +12,7 @@ import { NativeSearchError, } from '@/lib/sim-search/live/http' import { readHubSpotMcp, searchHubSpotMcp } from '@/lib/sim-search/live/hubspot-mcp' +import { readLucidMcp, searchLucidMcp } from '@/lib/sim-search/live/lucid-mcp' import { createManagedSearchMcpClient } from '@/lib/sim-search/live/managed-mcp' import { isManagedSearchMcpProvider } from '@/lib/sim-search/live/managed-mcp-config' import { readNotionMcp, searchNotionMcp } from '@/lib/sim-search/live/notion-mcp' @@ -113,11 +114,14 @@ export async function openLiveAccountSession( return searchNotionMcp(mcp, search) case 'hubspot': return searchHubSpotMcp(mcp, search) + case 'lucid': + return searchLucidMcp(mcp, search) default: throw new NativeSearchError('unavailable', 'Unsupported managed MCP provider.') } } - const readMcp = (id: string) => { + const readMcp = (reference: Reference) => { + const { id } = reference if (!mcp) throw new NativeSearchError('unavailable', 'Managed MCP connection unavailable.') switch (provider) { case 'coda': @@ -130,6 +134,8 @@ export async function openLiveAccountSession( return readNotionMcp(mcp, id) case 'hubspot': return readHubSpotMcp(mcp, id) + case 'lucid': + return readLucidMcp(mcp, reference) default: throw new NativeSearchError('unavailable', 'Unsupported managed MCP provider.') } @@ -184,7 +190,7 @@ export async function openLiveAccountSession( signal, verify: boundary.verify, }) - return readMcp(reference.id) + return readMcp(reference) }, async verifyCurrent(document) { const current = await sourceBoundary( diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 35920713c3e..4ed85fc9559 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -287,20 +287,34 @@ describe('authorized live retrieval', () => { } ) }) - it.each([ - { startDate: '2026-08-01T00:00:00Z' }, - { source: ' notion ', startDate: '2026-08-01T00:00:00Z' }, - { sortBy: 'newest' as const }, - { sortBy: 'oldest' as const }, - ])('rejects a Notion-only live listing with %j', async (bound) => { - await expect( - searchLiveKnowledge.execute({ - principal, - input: { ...input, query: ' \t ', filters: { source: 'notion', ...bound } }, + describe.each(['notion', 'lucid'] as const)('%s requires terms before dispatch', (provider) => { + it.each([ + { startDate: '2026-08-01T00:00:00Z' }, + { source: ` ${provider} `, startDate: '2026-08-01T00:00:00Z' }, + { sortBy: 'newest' as const }, + { sortBy: 'oldest' as const }, + ])('rejects a provider-only listing with %j', async (bound) => { + await expect( + searchLiveKnowledge.execute({ + principal, + input: { ...input, query: ' \t ', filters: { source: provider, ...bound } }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + }) + it('rejects an empty native query even with a date bound', async () => { + await expect( + searchLiveKnowledge.execute({ + principal, + input: { + ...input, + query: 'topology', + filters: { startDate: '2026-08-01T00:00:00Z' }, + nativeQueries: [{ provider, query: ' \t ' }], + }, + }) + ).rejects.toMatchObject({ + issues: expect.arrayContaining([expect.objectContaining({ path: [0, 'query'] })]), }) - ).rejects.toMatchObject({ - code: 'validation', - message: 'Notion requires search terms. Add keywords or a concise question.', }) }) it.each([undefined, 'google_drive'])( diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 3eb5ddb69af..93ef5706fbe 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -10,8 +10,8 @@ import { type LiveSearchAccountStatus, liveSearchProviderSchema, type NativeSearchQuery, - NOTION_SEARCH_TERMS_REQUIRED, nativeSearchQueriesSchema, + SEARCH_TERMS_REQUIRED, workspaceSearchFiltersSchema, } from '@/lib/api/contracts/mothership-assistant-tools' import { canonicalJson, fingerprint, instantScopePart } from '@/lib/api/cursor-binding' @@ -344,8 +344,12 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ ) throw new OrchestrationError('validation', 'Invalid live search query or result limit') const filters = input.filters - if (!queries && filters?.source === 'notion' && !input.query.trim()) - throw new OrchestrationError('validation', NOTION_SEARCH_TERMS_REQUIRED) + if ( + !queries && + (filters?.source === 'notion' || filters?.source === 'lucid') && + !input.query.trim() + ) + throw new OrchestrationError('validation', SEARCH_TERMS_REQUIRED[filters.source]) if ( filters?.startDate && filters.endDate && diff --git a/apps/sim/lib/sim-search/live/coda-mcp.test.ts b/apps/sim/lib/sim-search/live/coda-mcp.test.ts index d05d45b745f..5d456aa1d81 100644 --- a/apps/sim/lib/sim-search/live/coda-mcp.test.ts +++ b/apps/sim/lib/sim-search/live/coda-mcp.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import type { McpToolResult } from '@/lib/mcp/types' import { type CodaMcpClient, readCodaMcp, searchCodaMcp } from '@/lib/sim-search/live/coda-mcp' -import { managedMcpPayload } from '@/lib/sim-search/live/managed-mcp' +import { managedMcpPayload } from '@/lib/sim-search/live/managed-mcp-payload' const codaMcpPayload = (result: McpToolResult) => managedMcpPayload(result, 'Coda') diff --git a/apps/sim/lib/sim-search/live/lucid-mcp.ts b/apps/sim/lib/sim-search/live/lucid-mcp.ts new file mode 100644 index 00000000000..23dcdf357bd --- /dev/null +++ b/apps/sim/lib/sim-search/live/lucid-mcp.ts @@ -0,0 +1,324 @@ +import { isRecordLike } from '@sim/utils/object' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { + dateSortDirection, + hasDateBounds, + nativeDateBounds, + nativeText, +} from '@/lib/sim-search/live/dates' +import { NativeSearchError, object, string } from '@/lib/sim-search/live/http' +import type { ManagedSearchMcpClient } from '@/lib/sim-search/live/managed-mcp' +import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' + +const UUID = /^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/i +const PRODUCTS = ['lucidchart', 'lucidspark'] as const +const MAX_CANDIDATES = 10 +const MAX_REGIONS = 8 +const MAX_CONTENT_BYTES = 512 * 1024 + +function invalid(message: string): never { + throw new NativeSearchError('unavailable', message) +} + +function resource(value: string): { id: string; kind?: string } | undefined { + if (UUID.test(value)) return { id: value.toLowerCase() } + try { + const url = new URL(value) + const parts = url.pathname.split('/').filter(Boolean) + if ( + url.origin !== 'https://lucid.app' || + url.username || + url.password || + parts.length !== 3 || + !PRODUCTS.some((product) => product === parts[0]) || + !UUID.test(parts[1] ?? '') || + !['edit', 'view'].includes(parts[2] ?? '') + ) + return undefined + return { id: parts[1]!.toLowerCase(), kind: parts[0] } + } catch { + return undefined + } +} + +async function metadata( + client: ManagedSearchMcpClient, + id: string +): Promise { + if (!UUID.test(id)) invalid('Invalid Lucid document identity.') + const row = object(await client.call('lucid_get_document_metadata', { document_id: id })) + const url = resource(string(row.viewUrl)) + if ( + row.documentId !== id || + url?.id !== id || + url.kind !== row.product || + typeof row.title !== 'string' || + row.trashed || + !Number.isSafeInteger(row.version) || + Number(row.version) < 0 || + !Number.isSafeInteger(row.pageCount) || + Number(row.pageCount) < 1 || + typeof row.lastModified !== 'string' || + !Number.isFinite(Date.parse(row.lastModified)) + ) + return undefined + return { + id, + kind: url.kind, + title: row.title, + url: `https://lucid.app/${url.kind}/${id}/view`, + revision: String(row.version), + modifiedAt: row.lastModified, + accessMetadata: { pageCount: row.pageCount }, + content: + 'Lucid document match. Read this document for its structured diagram or board content.', + } +} + +export async function searchLucidMcp( + client: ManagedSearchMcpClient, + input: NativeSearchInput +): Promise { + const query = nativeText(input) + if (!query.trim()) invalid('Lucid requires search terms, including for date-filtered searches.') + const project = input.native?.project + if ([...query].length > (project ? 200 : 400)) + invalid(`Lucid search terms must be at most ${project ? 200 : 400} characters.`) + if ( + input.native?.cursor || + input.native?.modifiers || + input.native?.termClauses?.length || + input.native?.keywordOnly + ) + invalid('Lucid accepts plain terms; query operators and search continuations are unsupported.') + const products = PRODUCTS.filter( + (product) => !input.native?.kind || input.native.kind === product + ) + if (!products.length) invalid('Lucid supports lucidchart and lucidspark document kinds.') + if (project) { + const scope = resource(project) + if (!scope || !query.trim()) + invalid( + 'Lucid document search requires a document UUID or Lucid URL and a literal text query.' + ) + const document = await metadata(client, scope.id) + if (!document) invalid('Lucid document metadata is incomplete or no longer readable.') + if ( + (scope.kind && document.kind !== scope.kind) || + !products.some((product) => product === document.kind) + ) + invalid('The Lucid document does not match the requested product.') + const result = object( + await client.call('lucid_search_document', { id: document.id, queries: [query] }) + ) + const matchedResource = resource(string(result.edit_url)) + if ( + result.document_id !== document.id || + matchedResource?.id !== document.id || + matchedResource.kind !== document.kind + ) + invalid('Lucid document-search identity does not match the requested document.') + const matches = object(result.matches)[query] + if ( + !Array.isArray(matches) || + matches.some( + (match) => + !isRecordLike(match) || + !Number.isSafeInteger(match.pageIndex) || + Number(match.pageIndex) < 1 || + Number(match.pageIndex) > Number(document.accessMetadata?.pageCount) || + !Number.isSafeInteger(match.regionIndex) || + Number(match.regionIndex) < 1 || + !Array.isArray(match.context) || + match.context.some((text) => typeof text !== 'string') + ) + ) + invalid('Lucid returned an unsupported document-search result.') + const preview = matches + .map( + (match) => + `Page ${match.pageIndex}, region ${match.regionIndex}: ${match.context.join('\n')}` + ) + .join('\n') + if (Buffer.byteLength(preview, 'utf8') > MAX_CONTENT_BYTES) + invalid('Lucid matches exceed the search size limit. Narrow the query.') + return { + documents: matches.length ? [{ ...document, content: preview }] : [], + message: + 'Lucid matched literal, case-insensitive substrings in shape labels within the specified document. Notes, tags, links and comments are not searched. Read the document for surrounding structure.', + } + } + const bounds = nativeDateBounds(input) + const result = object( + await client.call('search', { + query, + product: products, + ...(bounds.start + ? { last_modified_after: new Date(Date.parse(bounds.start) - 1000).toISOString() } + : {}), + }) + ) + if (!Array.isArray(result.results)) invalid('Lucid search returned an unsupported result format.') + const candidates: { id: string; kind: string }[] = [] + let dropped = false + const seen = new Set() + for (const row of result.results) { + const reference = isRecordLike(row) ? resource(string(row.url)) : undefined + if ( + !reference?.kind || + !isRecordLike(row) || + reference.id !== row.id || + !products.some((product) => product === reference.kind) + ) { + dropped = true + continue + } + if (seen.has(reference.id)) continue + seen.add(reference.id) + candidates.push({ id: reference.id, kind: reference.kind }) + } + const limit = Math.max(1, Math.min(MAX_CANDIDATES, input.limit)) + const documents = await mapWithConcurrency(candidates.slice(0, limit), 3, async (candidate) => { + const document = await metadata(client, candidate.id) + if (!document || document.kind !== candidate.kind) { + dropped = true + return undefined + } + return document + }) + const capped = candidates.length > limit || result.results.length >= 200 + const localDates = hasDateBounds(input.filters) || Boolean(dateSortDirection(input.filters)) + return { + documents: documents.filter((document) => document !== undefined), + hasMore: capped, + partial: dropped || capped || localDates, + message: + 'Lucid finds documents by title keywords, with up to 200 relevance-ranked candidates from the provider and at most 10 current metadata reads. Previews are metadata, not diagram evidence. For shape text, find a document, then set project to its UUID or Lucid URL. No search continuation is available.' + + (localDates + ? ' Dates use current modification timestamps; sorting and the end-date filter cover only the retrieved candidates, not the entire account.' + : '') + + (capped ? ' The candidate limit was reached; narrow the title query.' : '') + + (dropped ? ' Unsupported or no-longer-readable search results were excluded.' : ''), + } +} + +function manifest(value: unknown, id: string, kind: string | undefined) { + const row = object(value) + const url = resource(string(row.edit_url)) + const details = object(row.metadata) + const counts = details.page_region_counts + if ( + row.document_id !== id || + url?.id !== id || + url.kind !== kind || + typeof row.title !== 'string' || + !Number.isSafeInteger(details.page_count) || + Number(details.page_count) < 1 || + !Array.isArray(counts) || + counts.length !== details.page_count || + counts.some((count) => !Number.isSafeInteger(count) || count < 0) + ) + invalid('Lucid returned incomplete document coverage or mismatched identity.') + if (counts.reduce((total: number, count: number) => total + Math.max(1, count), 0) > MAX_REGIONS) + invalid( + 'Lucid document exceeds the complete-read limit of 8 page regions. Open the source document or narrow it into smaller documents.' + ) + return { row, counts: counts as number[] } +} + +/** Complete, bounded provider pages preserve graph data; no returned URL is fetched. */ +export async function readLucidMcp( + client: ManagedSearchMcpClient, + reference: Pick +): Promise { + const before = await metadata(client, reference.id) + if (!before) invalid('Lucid document metadata is incomplete or no longer readable.') + if ( + (reference.kind && reference.kind !== before.kind) || + (reference.revision && reference.revision !== before.revision) + ) + invalid('Lucid document changed since this result was issued. Search again before reading.') + const initial = manifest( + await client.call('fetch', { id: before.id, metadata_only: true }), + before.id, + before.kind + ) + if ( + initial.counts.length !== before.accessMetadata?.pageCount || + initial.row.title !== before.title + ) + invalid('Lucid document coverage changed before reading. Search again.') + const pages: Record[] = [] + const output = () => JSON.stringify({ document_id: before.id, title: before.title, pages }) + for (let pageIndex = 0; pageIndex < initial.counts.length; pageIndex++) { + const count = initial.counts[pageIndex]! + let assembled: Record | undefined + const chunks: Record[] = [] + for (let region = 0; region < Math.max(1, count); region++) { + const fetched = manifest( + await client.call('fetch', { + id: before.id, + page_index: pageIndex + 1, + ...(count ? { region_index: [region + 1] } : {}), + }), + before.id, + before.kind + ) + if ( + fetched.counts.some((value, index) => value !== initial.counts[index]) || + fetched.counts.length !== initial.counts.length || + object(fetched.row.metadata).page_index !== pageIndex + 1 || + fetched.row.page_index !== pageIndex + 1 || + typeof fetched.row.text !== 'string' + ) + invalid('Lucid document coverage changed during reading. Search again.') + let parsed: unknown + try { + parsed = JSON.parse(fetched.row.text) + } catch { + invalid('Lucid returned malformed diagram content.') + } + const returnedPages = object(parsed).pages + if (!Array.isArray(returnedPages) || returnedPages.length !== 1) + invalid('Lucid returned incomplete page content.') + const page = object(returnedPages[0]) + const returnedChunks = page.requestedChunks + if ( + page.pageIndex !== pageIndex || + typeof page.pageId !== 'string' || + !page.pageId || + page.pageId !== fetched.row.page_id || + typeof page.pageTitle !== 'string' || + page.totalChunks !== count || + !Array.isArray(returnedChunks) || + returnedChunks.length !== (count ? 1 : 0) || + returnedChunks.some( + (chunk) => + !isRecordLike(chunk) || chunk.chunkIndex !== region || !isRecordLike(chunk.data) + ) || + (assembled && (assembled.pageId !== page.pageId || assembled.pageTitle !== page.pageTitle)) + ) + invalid('Lucid returned missing, duplicate, or mismatched page regions.') + if (!assembled) { + if (pages.some((existing) => existing.pageId === page.pageId)) + invalid('Lucid returned duplicate page identities.') + assembled = { ...page, requestedChunks: chunks } + pages.push(assembled) + } + chunks.push(...returnedChunks) + if (Buffer.byteLength(output(), 'utf8') > MAX_CONTENT_BYTES) + invalid('Lucid document exceeds the 512 KiB complete-read limit. Open the source document.') + } + } + const after = await metadata(client, before.id) + if ( + !after || + after.revision !== before.revision || + after.kind !== before.kind || + after.modifiedAt !== before.modifiedAt || + after.title !== before.title || + after.accessMetadata?.pageCount !== before.accessMetadata?.pageCount + ) + invalid('Lucid document changed while being read. Search again before reading.') + return { ...before, content: output() } +} diff --git a/apps/sim/lib/sim-search/live/managed-mcp-config.ts b/apps/sim/lib/sim-search/live/managed-mcp-config.ts index f9e38090465..89dac3b7330 100644 --- a/apps/sim/lib/sim-search/live/managed-mcp-config.ts +++ b/apps/sim/lib/sim-search/live/managed-mcp-config.ts @@ -4,6 +4,7 @@ export const MANAGED_SEARCH_MCP_READ_TOOLS = { fireflies: ['fireflies_get_transcripts', 'fireflies_get_transcript', 'fireflies_get_summary'], granola: ['query_granola_meetings', 'list_meetings', 'get_meetings', 'get_meeting_transcript'], hubspot: ['get_user_details', 'search_crm_objects', 'get_crm_objects'], + lucid: ['search', 'fetch', 'lucid_search_document', 'lucid_get_document_metadata'], notion: ['notion-get-tool-access', 'notion-search', 'notion-ai-search', 'notion-fetch'], } as const diff --git a/apps/sim/lib/sim-search/live/managed-mcp-payload.ts b/apps/sim/lib/sim-search/live/managed-mcp-payload.ts new file mode 100644 index 00000000000..34f69e0a608 --- /dev/null +++ b/apps/sim/lib/sim-search/live/managed-mcp-payload.ts @@ -0,0 +1,59 @@ +import { isRecordLike } from '@sim/utils/object' +import type { McpToolResult } from '@/lib/mcp/types' +import { NativeSearchError } from '@/lib/sim-search/live/http' + +const MAX_SEARCH_MCP_PAYLOAD_BYTES = 4 * 1024 * 1024 + +/** MCP text is untrusted provider data; malformed structured search output is never an empty success. */ +export function managedMcpPayload(result: McpToolResult, label: string): unknown { + if (Buffer.byteLength(JSON.stringify(result), 'utf8') > MAX_SEARCH_MCP_PAYLOAD_BYTES) + throw new NativeSearchError( + 'unavailable', + `${label} response exceeded the search size limit. Narrow the query.` + ) + if (result.isError) { + if ( + label === 'Granola' && + result.content?.some( + (block) => + block.type === 'text' && + /Unauthorized: user has not created a Granola account yet\./i.test(block.text ?? '') + ) + ) + throw new NativeSearchError( + 'reconnect', + 'Reconnect using an existing Granola account. Check the account email in the Granola app.' + ) + const quota = result.content?.some( + (block) => + block.type === 'text' && + /(?:rate.?limit|quota|weekly limit of \d+ MCP requests)/i.test(block.text ?? '') + ) + throw new NativeSearchError( + quota ? 'rate_limited' : 'unavailable', + quota + ? `${label} MCP request limit reached. Try again when it resets.` + : `${label} could not complete this read. Check the query and your access.` + ) + } + const unwrap = (value: unknown) => + isRecordLike(value) && typeof value.toolName === 'string' && 'result' in value + ? value.result + : value + const lucidWidgetMetadata = + label === 'Lucid' && + isRecordLike(result.structuredContent) && + Object.keys(result.structuredContent).every((key) => key === 'widgetInstance') + if (result.structuredContent !== undefined && !lucidWidgetMetadata) + return unwrap(result.structuredContent) + const text = (result.content ?? []) + .filter((block) => block.type === 'text') + .map((block) => block.text ?? '') + .join('\n') + if (!text) throw new NativeSearchError('unavailable', `${label} returned no readable content.`) + try { + return unwrap(JSON.parse(text)) + } catch { + return { text } + } +} diff --git a/apps/sim/lib/sim-search/live/managed-mcp.test.ts b/apps/sim/lib/sim-search/live/managed-mcp.test.ts index ad310ba6c63..9fe3c98d2fc 100644 --- a/apps/sim/lib/sim-search/live/managed-mcp.test.ts +++ b/apps/sim/lib/sim-search/live/managed-mcp.test.ts @@ -9,7 +9,8 @@ vi.mock('@/lib/mcp/application/managed-auth-provider', () => ({ })) import { NativeSearchError } from '@/lib/sim-search/live/http' -import { createManagedSearchMcpClient, managedMcpPayload } from '@/lib/sim-search/live/managed-mcp' +import { createManagedSearchMcpClient } from '@/lib/sim-search/live/managed-mcp' +import { managedMcpPayload } from '@/lib/sim-search/live/managed-mcp-payload' /** Failure modes: a write tool escapes the allowlist; replaced grants stay usable; payloads exhaust memory; schema drift changes tool meaning. */ describe('managed search MCP read boundary', () => { diff --git a/apps/sim/lib/sim-search/live/managed-mcp.ts b/apps/sim/lib/sim-search/live/managed-mcp.ts index ef6159950bf..feedc1e89bf 100644 --- a/apps/sim/lib/sim-search/live/managed-mcp.ts +++ b/apps/sim/lib/sim-search/live/managed-mcp.ts @@ -4,16 +4,14 @@ import { MANAGED_MCP_CONNECTORS } from '@/lib/credential-groups/managed-mcp-conn import { createManagedMcpAuthProvider } from '@/lib/mcp/application/managed-auth-provider' import { mcpService } from '@/lib/mcp/service' import { compileMcpToolSchema } from '@/lib/mcp/tool-schema' -import type { McpToolResult } from '@/lib/mcp/types' import { NativeSearchError } from '@/lib/sim-search/live/http' import { MANAGED_SEARCH_MCP_READ_TOOLS, type ManagedSearchMcpProvider, } from '@/lib/sim-search/live/managed-mcp-config' +import { managedMcpPayload } from '@/lib/sim-search/live/managed-mcp-payload' import { loadOwnManagedMcpRuntime } from '@/lib/sim-search/live/mcp-accounts' -const MAX_SEARCH_MCP_PAYLOAD_BYTES = 4 * 1024 * 1024 - export interface ManagedSearchMcpClient { call(name: string, args: Record): Promise /** Optional provider features are used only when the current server advertises them. */ @@ -104,52 +102,3 @@ export async function createManagedSearchMcpClient( }, } } - -/** MCP text is untrusted provider data; malformed structured search output is never an empty success. */ -export function managedMcpPayload(result: McpToolResult, label: string): unknown { - if (Buffer.byteLength(JSON.stringify(result), 'utf8') > MAX_SEARCH_MCP_PAYLOAD_BYTES) - throw new NativeSearchError( - 'unavailable', - `${label} response exceeded the search size limit. Narrow the query.` - ) - if (result.isError) { - if ( - label === 'Granola' && - result.content?.some( - (block) => - block.type === 'text' && - /Unauthorized: user has not created a Granola account yet\./i.test(block.text ?? '') - ) - ) - throw new NativeSearchError( - 'reconnect', - 'Reconnect using an existing Granola account. Check the account email in the Granola app.' - ) - const quota = result.content?.some( - (block) => - block.type === 'text' && - /(?:rate.?limit|quota|weekly limit of \d+ MCP requests)/i.test(block.text ?? '') - ) - throw new NativeSearchError( - quota ? 'rate_limited' : 'unavailable', - quota - ? `${label} MCP request limit reached. Try again when it resets.` - : `${label} could not complete this read. Check the query and your access.` - ) - } - const unwrap = (value: unknown) => - isRecordLike(value) && typeof value.toolName === 'string' && 'result' in value - ? value.result - : value - if (result.structuredContent !== undefined) return unwrap(result.structuredContent) - const text = (result.content ?? []) - .filter((block) => block.type === 'text') - .map((block) => block.text ?? '') - .join('\n') - if (!text) throw new NativeSearchError('unavailable', `${label} returned no readable content.`) - try { - return unwrap(JSON.parse(text)) - } catch { - return { text } - } -} diff --git a/apps/sim/lib/sim-search/live/policy-schema.ts b/apps/sim/lib/sim-search/live/policy-schema.ts index 52dddaecc75..b37600a4b1e 100644 --- a/apps/sim/lib/sim-search/live/policy-schema.ts +++ b/apps/sim/lib/sim-search/live/policy-schema.ts @@ -104,6 +104,11 @@ export const LIVE_SEARCH_SCOPE_FIELDS: Record< hint: 'Use Confluence space keys. Restrict the site below when spaces share a key.', example: 'ENG, TEAM', }, + lucid: { + label: 'Documents', + hint: 'Member accounts search accessible Lucidchart diagrams and Lucidspark boards.', + example: '', + }, linear: { label: 'Projects', hint: 'Member accounts search all accessible issues.', example: '' }, hubspot: { label: 'CRM records', diff --git a/apps/sim/lib/sim-search/live/provider-catalog.ts b/apps/sim/lib/sim-search/live/provider-catalog.ts index 320c45f0c61..3425d2c72b7 100644 --- a/apps/sim/lib/sim-search/live/provider-catalog.ts +++ b/apps/sim/lib/sim-search/live/provider-catalog.ts @@ -51,6 +51,11 @@ export const LIVE_SEARCH_PROVIDER_CATALOG = { credentialProviderIds: ['linear'], modes: ['member'], }, + lucid: { + origin: 'https://mcp.lucid.app', + credentialProviderIds: ['mcp:lucid'], + modes: ['member'], + }, hubspot: { origin: 'https://mcp.hubspot.com', credentialProviderIds: ['mcp:hubspot'], diff --git a/apps/sim/lib/sim-search/live/providers.ts b/apps/sim/lib/sim-search/live/providers.ts index 107d45afae5..6ec8959be4f 100644 --- a/apps/sim/lib/sim-search/live/providers.ts +++ b/apps/sim/lib/sim-search/live/providers.ts @@ -59,6 +59,18 @@ interface ManagedMcpProvider { /** Native providers implement both reads; managed MCP retrieval is dispatched by account-session. */ export const LIVE_SEARCH_PROVIDERS = { + lucid: { + transport: 'managed_mcp', + guide: { + syntax: + 'Nonempty document-title keywords, at most 400 characters. Results are relevance-ranked, not guaranteed literal title matches. The provider returns at most 200 relevance-ranked candidates; Sim verifies metadata for at most 10. Search has no continuation.', + scope: + 'kind lucidchart or lucidspark selects a product; omit to search both. To search shape text within a known document, set project to its UUID or Lucid URL and use one literal substring of at most 200 characters. Dates use modification time; sorting and end dates apply only to retrieved candidates, not the entire account.', + example: 'deployment architecture', + avoid: + 'Boolean/field operators, ownership filters, claiming exhaustive account-wide body search or global newest/oldest results. Title previews are metadata; read results for structured pages, nodes, edges and properties. Preserve explicit endpoint styles when interpreting arrows. Reads require a stable version and reject documents over 8 page regions or 512 KiB. Images, linked websites, comments, Lucidscale and documents owned outside the connected account are not included.', + }, + }, google_drive: { guide: { syntax: diff --git a/apps/sim/scripts/test-search-lucid-e2e.ts b/apps/sim/scripts/test-search-lucid-e2e.ts new file mode 100644 index 00000000000..d09eb0e8b8c --- /dev/null +++ b/apps/sim/scripts/test-search-lucid-e2e.ts @@ -0,0 +1,489 @@ +import assert from 'node:assert/strict' +import { mkdir, writeFile } from 'node:fs/promises' +import http from 'node:http' +import type { AddressInfo } from 'node:net' +import { dirname } from 'node:path' +import { Server } from '@modelcontextprotocol/sdk/server/index.js' +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js' +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { isHosted } from '@/lib/core/config/env-flags' +import { McpClient } from '@/lib/mcp/client' +import { NativeSearchError } from '@/lib/sim-search/live/http' +import { readLucidMcp, searchLucidMcp } from '@/lib/sim-search/live/lucid-mcp' +import type { ManagedSearchMcpClient } from '@/lib/sim-search/live/managed-mcp' +import { managedMcpPayload } from '@/lib/sim-search/live/managed-mcp-payload' + +/** Real MCP transport with synthetic provider fixtures; not a live Lucid-account run. */ +const logger = createLogger('SearchLucidE2E') +const reportPath = process.env.SEARCH_LUCID_REPORT_PATH +assert(reportPath, 'Set SEARCH_LUCID_REPORT_PATH') +assert(!isHosted, 'Use a local self-hosted URL with NEXT_PUBLIC_FORCE_HOSTED=false') +const ID = '00000000-0000-4000-8000-000000000001' +const OTHER_ID = '00000000-0000-4000-8000-000000000002' +const TITLE = 'Synthetic topology' +const checks: { name: string; status: string; durationMs: number; error?: string }[] = [] +const requests: { tool: string; status: string }[] = [] +let mode = '' +let counts = [2, 1] +let metadataReads = 0 +let calls = 0 +let contentCalls = 0 +let padding = '' +let blockedContent: (() => void) | undefined +let enteredContent: (() => void) | undefined +let observerRequests = 0 +const url = (id = ID, product = 'lucidchart') => `https://lucid.app/${product}/${id}/edit` +const nodeData = (page: number, region: number) => ({ + nodes: [ + { + id: `node-${page}-${region}`, + label: `Page ${page} region ${region} — café`, + properties: { image: `${origin}/observer`, link: `${origin}/observer`, padding }, + }, + ], + edges: [ + { + id: `edge-${page}-${region}`, + sourceId: 'api', + targetId: 'database', + properties: { + Endpoint1: 'style: None, connectedBlockId: api', + Endpoint2: 'style: Arrow, connectedBlockId: database', + }, + }, + ], + customDiagramFamily: { preserved: ['custom data', 'group membership'] }, +}) +const fixturePage = (page: number, regions: number[]) => ({ + pageId: `page-${page}`, + pageTitle: `Page ${page}`, + pageIndex: page - 1, + textDefaults: { fontSize: '10' }, + totalChunks: counts[page - 1], + requestedChunks: regions.map((region) => ({ + chunkIndex: region - 1, + data: nodeData(page, region), + })), +}) +const result = (value: unknown) => ({ + content: [{ type: 'text' as const, text: JSON.stringify(value) }], + structuredContent: { widgetInstance: { toolName: 'fixture', id: 'widget-only' } }, + isError: false, +}) +const failed = () => ({ + content: [{ type: 'text' as const, text: 'private-provider-error-sentinel' }], + isError: true, +}) +const toolNames = ['search', 'fetch', 'lucid_search_document', 'lucid_get_document_metadata'] +const protocol = new Server( + { name: 'synthetic-lucid', version: '1.0.0' }, + { capabilities: { tools: {} } } +) +protocol.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: toolNames.map((name) => ({ name, inputSchema: { type: 'object' as const } })), +})) +protocol.setRequestHandler(CallToolRequestSchema, async (request) => { + const { name, arguments: args = {} } = request.params + calls++ + requests.push({ tool: name, status: mode || 'success' }) + assert(calls <= 12, 'Exceeded the production per-read MCP request budget') + if (mode === 'tool-error') return failed() + if (name === 'lucid_get_document_metadata') { + metadataReads++ + if (args.document_id === ID && mode === 'candidate-error') return failed() + if (args.document_id === ID && mode === 'candidate-rate') + return { ...failed(), content: [{ type: 'text' as const, text: 'Rate limit reached' }] } + if (mode === 'revoked' && metadataReads > 1) return failed() + return result({ + documentId: + mode === 'metadata-id' || (mode === 'stale-candidate' && args.document_id === ID) + ? OTHER_ID + : args.document_id, + title: TITLE, + product: mode === 'spark' ? 'lucidspark' : 'lucidchart', + viewUrl: + mode === 'unsafe-url' + ? 'https://attacker.invalid/other' + : url(String(args.document_id), mode === 'spark' ? 'lucidspark' : 'lucidchart'), + lastModified: '2026-09-01T12:00:00Z', + created: '2020-01-01T00:00:00Z', + version: mode === 'changed' && metadataReads > 1 ? 8 : 7, + pageCount: mode === 'metadata-pages' ? 3 : counts.length, + canEdit: false, + trashed: mode === 'trashed' ? true : null, + }) + } + if (name === 'search') { + assert.equal(typeof args.query, 'string') + assert( + Array.isArray(args.product) && + args.product.every((product) => product === 'lucidchart' || product === 'lucidspark') + ) + assert( + Object.keys(args).every((key) => ['query', 'product', 'last_modified_after'].includes(key)) + ) + const rowCount = + mode === 'search-cap' + ? 200 + : ['stale-candidate', 'candidate-error', 'candidate-rate'].includes(mode) + ? 2 + : 1 + return result({ + query: args.query, + results: + args.query === 'absent' + ? [] + : Array.from({ length: rowCount }, (_, index) => { + const id = `00000000-0000-4000-8000-${String(index + 1).padStart(12, '0')}` + return { id, title: TITLE, url: url(id), parent: null } + }), + }) + } + if (name === 'lucid_search_document') { + assert.equal(args.id, ID) + assert(Array.isArray(args.queries) && args.queries.every((query) => typeof query === 'string')) + return result({ + document_id: mode === 'scoped-id' ? OTHER_ID : ID, + title: TITLE, + edit_url: url(), + matches: Object.fromEntries( + (args.queries as string[]).map((query) => [ + query, + query === 'absent' + ? [] + : [ + { + pageIndex: mode === 'scoped-page' ? 3 : 2, + regionIndex: 1, + context: ['API Gateway'], + }, + ], + ]) + ), + }) + } + assert.equal(name, 'fetch') + assert.equal(args.id, ID) + const manifest = { + document_id: mode === 'content-id' ? OTHER_ID : ID, + title: TITLE, + edit_url: url(ID, mode === 'spark' ? 'lucidspark' : 'lucidchart'), + metadata: { + page_count: counts.length, + page_region_counts: mode === 'fractional' ? [1.5, 1] : counts, + }, + } + if (args.metadata_only) return result(manifest) + contentCalls++ + if (mode === 'cancel') { + enteredContent?.() + await new Promise((resolve) => { + blockedContent = resolve + }) + } + assert(typeof args.page_index === 'number') + const page = args.page_index + assert(Number.isInteger(page) && page >= 1 && page <= counts.length) + const regions = Array.isArray(args.region_index) ? args.region_index : [] + assert( + regions.every( + (region) => Number.isInteger(region) && region >= 1 && region <= counts[page - 1]! + ) + ) + const data = fixturePage(page, regions) + if (mode === 'duplicate-page-id') data.pageId = 'page-1' + if (mode === 'wrong-page') data.pageIndex++ + if (mode === 'wrong-region' && data.requestedChunks[0]) data.requestedChunks[0].chunkIndex++ + if (mode === 'missing-region') data.requestedChunks = [] + if (mode === 'duplicate-region' && data.requestedChunks[0]) + data.requestedChunks.push(data.requestedChunks[0]) + if (mode === 'changed-page-id' && regions[0] === 2) data.pageId = 'replacement-page' + return result({ + ...manifest, + page_id: data.pageId, + page_index: page, + metadata: { ...manifest.metadata, page_index: page }, + text: mode === 'malformed-json' ? '{invalid' : JSON.stringify({ pages: [data] }), + }) +}) +const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: generateId, + enableJsonResponse: true, +}) +await protocol.connect(transport) +const server = http.createServer((request, response) => { + if (request.url !== '/mcp') { + observerRequests++ + response.writeHead(404).end() + return + } + void transport.handleRequest(request, response).catch(() => { + if (!response.headersSent) response.writeHead(500) + response.end() + }) +}) +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) +const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}` +const client = new McpClient({ + config: { + id: 'synthetic-lucid', + name: 'Synthetic Lucid', + transport: 'streamable-http', + url: `${origin}/mcp`, + authType: 'none', + }, + resolvedIP: '127.0.0.1', + securityPolicy: { requireConsent: false, auditLevel: 'none' }, +}) +let signal = new AbortController().signal +const reader: ManagedSearchMcpClient = { + async call(name, args) { + signal.throwIfAborted() + assert(toolNames.includes(name)) + return managedMcpPayload( + await client.callTool({ name, arguments: args }, { signal, timeoutMs: 5000 }), + 'Lucid' + ) + }, +} +const read = (revision = '7') => readLucidMcp(reader, { id: ID, revision }) +const search = (query = 'topology') => searchLucidMcp(reader, { query, scopes: [], limit: 10 }) +async function check(name: string, run: () => Promise) { + mode = '' + counts = [2, 1] + metadataReads = 0 + contentCalls = 0 + calls = 0 + padding = '' + signal = new AbortController().signal + const start = performance.now() + try { + await run() + checks.push({ name, status: 'passed', durationMs: performance.now() - start }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: performance.now() - start, + error: getErrorMessage(error), + }) + process.exitCode = 1 + } +} +try { + await client.connect() + await client.listTools() + await check( + 'Real MCP widget envelope preserves title result and modification timestamp', + async () => { + const page = await search() + assert.deepEqual( + page.documents.map((document) => document.id), + [ID] + ) + assert.equal(page.documents[0]?.modifiedAt, '2026-09-01T12:00:00Z') + assert.equal((await search('absent')).documents.length, 0) + } + ) + for (const product of ['chart', 'spark']) + await check( + `${product} complete read preserves all regions and directed edges without fetching links`, + async () => { + mode = product === 'spark' ? 'spark' : '' + const document = await read() + const data = JSON.parse(document.content) + assert.equal(document.kind, product === 'spark' ? 'lucidspark' : 'lucidchart') + assert.equal(data.pages.length, 2) + assert.deepEqual( + data.pages.map((page: Record) => page.pageId), + ['page-1', 'page-2'] + ) + assert.equal(data.pages[1].requestedChunks[0].data.nodes[0].label, 'Page 2 region 1 — café') + assert.equal(data.pages[0].requestedChunks[1].data.edges[0].sourceId, 'api') + assert.equal(data.pages[0].requestedChunks[1].data.edges[0].targetId, 'database') + assert.deepEqual(data.pages[1].requestedChunks[0].data.customDiagramFamily.preserved, [ + 'custom data', + 'group membership', + ]) + assert.equal(observerRequests, 0) + assert(calls <= 12) + } + ) + await check('Scoped text search binds matches to the requested document', async () => { + const page = await searchLucidMcp(reader, { + query: 'API Gateway', + scopes: [], + limit: 10, + native: { provider: 'lucid', query: 'API Gateway', project: url() }, + }) + assert.equal(page.documents[0]?.id, ID) + assert(page.documents[0]?.content.includes('Page 2, region 1: API Gateway')) + mode = 'scoped-id' + await assert.rejects( + () => + searchLucidMcp(reader, { + query: 'API Gateway', + scopes: [], + limit: 10, + native: { provider: 'lucid', query: 'API Gateway', project: ID }, + }), + /identity/ + ) + }) + await check('Scoped text cannot claim a nonexistent page', async () => { + mode = 'scoped-page' + await assert.rejects( + () => + searchLucidMcp(reader, { + query: 'API', + scopes: [], + limit: 10, + native: { provider: 'lucid', query: 'API', project: ID }, + }), + /document-search/ + ) + }) + for (const failure of [ + 'metadata-id', + 'unsafe-url', + 'trashed', + 'metadata-pages', + 'duplicate-page-id', + 'content-id', + 'fractional', + 'wrong-page', + 'wrong-region', + 'missing-region', + 'duplicate-region', + 'changed-page-id', + 'malformed-json', + 'changed', + 'revoked', + 'tool-error', + ]) + await check(`Withhold complete read on ${failure}`, async () => { + mode = failure + await assert.rejects( + read, + (error) => + error instanceof NativeSearchError && + !error.message.includes('private-provider-error-sentinel') + ) + }) + await check('Older reference revision cannot splice a new read window', async () => { + await assert.rejects(() => read('6'), /changed/) + }) + await check('Manifest preflight rejects over-budget reads before fetching content', async () => { + counts = [8, 1] + await assert.rejects(read, /limit/) + assert.equal(contentCalls, 0) + }) + await check('All eight regions fit the existing request budget', async () => { + counts = [4, 4] + const document = await read() + assert.equal(JSON.parse(document.content).pages[1].requestedChunks.length, 4) + assert(calls <= 12) + }) + await check('Complete UTF8 output at cap succeeds; one extra byte fails', async () => { + counts = [1] + const expected = { document_id: ID, title: TITLE, pages: [fixturePage(1, [1])] } + const remaining = 512 * 1024 - Buffer.byteLength(JSON.stringify(expected), 'utf8') + padding = 'é'.repeat(Math.floor(remaining / 2)) + 'x'.repeat(remaining % 2) + assert.equal(Buffer.byteLength((await read()).content, 'utf8'), 512 * 1024) + calls = 0 + metadataReads = 0 + padding += 'x' + await assert.rejects(read, /512 KiB/) + }) + await check( + 'A stale candidate cannot discard another independently readable document', + async () => { + mode = 'stale-candidate' + const page = await search() + assert.deepEqual( + page.documents.map((document) => document.id), + [OTHER_ID] + ) + assert.equal(page.partial, true) + assert.match(page.message ?? '', /excluded/i) + } + ) + for (const [failure, status] of [ + ['candidate-error', 'unavailable'], + ['candidate-rate', 'rate_limited'], + ] as const) { + await check(`Candidate provider failure ${status} remains terminal`, async () => { + mode = failure + await assert.rejects( + () => search(), + (error: unknown) => error instanceof NativeSearchError && error.status === status + ) + }) + } + await check('Capped title search discloses coverage without inventing a cursor', async () => { + mode = 'search-cap' + const page = await search() + assert.equal(page.documents.length, 10) + assert.equal(page.partial, true) + assert.equal(page.hasMore, true) + assert.equal(page.nextCursor, undefined) + assert(calls <= 12) + }) + await check('Unsupported cursor and oversized terms fail before a provider request', async () => { + await assert.rejects(() => search(''), /requires search terms/) + await assert.rejects(() => search('x'.repeat(401)), /400/) + await assert.rejects( + () => + searchLucidMcp(reader, { + query: 'a', + scopes: [], + limit: 10, + native: { provider: 'lucid', query: 'a', cursor: 'opaque' }, + }), + /continuation/ + ) + assert.equal(calls, 0) + }) + await check( + 'Cancellation during a content request cannot return a complete document', + async () => { + mode = 'cancel' + const controller = new AbortController() + signal = controller.signal + const arrived = new Promise((resolve) => { + enteredContent = resolve + }) + const reading = read() + const rejection = assert.rejects(reading) + await Promise.race([ + arrived, + reading.then(() => { + throw new Error('Read finished without reaching the held content request') + }), + ]) + controller.abort(new Error('cancelled Lucid verification')) + blockedContent?.() + await rejection + assert.equal(contentCalls, 1) + } + ) +} finally { + blockedContent?.() + await client.disconnect() + await protocol.close() + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + await mkdir(dirname(reportPath), { recursive: true }) + await writeFile( + reportPath, + JSON.stringify({ fixture: 'synthetic-loopback-mcp', checks, requests }, null, 2) + ) + logger.info('Lucid MCP verification finished', { + passed: checks.filter((check) => check.status === 'passed').length, + failed: checks.filter((check) => check.status === 'failed').length, + reportPath, + }) +}