From cde225694e50abb0874bec4a777399f76da4c271 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 26 Sep 2026 10:28:55 -0700 Subject: [PATCH 1/3] fix(knowledge): count documents through the reader's access and only where counts are shown --- apps/sim/app/api/knowledge/route.ts | 6 +- apps/sim/app/api/v1/knowledge/[id]/route.ts | 23 ++- .../app/api/v1/knowledge/route.integration.ts | 137 +++++++++++++ apps/sim/app/api/v1/knowledge/route.ts | 14 +- apps/sim/app/api/v1/knowledge/utils.ts | 7 +- .../knowledge/hooks/use-knowledge-upload.ts | 4 +- .../[workspaceId]/knowledge/knowledge.tsx | 5 +- .../[workspaceId]/knowledge/prefetch.ts | 12 +- apps/sim/hooks/kb/use-knowledge.ts | 4 +- apps/sim/hooks/queries/kb/connectors.ts | 2 +- apps/sim/hooks/queries/kb/knowledge.ts | 22 +- .../sim/hooks/queries/utils/knowledge-keys.ts | 8 + apps/sim/lib/api/contracts/knowledge/base.ts | 4 +- apps/sim/lib/api/contracts/v2/knowledge.ts | 1 + .../search-index-policy.integration.ts | 2 +- apps/sim/lib/knowledge/api/internal-route.ts | 6 +- .../knowledge/application/contexts.test.ts | 6 - .../sim/lib/knowledge/application/contexts.ts | 9 +- .../application/knowledge-bases.test.ts | 6 +- .../knowledge/application/knowledge-bases.ts | 27 ++- apps/sim/lib/knowledge/service.test.ts | 2 +- apps/sim/lib/knowledge/service.ts | 194 ++++++++---------- apps/sim/lib/knowledge/types.ts | 5 +- .../tools/client/resource-display.ts | 6 +- .../src/mocks/knowledge-service.mock.ts | 4 +- 25 files changed, 339 insertions(+), 177 deletions(-) create mode 100644 apps/sim/app/api/v1/knowledge/route.integration.ts diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index 40ef69cb20f..758c9fbfbc0 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -33,7 +33,11 @@ export const GET = defineInternalJsonRoute({ parseOptions: { validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid query parameters'), }, - mapInput: ({ query }) => ({ workspaceId: query.workspaceId, scope: query.scope }), + mapInput: ({ query }) => ({ + workspaceId: query.workspaceId, + scope: query.scope, + includeCounts: query.includeCounts, + }), useCase: listInternalKnowledgeBases, present: internalKnowledgePresenters.list, }) diff --git a/apps/sim/app/api/v1/knowledge/[id]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/route.ts index 3dda06cd9fd..79d0043e9c8 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/route.ts @@ -14,10 +14,12 @@ import { performDeleteKnowledgeBase, performUpdateKnowledgeBase, } from '@/lib/knowledge/orchestration' +import { attachKnowledgeBaseConnectors } from '@/lib/knowledge/service' import { formatKnowledgeBase, handleError, resolveKnowledgeBase, + resolveV1KnowledgeReadAccess, } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' @@ -41,19 +43,18 @@ export const GET = withRouteHandler(async (request: NextRequest, context: Knowle if (!parsed.success) return parsed.response const { id } = parsed.data.params - const result = await resolveKnowledgeBase( - id, - parsed.data.query.workspaceId, - userId, - rateLimit, - 'knowledge.use' - ) + const { workspaceId } = parsed.data.query + const result = await resolveKnowledgeBase(id, workspaceId, userId, rateLimit, 'knowledge.use') if (result instanceof NextResponse) return result + const knowledgeBase = await attachKnowledgeBaseConnectors( + result.kb, + await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) + ) return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(result.kb), + knowledgeBase: formatKnowledgeBase(knowledgeBase), }, }) } catch (error) { @@ -102,10 +103,14 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle ) } + const knowledgeBase = await attachKnowledgeBaseConnectors( + outcome.knowledgeBase, + await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) + ) return NextResponse.json({ success: true, data: { - knowledgeBase: formatKnowledgeBase(outcome.knowledgeBase), + knowledgeBase: formatKnowledgeBase(knowledgeBase), message: 'Knowledge base updated successfully', }, }) diff --git a/apps/sim/app/api/v1/knowledge/route.integration.ts b/apps/sim/app/api/v1/knowledge/route.integration.ts new file mode 100644 index 00000000000..4e8b63ce120 --- /dev/null +++ b/apps/sim/app/api/v1/knowledge/route.integration.ts @@ -0,0 +1,137 @@ +/** + * Knowledge-base document totals against real PostgreSQL: the public v1 list and detail count + * only the documents their caller can read, and the internal list reads no document at all + * unless the caller asks for totals. + */ +import type { Principal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { document, organization, user, workspace } from '@sim/db/schema' +import { createMockRequest } from '@sim/testing' +import { generateId } from '@sim/utils/id' +import { eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const caller = vi.hoisted(() => ({ userId: '' })) + +vi.mock('@/app/api/v1/middleware', async (importOriginal) => ({ + ...(await importOriginal()), + authenticateRequest: async () => ({ + requestId: 'fixture-request', + userId: caller.userId, + rateLimit: { + allowed: true, + remaining: 1, + limit: 1, + resetAt: new Date(), + userId: caller.userId, + keyType: 'personal', + }, + }), +})) + +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { listInternalKnowledgeBases } from '@/lib/knowledge/application/knowledge-bases' +import { GET as getKnowledgeBase } from '@/app/api/v1/knowledge/[id]/route' +import { GET as listKnowledgeBases } from '@/app/api/v1/knowledge/route' + +const ids = createKnowledgeAclFixtureIds() +const reader: Principal = { kind: 'session', userId: ids.bobId, sessionId: 'fixture-reader' } + +describe('knowledge-base document totals in PostgreSQL', () => { + beforeAll(async () => { + vi.stubGlobal('fetch', async () => { + throw new Error('Unexpected provider request in knowledge-base count tests') + }) + caller.userId = ids.bobId + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) + await db.insert(document).values([ + { + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + filename: 'Uploaded handbook', + fileUrl: 'https://fixture.test/uploaded', + fileSize: 10, + mimeType: 'text/plain', + tokenCount: 10, + processingStatus: 'completed', + }, + { + id: generateId(), + knowledgeBaseId: ids.knowledgeBaseId, + filename: 'Private source document', + fileUrl: 'https://fixture.test/private', + fileSize: 20, + mimeType: 'text/plain', + tokenCount: 20, + processingStatus: 'completed', + connectorId: ids.connectorId, + externalId: 'private-fixture', + contentHash: 'fixture', + acl: [`u:${ids.aliceId}@fixture.test`], + aclVerifiedAt: new Date(), + }, + ]) + }) + + afterAll(async () => { + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + vi.unstubAllGlobals() + }) + + it('lists only the documents a v1 caller can read', async () => { + const response = await listKnowledgeBases( + createMockRequest( + 'GET', + undefined, + {}, + `http://localhost/api/v1/knowledge?workspaceId=${ids.workspaceId}` + ), + { params: Promise.resolve({}) } + ) + expect(response.status).toBe(200) + const { data } = await response.json() + expect(data.knowledgeBases).toEqual([ + expect.objectContaining({ id: ids.knowledgeBaseId, docCount: 1, tokenCount: 10 }), + ]) + }) + + it('details only the documents a v1 caller can read', async () => { + const response = await getKnowledgeBase( + createMockRequest( + 'GET', + undefined, + {}, + `http://localhost/api/v1/knowledge/${ids.knowledgeBaseId}?workspaceId=${ids.workspaceId}` + ), + { params: Promise.resolve({ id: ids.knowledgeBaseId }) } + ) + expect(response.status).toBe(200) + const { data } = await response.json() + expect(data.knowledgeBase).toMatchObject({ + id: ids.knowledgeBaseId, + docCount: 1, + tokenCount: 10, + }) + }) + + it('omits totals from the internal list unless the caller asks for them', async () => { + const input = { workspaceId: ids.workspaceId, scope: 'active' } as const + const [plain] = (await listInternalKnowledgeBases.execute({ principal: reader, input })) + .knowledgeBases + expect(plain).not.toHaveProperty('docCount') + expect(plain).not.toHaveProperty('tokenCount') + + const [counted] = ( + await listInternalKnowledgeBases.execute({ + principal: reader, + input: { ...input, includeCounts: true }, + }) + ).knowledgeBases + expect(counted).toMatchObject({ docCount: 1, tokenCount: 10 }) + }) +}) diff --git a/apps/sim/app/api/v1/knowledge/route.ts b/apps/sim/app/api/v1/knowledge/route.ts index 51ec8ab741d..67bfd1eafaf 100644 --- a/apps/sim/app/api/v1/knowledge/route.ts +++ b/apps/sim/app/api/v1/knowledge/route.ts @@ -11,7 +11,11 @@ import { import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { performCreateKnowledgeBase } from '@/lib/knowledge/orchestration' import { getWorkspaceKnowledgeBases } from '@/lib/knowledge/service' -import { formatKnowledgeBase, handleError } from '@/app/api/v1/knowledge/utils' +import { + formatKnowledgeBase, + handleError, + resolveV1KnowledgeReadAccess, +} from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse, @@ -48,9 +52,11 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ) if (accessError) return accessError - /** Read only after `validateWorkspaceAccess` authorized this caller; same list the - * internal surface serves, from the same place. */ - const { data: knowledgeBases } = await getWorkspaceKnowledgeBases(workspaceId) + /** Read only after `validateWorkspaceAccess` authorized this caller, and totalled as the + * caller reads, exactly as the v1 document routes list. */ + const { data: knowledgeBases } = await getWorkspaceKnowledgeBases(workspaceId, 'active', { + countsFor: await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId), + }) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/utils.ts b/apps/sim/app/api/v1/knowledge/utils.ts index 34fcc72784c..af317955115 100644 --- a/apps/sim/app/api/v1/knowledge/utils.ts +++ b/apps/sim/app/api/v1/knowledge/utils.ts @@ -6,7 +6,8 @@ import { WORKSPACE_ACCESS_SCOPE, } from '@/lib/knowledge/access/scope' import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' -import { getKnowledgeBaseById } from '@/lib/knowledge/service' +import type { ActiveKnowledgeBaseReference } from '@/lib/knowledge/knowledge-base-reference' +import { getActiveKnowledgeBaseReference } from '@/lib/knowledge/service' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { type RateLimitResult, @@ -32,7 +33,7 @@ export async function resolveKnowledgeBase( rateLimit: RateLimitResult, capability: V1RouteCapability, level: 'read' | 'write' = 'read' -): Promise<{ kb: KnowledgeBaseWithCounts } | NextResponse> { +): Promise<{ kb: ActiveKnowledgeBaseReference } | NextResponse> { const accessError = await validateWorkspaceAccess( rateLimit, userId, @@ -42,7 +43,7 @@ export async function resolveKnowledgeBase( ) if (accessError) return accessError - const kb = await getKnowledgeBaseById(id) + const kb = await getActiveKnowledgeBaseReference(id) if (!kb) { return NextResponse.json({ error: 'Knowledge base not found' }, { status: 404 }) } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index 1113e13cbc8..10a543c09ac 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -120,11 +120,11 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { }) } - /** Reconciles both caches an upload moves: the base's documents and the list's `docCount`. */ + /** Reconciles both caches an upload moves: the base's documents and the counted `docCount`. */ const invalidateKnowledgeCaches = async (knowledgeBaseId: string) => { await Promise.all([ queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId) }), - queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }), + queryClient.invalidateQueries({ queryKey: knowledgeKeys.countedLists() }), ]) } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 051acdec1f8..bc26441338b 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -217,7 +217,10 @@ function KnowledgeContent() { } }, [permissionConfig.hideKnowledgeBaseTab, router, workspaceId]) - const { knowledgeBases, isLoading, isPlaceholderData, error } = useKnowledgeBasesList(workspaceId) + const { knowledgeBases, isLoading, isPlaceholderData, error } = useKnowledgeBasesList( + workspaceId, + { includeCounts: true } + ) const { data: members } = useWorkspaceMembersQuery(workspaceId) /** * Indexed once: `ownerCell` resolves a member per row, so passing the raw array makes the diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts index f5080114a9c..7801afdefd9 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts @@ -12,10 +12,10 @@ import { prefetchResourceListChrome } from '@/app/workspace/[workspaceId]/lib/pr import { KNOWLEDGE_BASE_LIST_STALE_TIME, knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' /** - * Prefetches the workspace's knowledge-bases list AND its knowledge-base folder tree — plus - * the pinned ids and members {@link prefetchResourceListChrome} covers — under - * the same query keys the client `useKnowledgeBasesQuery` / `useFolders` hooks use (scope - * `active`), so the list paints populated on first render. + * Prefetches the workspace's knowledge-bases list with its document totals AND its + * knowledge-base folder tree — plus the pinned ids and members {@link prefetchResourceListChrome} + * covers — under the same query keys the Knowledge page's `useKnowledgeBasesQuery` (scope + * `active`, counted) and `useFolders` hooks use, so the list paints populated on first render. * * Both are needed: a base row is only placed correctly relative to the folder rows it sits * beside, so prefetching one without the other still flashes an ungrouped list — and a @@ -45,12 +45,12 @@ export async function prefetchKnowledgeBases( await Promise.all([ queryClient.prefetchQuery({ - queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryKey: knowledgeKeys.countedList(workspaceId, 'active'), queryFn: async () => { const principal = await internalSessionAuth.authenticate() const result = await listInternalKnowledgeBases.execute({ principal, - input: { workspaceId, scope: 'active' }, + input: { workspaceId, scope: 'active', includeCounts: true }, }) return listKnowledgeBasesContract.response.schema.parse( internalKnowledgePresenters.list(result) diff --git a/apps/sim/hooks/kb/use-knowledge.ts b/apps/sim/hooks/kb/use-knowledge.ts index 30f94c98691..bc5baf10de2 100644 --- a/apps/sim/hooks/kb/use-knowledge.ts +++ b/apps/sim/hooks/kb/use-knowledge.ts @@ -178,8 +178,8 @@ export function useKnowledgeBaseDocuments( * Hook to fetch and manage knowledge bases list * Uses React Query as single source of truth */ -export function useKnowledgeBasesList(workspaceId?: string) { - const query = useKnowledgeBasesQuery(workspaceId) +export function useKnowledgeBasesList(workspaceId?: string, options?: { includeCounts?: boolean }) { + const query = useKnowledgeBasesQuery(workspaceId, options) return { knowledgeBases: query.data ?? [], diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index bdfb1fb6a0a..6347604ac6c 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -376,7 +376,7 @@ export function useUpdateConnector() { queryKey: knowledgeKeys.detail(knowledgeBaseId), exact: true, }) - queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.countedLists() }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.searches() }) } }, diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 7f7258b6dfc..ce2833fd81c 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -98,10 +98,11 @@ export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 export async function fetchKnowledgeBases( workspaceId?: string, scope: KnowledgeQueryScope = 'active', - signal?: AbortSignal + signal?: AbortSignal, + includeCounts = false ): Promise { const result = await requestJson(listKnowledgeBasesContract, { - query: { workspaceId, scope }, + query: { workspaceId, scope, includeCounts }, signal, }) @@ -238,12 +239,17 @@ export function useKnowledgeBasesQuery( options?: { enabled?: boolean scope?: KnowledgeQueryScope + /** Adds each base's `docCount` and `tokenCount`, for the one surface that renders them. */ + includeCounts?: boolean } ) { const scope = options?.scope ?? 'active' + const includeCounts = options?.includeCounts ?? false return useQuery({ - queryKey: knowledgeKeys.list(workspaceId, scope), - queryFn: ({ signal }) => fetchKnowledgeBases(workspaceId, scope, signal), + queryKey: includeCounts + ? knowledgeKeys.countedList(workspaceId, scope) + : knowledgeKeys.list(workspaceId, scope), + queryFn: ({ signal }) => fetchKnowledgeBases(workspaceId, scope, signal, includeCounts), enabled: options?.enabled ?? true, staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME, }) @@ -590,9 +596,9 @@ export function useDeleteDocument() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), }) - /** The knowledge-base list rows carry `docCount`, so removing a document changes them too. */ + /** The counted list rows carry `docCount`, so removing a document changes them too. */ queryClient.invalidateQueries({ - queryKey: knowledgeKeys.lists(), + queryKey: knowledgeKeys.countedLists(), }) }, }) @@ -632,10 +638,10 @@ export function useBulkDocumentOperation() { queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), }) - /** Only a bulk delete changes the `docCount` the knowledge-base list rows render. */ + /** Only a bulk delete changes the `docCount` the counted list rows render. */ if (operation === 'delete') { queryClient.invalidateQueries({ - queryKey: knowledgeKeys.lists(), + queryKey: knowledgeKeys.countedLists(), }) } }, diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index 781f912b4f2..944b2783986 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -29,6 +29,14 @@ export const knowledgeKeys = { lists: () => [...knowledgeKeys.all, 'list'] as const, list: (workspaceId?: string, scope: KnowledgeQueryScope = 'active') => [...knowledgeKeys.lists(), workspaceId ?? 'all', scope] as const, + /** + * Lists carrying document totals, which only the Knowledge page renders. Under `lists()` so a + * knowledge-base mutation refreshes them with the plain lists; beside `list()` so a document + * mutation refreshes only these and never the pickers. + */ + countedLists: () => [...knowledgeKeys.lists(), 'counted'] as const, + countedList: (workspaceId?: string, scope: KnowledgeQueryScope = 'active') => + [...knowledgeKeys.countedLists(), workspaceId ?? 'all', scope] as const, details: () => [...knowledgeKeys.all, 'detail'] as const, detail: (knowledgeBaseId?: string) => [...knowledgeKeys.details(), knowledgeBaseId ?? ''] as const, diff --git a/apps/sim/lib/api/contracts/knowledge/base.ts b/apps/sim/lib/api/contracts/knowledge/base.ts index fd09ef0f463..732d7276539 100644 --- a/apps/sim/lib/api/contracts/knowledge/base.ts +++ b/apps/sim/lib/api/contracts/knowledge/base.ts @@ -26,6 +26,8 @@ export type KnowledgeScope = z.output export const listKnowledgeBasesQuerySchema = z.object({ workspaceId: z.string().min(1).optional(), scope: knowledgeScopeSchema.default('active'), + /** Adds `docCount` and `tokenCount` for the documents the caller can see; costs a document scan. */ + includeCounts: booleanQueryFlagSchema.optional().default(false), }) /** @@ -193,7 +195,7 @@ export const knowledgeBaseDataSchema = z name: z.string(), isSearchIndex: z.boolean().optional(), description: z.string().nullable(), - tokenCount: z.number(), + tokenCount: z.number().optional(), embeddingModel: z.string(), embeddingDimension: z.number(), chunkingConfig: knowledgeChunkingConfigSchema, diff --git a/apps/sim/lib/api/contracts/v2/knowledge.ts b/apps/sim/lib/api/contracts/v2/knowledge.ts index 3aed6d903f0..f23869161d3 100644 --- a/apps/sim/lib/api/contracts/v2/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/knowledge.ts @@ -139,6 +139,7 @@ export const v2KnowledgeBaseSchema = knowledgeBaseDataSchema .describe('Knowledge base description, or null when none is set.') .meta({ examples: ['All product documentation and guides'] }), tokenCount: knowledgeBaseDataSchema.shape.tokenCount + .unwrap() .describe('Total tokens across indexed documents.') .meta({ examples: [48213] }), embeddingModel: knowledgeBaseDataSchema.shape.embeddingModel diff --git a/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts b/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts index 184fb5524e9..75880eba264 100644 --- a/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts @@ -138,7 +138,7 @@ describe('canonical search knowledge-base policy', () => { ( await listInternalKnowledgeBases.execute({ principal, - input: { workspaceId: ids.workspaceId, scope: 'active' }, + input: { workspaceId: ids.workspaceId, scope: 'active', includeCounts: true }, }) ).knowledgeBases[0] ).toMatchObject({ isSearchIndex: true, docCount, tokenCount }) diff --git a/apps/sim/lib/knowledge/api/internal-route.ts b/apps/sim/lib/knowledge/api/internal-route.ts index 10c0ebcf729..0b379323e96 100644 --- a/apps/sim/lib/knowledge/api/internal-route.ts +++ b/apps/sim/lib/knowledge/api/internal-route.ts @@ -29,7 +29,7 @@ import type { KnowledgeBaseResult, } from '@/lib/knowledge/application/knowledge-bases' import type { CreatedKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' -import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import type { KnowledgeBaseSummary } from '@/lib/knowledge/types' import { captureServerEvent } from '@/lib/posthog/server' import type { UploadSessionRecord } from '@/lib/uploads/upload-session/service' @@ -216,7 +216,7 @@ export function toInternalKnowledgeDocumentUpload( } } -function toInternalKnowledgeBase(knowledgeBase: KnowledgeBaseWithCounts): KnowledgeBaseData { +function toInternalKnowledgeBase(knowledgeBase: KnowledgeBaseSummary): KnowledgeBaseData { return { ...knowledgeBase, chunkingConfig: { ...knowledgeBase.chunkingConfig }, @@ -227,7 +227,7 @@ function toInternalKnowledgeBase(knowledgeBase: KnowledgeBaseWithCounts): Knowle } export const internalKnowledgePresenters = { - list({ knowledgeBases }: { knowledgeBases: KnowledgeBaseWithCounts[] }) { + list({ knowledgeBases }: { knowledgeBases: KnowledgeBaseSummary[] }) { return { success: true as const, data: knowledgeBases.map(toInternalKnowledgeBase) } }, create({ knowledgeBase }: KnowledgeBaseResult) { diff --git a/apps/sim/lib/knowledge/application/contexts.test.ts b/apps/sim/lib/knowledge/application/contexts.test.ts index 0924a6354e5..7944f5cd237 100644 --- a/apps/sim/lib/knowledge/application/contexts.test.ts +++ b/apps/sim/lib/knowledge/application/contexts.test.ts @@ -62,7 +62,6 @@ import { knowledgeOperations } from '@/lib/knowledge/application/operations' const mocks = { ...hoisted, getKnowledgeBase: knowledgeServiceMockFns.mockGetActiveKnowledgeBaseReference, - getKnowledgeBaseWithCounts: knowledgeServiceMockFns.mockGetKnowledgeBaseById, getDocument: knowledgeDocumentsServiceMockFns.mockGetKnowledgeDocument, getDocumentById: knowledgeDocumentsServiceMockFns.mockGetKnowledgeDocumentById, getTag: knowledgeTagsServiceMockFns.mockGetTagDefinitionById, @@ -90,11 +89,6 @@ describe('knowledge application contexts', () => { null ) mocks.getKnowledgeBase.mockResolvedValue(knowledgeBase) - mocks.getKnowledgeBaseWithCounts.mockResolvedValue({ - ...knowledgeBase, - docCount: 3, - tokenCount: 1536, - }) workspaceContextMockFns.mockLoadActiveWorkspaceApplicationContext.mockResolvedValue(workspace) workspaceContextMockFns.mockLoadWorkspaceApplicationContext.mockResolvedValue(workspace) }) diff --git a/apps/sim/lib/knowledge/application/contexts.ts b/apps/sim/lib/knowledge/application/contexts.ts index 085f8b47447..87d7485dd35 100644 --- a/apps/sim/lib/knowledge/application/contexts.ts +++ b/apps/sim/lib/knowledge/application/contexts.ts @@ -23,10 +23,9 @@ import { getRestorableKnowledgeBase, type RestorableKnowledgeBase, } from '@/lib/knowledge/orchestration/restore' -import { getActiveKnowledgeBaseReference, getKnowledgeBaseById } from '@/lib/knowledge/service' +import { getActiveKnowledgeBaseReference } from '@/lib/knowledge/service' import { getTagDefinitionById } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' -import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { loadActiveWorkspaceApplicationContext, loadWorkspaceApplicationContext, @@ -85,7 +84,7 @@ export interface ActiveKnowledgeBaseContext extends KnowledgeWorkspaceContext, KnowledgeAccessBearingContext { knowledgeBaseId: string - knowledgeBase: KnowledgeBaseWithCounts + knowledgeBase: ActiveKnowledgeBaseReference } export type ActiveKnowledgeResourceBaseContext = KnowledgeResourceContext & @@ -117,7 +116,7 @@ export type ActiveKnowledgeChunkContext = ActiveKnowledgeDocumentContext & { /** * A knowledge base loaded regardless of `deletedAt`, for the one operation that * targets an archived row. It carries the restorable identity rather than the - * full {@link KnowledgeBaseWithCounts}, which is all the restore needs and all + * full {@link ActiveKnowledgeBaseReference}, which is all the restore needs and all * the archived read projects. */ export type ArchivedKnowledgeBaseContext = KnowledgeWorkspaceContext & { @@ -154,7 +153,7 @@ export async function resolveKnowledgeWorkspaceContext(input: { * search indexes from workspace operations are written once. */ async function requireKnowledgeBase(knowledgeBaseId: string, workspaceId: string | undefined) { - const knowledgeBase = await getKnowledgeBaseById(knowledgeBaseId) + const knowledgeBase = await getActiveKnowledgeBaseReference(knowledgeBaseId) if ( !knowledgeBase?.workspaceId || (workspaceId !== undefined && knowledgeBase.workspaceId !== workspaceId) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index 5e39e82ec1e..31c8267a3e0 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -74,7 +74,7 @@ const mocks = { createRecord: knowledgeServiceMockFns.mockCreateAuthorizedKnowledgeBase, updateRecord: knowledgeServiceMockFns.mockUpdateKnowledgeBase, deleteRecord: knowledgeServiceMockFns.mockDeleteKnowledgeBase, - getRecord: knowledgeServiceMockFns.mockGetKnowledgeBaseById, + getRecord: knowledgeServiceMockFns.mockGetActiveKnowledgeBaseReference, listRecords: knowledgeServiceMockFns.mockGetWorkspaceKnowledgeBases, attachConnectors: knowledgeServiceMockFns.mockAttachKnowledgeBaseConnectors, } @@ -152,7 +152,7 @@ describe('knowledge base application use cases', () => { it('authorizes a canonical workspace before listing its internal knowledge bases', async () => { await listInternalKnowledgeBases.execute({ principal: createSessionPrincipal(), - input: { workspaceId: 'workspace-1', scope: 'archived' }, + input: { workspaceId: 'workspace-1', scope: 'archived', includeCounts: true }, }) expect(knowledgeContextsMockFns.mockResolveKnowledgeWorkspaceContext).toHaveBeenCalledWith({ @@ -166,7 +166,7 @@ describe('knowledge base application use cases', () => { { forUpdate: undefined } ) expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived', { - access: expect.objectContaining({ get: mocks.resolveAccess }), + countsFor: expect.objectContaining({ get: mocks.resolveAccess }), }) }) diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index d85652917ba..db8677b55ac 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -45,6 +45,7 @@ import { MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE, } from '@/lib/knowledge/constants' import { getConfiguredKbEmbedding } from '@/lib/knowledge/embeddings' +import type { ActiveKnowledgeBaseReference } from '@/lib/knowledge/knowledge-base-reference' import { performDeleteKnowledgeBase, performRestoreKnowledgeBase, @@ -58,12 +59,16 @@ import { attachKnowledgeBaseConnectors, createAuthorizedKnowledgeBase, deleteKnowledgeBase, - getKnowledgeBaseById, + getActiveKnowledgeBaseReference, getWorkspaceKnowledgeBases, type KnowledgeBaseScope, updateKnowledgeBase, } from '@/lib/knowledge/service' -import type { ChunkingConfig, KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import type { + ChunkingConfig, + KnowledgeBaseSummary, + KnowledgeBaseWithCounts, +} from '@/lib/knowledge/types' const logger = createLogger('KnowledgeBaseApplication') @@ -128,10 +133,12 @@ export interface CreateKnowledgeBaseInput { export interface ListInternalKnowledgeBasesInput { workspaceId?: string scope: KnowledgeBaseScope + /** Totals each base's documents the principal can see; only the surfaces that show them ask. */ + includeCounts?: boolean } export interface ListInternalKnowledgeBasesResult { - knowledgeBases: KnowledgeBaseWithCounts[] + knowledgeBases: KnowledgeBaseSummary[] } export interface ReadKnowledgeBaseInput { @@ -210,8 +217,8 @@ function throwKnowledgeOrchestrationFailure( async function loadInternalActiveKnowledgeBase( knowledgeBaseId: string -): Promise { - const knowledgeBase = await getKnowledgeBaseById(knowledgeBaseId) +): Promise { + const knowledgeBase = await getActiveKnowledgeBaseReference(knowledgeBaseId) if (!knowledgeBase?.workspaceId || knowledgeBase.organizationId) { throw new OrchestrationError('not_found', 'Knowledge base not found') } @@ -260,7 +267,7 @@ async function executeListKnowledgeBases(args: { sortOrder: args.input.sortOrder, limit: args.input.limit, cursorKeys: args.input.cursorKeys, - access: createKnowledgeAccessProvider(args.principal, args.context), + countsFor: createKnowledgeAccessProvider(args.principal, args.context), }) return { knowledgeBases: page.data.map((knowledgeBase) => ({ @@ -438,7 +445,7 @@ export const restoreKnowledgeBase = defineAuthorizedKnowledgeUseCase({ throwKnowledgeOrchestrationFailure(outcome, 'Failed to restore knowledge base') } } - const knowledgeBase = await getKnowledgeBaseById(context.knowledgeBaseId) + const knowledgeBase = await getActiveKnowledgeBaseReference(context.knowledgeBaseId) if (!knowledgeBase) throw new OrchestrationError('not_found', 'Knowledge base not found') const index = await loadActiveFolderPathIndex( context.workspaceId, @@ -493,7 +500,11 @@ export const listInternalKnowledgeBases = { const { data: knowledgeBases } = await getWorkspaceKnowledgeBases( context.workspaceId, input.scope, - { access: createKnowledgeAccessProvider(principal, context) } + { + countsFor: input.includeCounts + ? createKnowledgeAccessProvider(principal, context) + : undefined, + } ) return { knowledgeBases } }, diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 3901ade4fa1..8e02cb151d2 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -327,7 +327,7 @@ describe('knowledge base counts with live source permissions', () => { createdAt: new Date('2026-01-01'), }, ]) - const result = await getWorkspaceKnowledgeBases('ws-1', 'archived', { access }) + const result = await getWorkspaceKnowledgeBases('ws-1', 'archived', { countsFor: access }) expect(result.data[0]).toMatchObject({ docCount: 2, tokenCount: 10 }) expect(getForConnectors).not.toHaveBeenCalled() expect(dbChainMockFns.select).not.toHaveBeenCalledWith({ diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 76463ca7f05..606c292913a 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -45,6 +45,7 @@ import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/know import type { ChunkingConfig, CreateKnowledgeBaseData, + KnowledgeBaseSummary, KnowledgeBaseWithCounts, } from '@/lib/knowledge/types' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' @@ -145,7 +146,8 @@ const KNOWLEDGE_BASE_SORTS = { } satisfies Record[]> export interface GetKnowledgeBasesOptions { - access?: KnowledgeReadAccess + /** Totals each base's documents as this reader sees them. Omitted, no document is read. */ + countsFor?: KnowledgeReadAccess /** Restrict to one knowledge-base folder; `undefined` lists all and `null` lists the root. */ folderId?: string | null /** Case-insensitive substring match on the knowledge base name. */ @@ -166,37 +168,47 @@ function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) { : isNull(knowledgeBase.deletedAt) } +/** A knowledge-base row before its connector summary is attached. */ +type KnowledgeBaseRow = Omit< + KnowledgeBaseSummary, + 'connectorTypes' | 'hasPermissionScopedConnector' +> + /** - * The one projection every knowledge-base list renders: the base's own columns plus its live - * document count. Both list queries read through here so a column added to one list can never - * be missing from the other — they are concatenated into a single rendered list. + * The base's own columns, without reading a single document. Every list shares this projection + * so a column added to one can never be missing from another. */ async function readKnowledgeBaseRows( where: SQL | undefined, orderBy: SQL[], - limit?: number, - access?: KnowledgeReadAccess + limit?: number +): Promise { + const query = db + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) + .from(knowledgeBase) + .where(where) + .orderBy(...orderBy) + const rows = limit === undefined ? await query : await query.limit(limit) + return rows.map(toActiveKnowledgeBaseReference) +} + +/** + * {@link readKnowledgeBaseRows} plus the live totals of the documents `access` admits. Only the + * surfaces that display totals pay for the document join, and they always count as a reader. + */ +async function readCountedKnowledgeBaseRows( + where: SQL | undefined, + orderBy: SQL[], + limit: number | undefined, + access: KnowledgeReadAccess ): Promise< Array> > { - const scope = access && 'get' in access ? await access.get() : access + const scope = 'get' in access ? await access.get() : access const query = db .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - isSearchIndex: knowledgeBase.isSearchIndex, - description: knowledgeBase.description, + ...ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS, tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - organizationId: knowledgeBase.organizationId, - folderId: knowledgeBase.folderId, docCount: count(document.knowledgeBaseId), }) .from(knowledgeBase) @@ -207,7 +219,7 @@ async function readKnowledgeBaseRows( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - scope ? knowledgeAccessCondition(scope) : undefined + knowledgeAccessCondition(scope) ) ) .where(where) @@ -224,7 +236,7 @@ async function readKnowledgeBaseRows( * never turns into hundreds of per-batch round trips. */ const liveCounts = - access && 'get' in access && rows.length > 0 + 'get' in access && rows.length > 0 ? await readLiveSourceDocumentCounts( limit === undefined && where ? where @@ -236,8 +248,7 @@ async function readKnowledgeBaseRows( ) : undefined return rows.map((kb) => ({ - ...kb, - chunkingConfig: kb.chunkingConfig as ChunkingConfig, + ...toActiveKnowledgeBaseReference(kb), docCount: Number(kb.docCount) + (liveCounts?.get(kb.id)?.docCount ?? 0), tokenCount: kb.tokenCount + (liveCounts?.get(kb.id)?.tokenCount ?? 0), })) @@ -300,11 +311,11 @@ async function readLiveSourceDocumentCounts( return counts } -async function attachConnectorTypes( - knowledgeBases: Array< - Omit - > -): Promise { +async function attachConnectorTypes( + knowledgeBases: Row[] +): Promise< + Array> +> { const kbIds = knowledgeBases.map((kb) => kb.id) const connectorRows = kbIds.length > 0 @@ -363,15 +374,23 @@ async function attachConnectorTypes( * authorization. Unlike the legacy user-oriented query, this never widens the * scope to workspace-less rows and never depends on a human permission join. */ -async function readWorkspaceKnowledgeBaseRows( +export async function getWorkspaceKnowledgeBases( + workspaceId: string, + scope: KnowledgeBaseScope | undefined, + options: GetKnowledgeBasesOptions & { countsFor: KnowledgeReadAccess } +): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> +export async function getWorkspaceKnowledgeBases( + workspaceId: string, + scope?: KnowledgeBaseScope, + options?: GetKnowledgeBasesOptions +): Promise<{ data: KnowledgeBaseSummary[]; nextCursorKeys: CursorKey[] | null }> +export async function getWorkspaceKnowledgeBases( workspaceId: string, - scope: KnowledgeBaseScope, + scope: KnowledgeBaseScope = 'active', options?: GetKnowledgeBasesOptions -): Promise<{ - data: Array> - nextCursorKeys: CursorKey[] | null -}> { +): Promise<{ data: KnowledgeBaseSummary[]; nextCursorKeys: CursorKey[] | null }> { const { + countsFor, folderId, search, sortBy = 'createdAt', @@ -380,6 +399,18 @@ async function readWorkspaceKnowledgeBaseRows( cursorKeys, } = options ?? {} const keys = KNOWLEDGE_BASE_SORTS[sortBy] + const where = and( + eq(knowledgeBase.workspaceId, workspaceId), + knowledgeBaseScopeCondition(scope), + folderId === undefined + ? undefined + : folderId === null + ? isNull(knowledgeBase.folderId) + : eq(knowledgeBase.folderId, folderId), + searchFilter(knowledgeBase.name, search), + resumeKeyset(keys, cursorKeys, sortOrder) + ) + const orderBy = listOrderBy(keysetColumns(keys), sortOrder) /** * An unpaged read is unbounded, matching the sibling internal lists (`listTables`, workspace @@ -388,32 +419,10 @@ async function readWorkspaceKnowledgeBaseRows( */ const readLimit = limit === undefined ? undefined : limit + 1 - const rows = await readKnowledgeBaseRows( - and( - eq(knowledgeBase.workspaceId, workspaceId), - knowledgeBaseScopeCondition(scope), - folderId === undefined - ? undefined - : folderId === null - ? isNull(knowledgeBase.folderId) - : eq(knowledgeBase.folderId, folderId), - searchFilter(knowledgeBase.name, search), - resumeKeyset(keys, cursorKeys, sortOrder) - ), - listOrderBy(keysetColumns(keys), sortOrder), - readLimit, - options?.access - ) - - return keysetPage(keys, rows, limit) -} - -export async function getWorkspaceKnowledgeBases( - workspaceId: string, - scope: KnowledgeBaseScope = 'active', - options?: GetKnowledgeBasesOptions -): Promise<{ data: KnowledgeBaseWithCounts[]; nextCursorKeys: CursorKey[] | null }> { - const page = await readWorkspaceKnowledgeBaseRows(workspaceId, scope, options) + const rows: KnowledgeBaseRow[] = countsFor + ? await readCountedKnowledgeBaseRows(where, orderBy, readLimit, countsFor) + : await readKnowledgeBaseRows(where, orderBy, readLimit) + const page = keysetPage(keys, rows, limit) return { data: await attachConnectorTypes(page.data), nextCursorKeys: page.nextCursorKeys, @@ -424,9 +433,7 @@ export async function getWorkspaceKnowledgeBases( export async function findActiveKnowledgeBasesByExactName( workspaceId: string, name: string -): Promise< - Array> -> { +): Promise { return readKnowledgeBaseRows( and( eq(knowledgeBase.workspaceId, workspaceId), @@ -944,8 +951,8 @@ export async function updateKnowledgeBase( * Display names for knowledge bases that live in `workspaceId`, keyed by id. * * Scoped by workspace in the query rather than checked afterwards, so an id belonging to another - * tenant resolves to nothing at all. Deliberately narrower than {@link getKnowledgeBaseById}, which - * joins `document` and aggregates counts — far more than a name lookup needs. + * tenant resolves to nothing at all. Deliberately narrower than + * {@link getActiveKnowledgeBaseReference}, which reads every column a use case needs. */ export async function getKnowledgeBaseNames( knowledgeBaseIds: readonly string[], @@ -1005,52 +1012,25 @@ export async function getActiveKnowledgeBaseReferences( } /** - * Get a single knowledge base by ID - */ -export async function getKnowledgeBaseById( - knowledgeBaseId: string -): Promise { - const result = await readKnowledgeBaseRows( - and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)), - [], - 1 - ) - - if (result.length === 0) { - return null - } - - return { - ...result[0], - chunkingConfig: result[0].chunkingConfig as ChunkingConfig, - docCount: Number(result[0].docCount), - connectorTypes: [], - hasPermissionScopedConnector: false, - } -} - -/** - * The knowledge base with its connector summary, for the surfaces that show - * it. Kept off {@link getKnowledgeBaseById} so every operation that only - * resolves its context does not pay for the connector read. + * The knowledge base with its connector summary and the document totals `access` can see, for + * the surfaces that show them. Kept off {@link getActiveKnowledgeBaseReference} so every + * operation that only resolves its context pays for neither the count nor the connector read. */ export async function attachKnowledgeBaseConnectors( - knowledgeBase: KnowledgeBaseWithCounts, - access?: KnowledgeReadAccess + knowledgeBase: KnowledgeBaseRow, + access: KnowledgeReadAccess ): Promise { - let visible = knowledgeBase - if (access) { - const subject = eq(document.knowledgeBaseId, knowledgeBase.id) - const scope = 'get' in access ? await access.get() : access - const [ordinary] = await countDocumentsByKnowledgeBase(subject, knowledgeAccessCondition(scope)) - const live = 'get' in access ? await readLiveSourceDocumentCounts(subject, access) : undefined - visible = { + const subject = eq(document.knowledgeBaseId, knowledgeBase.id) + const scope = 'get' in access ? await access.get() : access + const [ordinary] = await countDocumentsByKnowledgeBase(subject, knowledgeAccessCondition(scope)) + const live = 'get' in access ? await readLiveSourceDocumentCounts(subject, access) : undefined + const [withConnectors] = await attachConnectorTypes([ + { ...knowledgeBase, docCount: Number(ordinary?.docCount ?? 0) + (live?.get(knowledgeBase.id)?.docCount ?? 0), tokenCount: (ordinary?.tokenCount ?? 0) + (live?.get(knowledgeBase.id)?.tokenCount ?? 0), - } - } - const [withConnectors] = await attachConnectorTypes([visible]) + }, + ]) return withConnectors } diff --git a/apps/sim/lib/knowledge/types.ts b/apps/sim/lib/knowledge/types.ts index dfca6d3e5be..43ad5697ec8 100644 --- a/apps/sim/lib/knowledge/types.ts +++ b/apps/sim/lib/knowledge/types.ts @@ -41,6 +41,9 @@ export interface KnowledgeBaseWithCounts { hasPermissionScopedConnector: boolean } +/** A knowledge base without document totals, for every read that does not display them. */ +export type KnowledgeBaseSummary = Omit + export interface CreateKnowledgeBaseData { name: string isSearchIndex?: boolean @@ -120,7 +123,7 @@ export interface KnowledgeBaseData { name: string isSearchIndex?: boolean description: string | null - tokenCount: number + tokenCount?: number embeddingModel: string embeddingDimension: number chunkingConfig: ExtendedChunkingConfig diff --git a/apps/sim/lib/mothership/tools/client/resource-display.ts b/apps/sim/lib/mothership/tools/client/resource-display.ts index 3d3030ae287..4b04e09bd17 100644 --- a/apps/sim/lib/mothership/tools/client/resource-display.ts +++ b/apps/sim/lib/mothership/tools/client/resource-display.ts @@ -36,7 +36,11 @@ function inventoryKeys(type: NamedResource, workspaceId: string): readonly Query case 'table': return [tableKeys.list(workspaceId), tableKeys.list(workspaceId, 'archived')] case 'knowledgebase': - return [knowledgeKeys.list(workspaceId), knowledgeKeys.list(workspaceId, 'archived')] + return [ + knowledgeKeys.list(workspaceId), + knowledgeKeys.countedList(workspaceId), + knowledgeKeys.list(workspaceId, 'archived'), + ] case 'file': return [ workspaceFilesKeys.list(workspaceId), diff --git a/packages/testing/src/mocks/knowledge-service.mock.ts b/packages/testing/src/mocks/knowledge-service.mock.ts index 52bea0189ae..8e62b188ae6 100644 --- a/packages/testing/src/mocks/knowledge-service.mock.ts +++ b/packages/testing/src/mocks/knowledge-service.mock.ts @@ -49,7 +49,7 @@ export class MockKnowledgeBaseNotFoundError extends Error { * ```ts * import { knowledgeServiceMockFns } from '@sim/testing/mocks/knowledge-service.mock' * - * knowledgeServiceMockFns.mockGetKnowledgeBaseById.mockResolvedValue({ id: 'kb-1' }) + * knowledgeServiceMockFns.mockGetActiveKnowledgeBaseReference.mockResolvedValue({ id: 'kb-1' }) * ``` */ export const knowledgeServiceMockFns = { @@ -61,7 +61,6 @@ export const knowledgeServiceMockFns = { mockGetKnowledgeBaseNames: vi.fn(), mockGetActiveKnowledgeBaseReference: vi.fn(), mockGetActiveKnowledgeBaseReferences: vi.fn(), - mockGetKnowledgeBaseById: vi.fn(), mockAttachKnowledgeBaseConnectors: vi.fn(), mockDeleteKnowledgeBase: vi.fn(), mockRestoreKnowledgeBase: vi.fn(), @@ -90,7 +89,6 @@ export const knowledgeServiceMock = { getKnowledgeBaseNames: knowledgeServiceMockFns.mockGetKnowledgeBaseNames, getActiveKnowledgeBaseReference: knowledgeServiceMockFns.mockGetActiveKnowledgeBaseReference, getActiveKnowledgeBaseReferences: knowledgeServiceMockFns.mockGetActiveKnowledgeBaseReferences, - getKnowledgeBaseById: knowledgeServiceMockFns.mockGetKnowledgeBaseById, attachKnowledgeBaseConnectors: knowledgeServiceMockFns.mockAttachKnowledgeBaseConnectors, deleteKnowledgeBase: knowledgeServiceMockFns.mockDeleteKnowledgeBase, restoreKnowledgeBase: knowledgeServiceMockFns.mockRestoreKnowledgeBase, From 2286ae7e0811c86027331470971c546ac29c33bc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 26 Sep 2026 10:43:02 -0700 Subject: [PATCH 2/3] fix(knowledge): resolve the v1 reader before updating and drop unscoped update totals --- apps/sim/app/api/v1/knowledge/[id]/route.ts | 7 ++- .../[workspaceId]/lib/prefetch.test.ts | 2 +- .../orchestration/knowledge-bases.ts | 10 +++- apps/sim/lib/knowledge/service.ts | 47 +++---------------- 4 files changed, 19 insertions(+), 47 deletions(-) diff --git a/apps/sim/app/api/v1/knowledge/[id]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/route.ts index 79d0043e9c8..3ca5ec26b84 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/route.ts @@ -87,6 +87,8 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle ) if (result instanceof NextResponse) return result + /** Resolved before the write, so a reader that cannot be resolved changes nothing. */ + const access = await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) const outcome = await performUpdateKnowledgeBase({ knowledgeBaseId: id, workspaceId, @@ -103,10 +105,7 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle ) } - const knowledgeBase = await attachKnowledgeBaseConnectors( - outcome.knowledgeBase, - await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) - ) + const knowledgeBase = await attachKnowledgeBaseConnectors(outcome.knowledgeBase, access) return NextResponse.json({ success: true, data: { diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts index ea9e14da70a..604b87c7ffa 100644 --- a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -208,7 +208,7 @@ describe('workspace list prefetches', () => { await prefetchKnowledgeBases(client, WORKSPACE_ID, USER_ID) expect(mockListInternalKnowledgeBases).not.toHaveBeenCalled() - expect(client.getQueryData(knowledgeKeys.list(WORKSPACE_ID, 'active'))).toBeUndefined() + expect(client.getQueryData(knowledgeKeys.countedList(WORKSPACE_ID, 'active'))).toBeUndefined() }) }) diff --git a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts index 1114f3a356e..26caf24c2aa 100644 --- a/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/orchestration/knowledge-bases.ts @@ -4,6 +4,7 @@ import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' import { getConfiguredKbEmbedding } from '@/lib/knowledge/embeddings' +import type { ActiveKnowledgeBaseReference } from '@/lib/knowledge/knowledge-base-reference' import { auditActorFields, classifyKnowledgeFailure, @@ -25,6 +26,11 @@ export type PerformKnowledgeBaseResult = KnowledgeOrchestrationResult<{ knowledgeBase: KnowledgeBaseWithCounts }> +/** The updated base without document totals, which a surface reads as its caller. */ +export type PerformUpdateKnowledgeBaseResult = KnowledgeOrchestrationResult<{ + knowledgeBase: ActiveKnowledgeBaseReference +}> + export interface PerformCreateKnowledgeBaseParams extends KnowledgeOperationContext { workspaceId: string name: string @@ -143,7 +149,7 @@ export interface PerformUpdateKnowledgeBaseParams extends KnowledgeOperationCont */ export async function performUpdateKnowledgeBase( params: PerformUpdateKnowledgeBaseParams -): Promise { +): Promise { const { knowledgeBaseId, updates, request, source } = params const requestId = params.requestId ?? generateRequestId() @@ -154,7 +160,7 @@ export async function performUpdateKnowledgeBase( return fail('No updates specified', 'validation') } - let updated: KnowledgeBaseWithCounts + let updated: ActiveKnowledgeBaseReference try { updated = await updateKnowledgeBase(knowledgeBaseId, updates, requestId, { actorUserId: params.userId, diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 606c292913a..3a1be822afb 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -544,7 +544,8 @@ export async function createAuthorizedKnowledgeBase( } /** - * Update a knowledge base + * Updates a knowledge base and returns it without document totals; a surface that shows them + * reads them through {@link attachKnowledgeBaseConnectors} as its caller. */ export async function updateKnowledgeBase( knowledgeBaseId: string, @@ -557,7 +558,7 @@ export async function updateKnowledgeBase( }, requestId: string, options?: { actorUserId?: string; assertedWorkspaceId?: string } -): Promise { +): Promise { if (updates.workspaceId !== undefined && !updates.workspaceId) { throw new OrchestrationError('validation', 'Workspace ID is required') } @@ -890,35 +891,9 @@ export async function updateKnowledgeBase( } } - const updatedKb = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - isSearchIndex: knowledgeBase.isSearchIndex, - description: knowledgeBase.description, - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - organizationId: knowledgeBase.organizationId, - folderId: knowledgeBase.folderId, - docCount: count(document.knowledgeBaseId), - }) + const [updated] = await db + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) .from(knowledgeBase) - .leftJoin( - document, - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) .where( and( eq(knowledgeBase.id, knowledgeBaseId), @@ -928,23 +903,15 @@ export async function updateKnowledgeBase( : undefined ) ) - .groupBy(knowledgeBase.id) .limit(1) - if (updatedKb.length === 0) { + if (!updated) { throw new KnowledgeBaseNotFoundError(knowledgeBaseId) } logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`) - const [withConnectors] = await attachConnectorTypes([ - { - ...updatedKb[0], - chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig, - docCount: Number(updatedKb[0].docCount), - }, - ]) - return withConnectors + return toActiveKnowledgeBaseReference(updated) } /** From 5ef72897e322c5d31e15c73362575278b78a6c6a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 26 Sep 2026 13:16:04 -0700 Subject: [PATCH 3/3] fix(knowledge): resolve the v1 update reader where it is used and reuse the reference type --- apps/sim/app/api/v1/knowledge/[id]/route.ts | 7 ++++--- apps/sim/lib/knowledge/service.ts | 18 ++++++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/api/v1/knowledge/[id]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/route.ts index 3ca5ec26b84..79d0043e9c8 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/route.ts @@ -87,8 +87,6 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle ) if (result instanceof NextResponse) return result - /** Resolved before the write, so a reader that cannot be resolved changes nothing. */ - const access = await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) const outcome = await performUpdateKnowledgeBase({ knowledgeBaseId: id, workspaceId, @@ -105,7 +103,10 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: Knowle ) } - const knowledgeBase = await attachKnowledgeBaseConnectors(outcome.knowledgeBase, access) + const knowledgeBase = await attachKnowledgeBaseConnectors( + outcome.knowledgeBase, + await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) + ) return NextResponse.json({ success: true, data: { diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 3a1be822afb..19f8f6f453a 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -168,12 +168,6 @@ function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) { : isNull(knowledgeBase.deletedAt) } -/** A knowledge-base row before its connector summary is attached. */ -type KnowledgeBaseRow = Omit< - KnowledgeBaseSummary, - 'connectorTypes' | 'hasPermissionScopedConnector' -> - /** * The base's own columns, without reading a single document. Every list shares this projection * so a column added to one can never be missing from another. @@ -182,7 +176,7 @@ async function readKnowledgeBaseRows( where: SQL | undefined, orderBy: SQL[], limit?: number -): Promise { +): Promise { const query = db .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) .from(knowledgeBase) @@ -202,7 +196,7 @@ async function readCountedKnowledgeBaseRows( limit: number | undefined, access: KnowledgeReadAccess ): Promise< - Array> + Array> > { const scope = 'get' in access ? await access.get() : access const query = db @@ -311,7 +305,7 @@ async function readLiveSourceDocumentCounts( return counts } -async function attachConnectorTypes( +async function attachConnectorTypes( knowledgeBases: Row[] ): Promise< Array> @@ -419,7 +413,7 @@ export async function getWorkspaceKnowledgeBases( */ const readLimit = limit === undefined ? undefined : limit + 1 - const rows: KnowledgeBaseRow[] = countsFor + const rows: ActiveKnowledgeBaseReference[] = countsFor ? await readCountedKnowledgeBaseRows(where, orderBy, readLimit, countsFor) : await readKnowledgeBaseRows(where, orderBy, readLimit) const page = keysetPage(keys, rows, limit) @@ -433,7 +427,7 @@ export async function getWorkspaceKnowledgeBases( export async function findActiveKnowledgeBasesByExactName( workspaceId: string, name: string -): Promise { +): Promise { return readKnowledgeBaseRows( and( eq(knowledgeBase.workspaceId, workspaceId), @@ -984,7 +978,7 @@ export async function getActiveKnowledgeBaseReferences( * operation that only resolves its context pays for neither the count nor the connector read. */ export async function attachKnowledgeBaseConnectors( - knowledgeBase: KnowledgeBaseRow, + knowledgeBase: ActiveKnowledgeBaseReference, access: KnowledgeReadAccess ): Promise { const subject = eq(document.knowledgeBaseId, knowledgeBase.id)