From 9e58c08998064e8fbd1fb3300caf5353dedb6f23 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:53:33 +0700 Subject: [PATCH 1/9] fix(db): declare generated search_vector and full-text indexes in link schema The search_vector tsvector column and four GIN/trgm indexes exist in the database via a raw SQL migration but were never declared in the Drizzle schema. As a result any future `db:push` (a wired script) would treat them as drift and silently drop them, degrading or breaking search. Declare the generated column and indexes in `links.ts` and omit `searchVector` from the public select schemas so it cannot leak into API responses. The migration snapshot is reconciled so `db:generate` is a clean no-op rather than emitting duplicate DDL. No database change intended; this only aligns the schema source with the existing database. --- .../src/db/migrations/meta/0005_snapshot.json | 73 +++++++++++++++++++ apps/server/src/db/schemas/links.ts | 30 ++++++-- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/apps/server/src/db/migrations/meta/0005_snapshot.json b/apps/server/src/db/migrations/meta/0005_snapshot.json index e7a2cc3..f2ad3d2 100644 --- a/apps/server/src/db/migrations/meta/0005_snapshot.json +++ b/apps/server/src/db/migrations/meta/0005_snapshot.json @@ -1231,6 +1231,16 @@ "primaryKey": false, "notNull": false }, + "search_vector": { + "name": "search_vector", + "type": "tsvector", + "primaryKey": false, + "notNull": false, + "generated": { + "as": "setweight(to_tsvector('simple', COALESCE(title, '')), 'A') ||\n setweight(to_tsvector('simple', COALESCE(excerpt, '')), 'B') ||\n setweight(to_tsvector('simple', COALESCE(text_content, '')), 'C')", + "type": "stored" + } + }, "author": { "name": "author", "type": "text", @@ -1605,6 +1615,69 @@ "concurrently": false, "method": "btree", "with": {} + }, + "idx_links_search_vector": { + "name": "idx_links_search_vector", + "columns": [ + { + "expression": "search_vector", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_links_title_trgm": { + "name": "idx_links_title_trgm", + "columns": [ + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_links_url_trgm": { + "name": "idx_links_url_trgm", + "columns": [ + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_links_excerpt_trgm": { + "name": "idx_links_excerpt_trgm", + "columns": [ + { + "expression": "excerpt", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "gin_trgm_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} } }, "foreignKeys": { diff --git a/apps/server/src/db/schemas/links.ts b/apps/server/src/db/schemas/links.ts index 86c0a12..4b111da 100644 --- a/apps/server/src/db/schemas/links.ts +++ b/apps/server/src/db/schemas/links.ts @@ -1,5 +1,7 @@ +import { sql } from 'drizzle-orm'; import { boolean, + customType, index, integer, pgTable, @@ -16,6 +18,12 @@ import { importSessionsTable } from './import-sessions.js'; import { tagsTable } from './tags.js'; import { usersTable } from './users.js'; +const tsvector = customType<{ data: string; driverData: string }>({ + dataType() { + return 'tsvector'; + } +}); + export const linksTable = pgTable( 'links', { @@ -28,6 +36,11 @@ export const linksTable = pgTable( content: text('content'), textContent: text('text_content'), excerpt: text('excerpt'), + searchVector: tsvector('search_vector').generatedAlwaysAs( + sql`setweight(to_tsvector('simple', COALESCE(title, '')), 'A') || + setweight(to_tsvector('simple', COALESCE(excerpt, '')), 'B') || + setweight(to_tsvector('simple', COALESCE(text_content, '')), 'C')` + ), author: text('author'), favicon: text('favicon'), coverImage: text('cover_image'), @@ -82,7 +95,11 @@ export const linksTable = pgTable( index('idx_links_user_created').on(table.userId, table.createdAt.desc()), index('idx_links_user_last_read_at').on(table.userId, table.lastReadAt.desc()), index('idx_links_import_session_id').on(table.importSessionId), - index('idx_links_processing_started_at').on(table.processingStartedAt) + index('idx_links_processing_started_at').on(table.processingStartedAt), + index('idx_links_search_vector').using('gin', table.searchVector), + index('idx_links_title_trgm').using('gin', table.title.op('gin_trgm_ops')), + index('idx_links_url_trgm').using('gin', table.url.op('gin_trgm_ops')), + index('idx_links_excerpt_trgm').using('gin', table.excerpt.op('gin_trgm_ops')) ] ); @@ -91,7 +108,8 @@ export const selectLinksSchema = createSelectSchema(linksTable).omit({ createdAt: true, updatedAt: true, processingStartedAt: true, - importSessionId: true + importSessionId: true, + searchVector: true }); const tagWithGroupColorSchema = z.object({ @@ -128,9 +146,11 @@ export const selectLinksListSchema = createSelectSchema(linksTable) highlights: z.array(linkHighlightSchema) }); -export const selectLinksWithSignedUrlSchema = createSelectSchema(linksTable).extend({ - coverImageSignedUrl: z.string().optional() -}); +export const selectLinksWithSignedUrlSchema = createSelectSchema(linksTable) + .omit({ searchVector: true }) + .extend({ + coverImageSignedUrl: z.string().optional() + }); export const insertLinksSchema = createInsertSchema(linksTable).omit({ id: true, From 5a0f214bf923875906a44c8671e73cb3be0de174 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:54:31 +0700 Subject: [PATCH 2/9] fix(api): align create-link tag shape and drop text content from list/search The server create-link schema required a tag `id` that the web client never sent and the handler never used (tags are resolved by name + group). Remove the dead field and add a colocated contract test that fails on web/server request-shape drift. Also stop selecting `textContent` from the list and search queries and omit it from the list response schema. The heavy column is only needed by the reader detail view; returning it on every list/search row inflated payloads for no consumer. --- apps/server/src/db/schemas/links.ts | 5 +++-- apps/server/src/repositories/links.repository.ts | 7 ++++--- apps/server/src/routes/links/links.handlers.ts | 7 ++----- .../articles/api/create-link.contract.test.ts | 16 ++++++++++++++++ 4 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 apps/web/src/features/articles/api/create-link.contract.test.ts diff --git a/apps/server/src/db/schemas/links.ts b/apps/server/src/db/schemas/links.ts index 4b111da..05bbd58 100644 --- a/apps/server/src/db/schemas/links.ts +++ b/apps/server/src/db/schemas/links.ts @@ -135,9 +135,11 @@ const linkHighlightSchema = z.object({ export const selectLinksListSchema = createSelectSchema(linksTable) .omit({ content: true, + textContent: true, userId: true, processingStartedAt: true, - importSessionId: true + importSessionId: true, + searchVector: true }) .extend({ createdAt: z.string().optional(), @@ -193,7 +195,6 @@ export const createLinkSchema = z.object({ tags: z .array( z.object({ - id: z.string(), groupId: z.string(), name: z.string() }) diff --git a/apps/server/src/repositories/links.repository.ts b/apps/server/src/repositories/links.repository.ts index dfdb51b..f88e38b 100644 --- a/apps/server/src/repositories/links.repository.ts +++ b/apps/server/src/repositories/links.repository.ts @@ -35,7 +35,10 @@ type LinkHighlight = { }; // Omit 'highlights'/'tags' from LinkData to avoid conflict with the enriched shapes below -export type LinkListItem = Omit & { +export type LinkListItem = Omit< + LinkData, + 'content' | 'textContent' | 'userId' | 'highlights' | 'tags' +> & { tags: TagWithGroupColor[]; highlights: LinkHighlight[]; }; @@ -154,7 +157,6 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { publishedAt: linksTable.publishedAt, readingProgress: linksTable.readingProgress, readingTime: linksTable.readingTime, - textContent: linksTable.textContent, timeSpentReading: linksTable.timeSpentReading, title: linksTable.title, updatedAt: linksTable.updatedAt, @@ -730,7 +732,6 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { publishedAt: linksTable.publishedAt, readingProgress: linksTable.readingProgress, readingTime: linksTable.readingTime, - textContent: linksTable.textContent, timeSpentReading: linksTable.timeSpentReading, title: linksTable.title, updatedAt: linksTable.updatedAt, diff --git a/apps/server/src/routes/links/links.handlers.ts b/apps/server/src/routes/links/links.handlers.ts index d0db6b7..13068d5 100644 --- a/apps/server/src/routes/links/links.handlers.ts +++ b/apps/server/src/routes/links/links.handlers.ts @@ -11,7 +11,6 @@ import { isValidUrl } from '@/lib/url-validator.js'; import { saveLink } from '@/services/link-save.service.js'; import type { CursorPaginationOptions } from '@/types/pagination.js'; -import type { Tag } from '@/types/tags.js'; import type { CreateLinkRoute, @@ -173,15 +172,13 @@ export const createLink: AppRouteHandler = async (c) => { return c.json(response, response.status); } - const processedTags: Omit[] = []; + const processedTags: Array<{ groupId: string; name: string }> = []; for (const tag of tags) { if (tag && typeof tag === 'object' && tag.name && tag.groupId) { processedTags.push({ - id: tag.id, groupId: tag.groupId, - name: tag.name.trim(), - userId: user.id + name: tag.name.trim() }); } } diff --git a/apps/web/src/features/articles/api/create-link.contract.test.ts b/apps/web/src/features/articles/api/create-link.contract.test.ts new file mode 100644 index 0000000..4fe8e4c --- /dev/null +++ b/apps/web/src/features/articles/api/create-link.contract.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; + +import { createLinkSchema } from '../../../../../server/src/db/schemas/index.js'; + +import { createLinkBodySchema } from './create-link'; + +describe('createLink contract', () => { + it('accepts the web body shape on the server schema', () => { + const body = createLinkBodySchema.parse({ + url: 'https://example.com/new-article', + tags: [{ groupId: 'group-1', name: 'Research' }] + }); + + expect(createLinkSchema.safeParse(body).success).toBe(true); + }); +}); From d4b80ea6bbf6ce15fa59fc41dbab4f75900d2093 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:55:23 +0700 Subject: [PATCH 3/9] fix(repo): scope enrichment queries by user and speed up home suggestions The highlight and tag-link enrichment queries inside findUpcoming and search loaded related rows by link id alone, so a request for one user could surface another user's highlights or tag associations. Add the matching userId filter to every enrichment query, matching findMany. getHomeSuggestions issued six sequential reads, two of which (short/long counts) selected every matching readingTime row only to count and sum them in JS. Collapse those into a single conditional-aggregation query and run the four independent reads concurrently via Promise.all so home-view latency no longer grows with the sum of round-trips. Return shape is unchanged. Add repository tests covering cross-user enrichment scoping and the short/long count split. --- .../src/repositories/links.repository.test.ts | 124 ++++++++++++++ .../src/repositories/links.repository.ts | 151 ++++++++++-------- 2 files changed, 205 insertions(+), 70 deletions(-) create mode 100644 apps/server/src/repositories/links.repository.test.ts diff --git a/apps/server/src/repositories/links.repository.test.ts b/apps/server/src/repositories/links.repository.test.ts new file mode 100644 index 0000000..b7ceb18 --- /dev/null +++ b/apps/server/src/repositories/links.repository.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest'; + +import { db } from '@/db/index.js'; +import { highlightsTable, linksTable, usersTable } from '@/db/schemas/index.js'; + +import { createDrizzleLinksAdapter } from './links.repository.js'; + +const USER_A_ID = '10000000-0000-0000-0000-000000000021'; +const USER_B_ID = '10000000-0000-0000-0000-000000000022'; +const HOME_USER_ID = '10000000-0000-0000-0000-000000000031'; + +async function seedUser(id: string, email: string) { + await db.insert(usersTable).values({ + id, + email, + passwordHash: 'hashed-password', + name: email, + settings: {} + }); +} + +describe('links repository ownership', () => { + it('only returns the requesting user highlights in upcoming and search results', async () => { + await seedUser(USER_A_ID, 'links-a@example.com'); + await seedUser(USER_B_ID, 'links-b@example.com'); + + const [candidate] = await db + .insert(linksTable) + .values({ + createdAt: '2020-01-01T00:00:00.000Z', + readingTime: 5, + title: 'Scoped candidate article', + url: 'https://example.com/scoped-candidate', + userId: USER_A_ID + }) + .returning({ id: linksTable.id }); + const [current] = await db + .insert(linksTable) + .values({ + createdAt: '2020-01-02T00:00:00.000Z', + readingTime: 5, + title: 'Current article', + url: 'https://example.com/current', + userId: USER_A_ID + }) + .returning({ id: linksTable.id }); + + if (!candidate || !current) throw new Error('Expected article fixtures'); + + await db.insert(highlightsTable).values([ + { + color: '#000000', + endOffset: 12, + linkId: candidate.id, + startOffset: 0, + text: 'User A highlight', + userId: USER_A_ID + }, + { + color: '#ffffff', + endOffset: 24, + linkId: candidate.id, + startOffset: 12, + text: 'User B highlight', + userId: USER_B_ID + } + ]); + + const repo = createDrizzleLinksAdapter(db); + const upcoming = await repo.findUpcoming(USER_A_ID, current.id); + const search = await repo.search(USER_A_ID, 'Scoped candidate'); + + expect(upcoming[0]?.highlights?.map(({ text }) => text)).toEqual(['User A highlight']); + expect(search.items[0]?.highlights?.map(({ text }) => text)).toEqual(['User A highlight']); + }); +}); + +describe('links repository home suggestions', () => { + it('collapses short/long counts correctly and excludes archived/pending/unread-mismatched rows', async () => { + await seedUser(HOME_USER_ID, 'home@example.com'); + + const base = { userId: HOME_USER_ID, processingStatus: 'completed' as const }; + // Counted as short reads (unread, unarchived, completed). + await db.insert(linksTable).values([ + { ...base, readingTime: 5, title: 'Short 5', url: 'https://example.com/short-5' }, + { ...base, readingTime: 8, title: 'Short 8', url: 'https://example.com/short-8' } + ]); + // Counted as a long read. + await db + .insert(linksTable) + .values({ ...base, readingTime: 20, title: 'Long 20', url: 'https://example.com/long-20' }); + // Excluded: archived. + await db.insert(linksTable).values({ + ...base, + isArchived: true, + readingTime: 2, + title: 'Archived short', + url: 'https://example.com/archived' + }); + // Excluded: still pending processing. + await db.insert(linksTable).values({ + ...base, + processingStatus: 'pending', + readingTime: 4, + title: 'Pending short', + url: 'https://example.com/pending' + }); + // Excluded from counts (already read) but flips hasReadArticle. + await db.insert(linksTable).values({ + ...base, + isRead: true, + readingTime: 30, + title: 'Read long', + url: 'https://example.com/read-long' + }); + + const repo = createDrizzleLinksAdapter(db); + const suggestions = await repo.getHomeSuggestions(HOME_USER_ID); + + expect(suggestions.shortReads).toEqual({ totalArticles: 2, totalReadingTime: 13 }); + expect(suggestions.longReads).toEqual({ totalArticles: 1, totalReadingTime: 20 }); + expect(suggestions.hasReadArticle).toBe(true); + }); +}); diff --git a/apps/server/src/repositories/links.repository.ts b/apps/server/src/repositories/links.repository.ts index f88e38b..9d3590c 100644 --- a/apps/server/src/repositories/links.repository.ts +++ b/apps/server/src/repositories/links.repository.ts @@ -471,7 +471,7 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { text: highlightsTable.text }) .from(highlightsTable) - .where(inArray(highlightsTable.linkId, linkIds)); + .where(and(inArray(highlightsTable.linkId, linkIds), eq(highlightsTable.userId, userId))); const highlightsByLinkId = new Map[]>(); for (const row of highlightRows) { @@ -501,26 +501,70 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { async getHomeSuggestions(userId) { try { - const [cont] = await db - .select({ - coverImage: linksTable.coverImage, - id: linksTable.id, - lastReadAt: linksTable.lastReadAt, - readingProgress: linksTable.readingProgress, - readingTime: linksTable.readingTime, - title: linksTable.title - }) - .from(linksTable) - .where( - and( - eq(linksTable.userId, userId), - eq(linksTable.isRead, false), - eq(linksTable.processingStatus, 'completed'), - or(gt(linksTable.readingProgress, 0), isNotNull(linksTable.lastReadAt)) + // PERF-02: the four top-level reads are mutually independent, so run them + // concurrently (wall-clock = slowest, not the sum). The short/long counts + // collapse into a single conditional-aggregation scan of the unread set + // instead of two full SELECTs that only `.length`/`.reduce()` their rows. + const [continueRows, countRows, recentlySaved, readRows] = await Promise.all([ + db + .select({ + coverImage: linksTable.coverImage, + id: linksTable.id, + lastReadAt: linksTable.lastReadAt, + readingProgress: linksTable.readingProgress, + readingTime: linksTable.readingTime, + title: linksTable.title + }) + .from(linksTable) + .where( + and( + eq(linksTable.userId, userId), + eq(linksTable.isRead, false), + eq(linksTable.processingStatus, 'completed'), + or(gt(linksTable.readingProgress, 0), isNotNull(linksTable.lastReadAt)) + ) ) - ) - .orderBy(desc(linksTable.lastReadAt)) - .limit(1); + .orderBy(desc(linksTable.lastReadAt)) + .limit(1), + db + .select({ + longArticles: sql`count(*) filter (where ${linksTable.readingTime} >= 10)::int`, + longReadingTime: sql`coalesce(sum(${linksTable.readingTime}) filter (where ${linksTable.readingTime} >= 10), 0)::int`, + shortArticles: sql`count(*) filter (where ${linksTable.readingTime} < 10)::int`, + shortReadingTime: sql`coalesce(sum(${linksTable.readingTime}) filter (where ${linksTable.readingTime} < 10), 0)::int` + }) + .from(linksTable) + .where( + and( + eq(linksTable.userId, userId), + eq(linksTable.isRead, false), + eq(linksTable.isArchived, false), + eq(linksTable.processingStatus, 'completed') + ) + ), + findMany(userId, { limit: 3, orderDirection: 'desc' }), + db + .select({ id: linksTable.id }) + .from(linksTable) + .where( + and( + eq(linksTable.userId, userId), + or(eq(linksTable.isRead, true), gt(linksTable.readingProgress, 25)) + ) + ) + .limit(1) + ]); + + const cont = continueRows[0]; + // An aggregate with no GROUP BY always returns exactly one row (zeros over an + // empty set), but guard for type-safety regardless. + const counts = countRows[0] ?? { + longArticles: 0, + longReadingTime: 0, + shortArticles: 0, + shortReadingTime: 0 + }; + const readArticle = readRows[0]; const continueReading = cont ? { @@ -533,56 +577,17 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { } : null; - const shortRows = await db - .select({ readingTime: linksTable.readingTime }) - .from(linksTable) - .where( - and( - eq(linksTable.userId, userId), - eq(linksTable.isRead, false), - eq(linksTable.isArchived, false), - eq(linksTable.processingStatus, 'completed'), - lt(linksTable.readingTime, 10) - ) - ); - - const longRows = await db - .select({ readingTime: linksTable.readingTime }) - .from(linksTable) - .where( - and( - eq(linksTable.userId, userId), - eq(linksTable.isRead, false), - eq(linksTable.isArchived, false), - eq(linksTable.processingStatus, 'completed'), - gte(linksTable.readingTime, 10) - ) - ); - - const recentlySaved = await findMany(userId, { limit: 3, orderDirection: 'desc' }); - - const [readArticle] = await db - .select({ id: linksTable.id }) - .from(linksTable) - .where( - and( - eq(linksTable.userId, userId), - or(eq(linksTable.isRead, true), gt(linksTable.readingProgress, 25)) - ) - ) - .limit(1); - return { continueReading, hasReadArticle: readArticle != null, longReads: { - totalArticles: longRows.length, - totalReadingTime: longRows.reduce((sum, r) => sum + (r.readingTime || 0), 0) + totalArticles: counts.longArticles, + totalReadingTime: counts.longReadingTime }, recentlySaved: recentlySaved.items as unknown as HomeSuggestions['recentlySaved'], shortReads: { - totalArticles: shortRows.length, - totalReadingTime: shortRows.reduce((sum, r) => sum + (r.readingTime || 0), 0) + totalArticles: counts.shortArticles, + totalReadingTime: counts.shortReadingTime } }; } catch (error) { @@ -678,9 +683,12 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { .select({ linkId: linkTagsTable.linkId }) .from(linkTagsTable) .where( - inArray( - linkTagsTable.tagId, - tagIdRows.map((t) => t.id) + and( + inArray( + linkTagsTable.tagId, + tagIdRows.map((t) => t.id) + ), + eq(linkTagsTable.userId, userId) ) ); if (linkIdRows.length === 0) return { items: [], hasMore: false, nextCursor: undefined }; @@ -699,9 +707,12 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { .select({ link_id: linkTagsTable.linkId }) .from(linkTagsTable) .where( - inArray( - linkTagsTable.tagId, - tagIdsRows.map((t) => t.id) + and( + inArray( + linkTagsTable.tagId, + tagIdsRows.map((t) => t.id) + ), + eq(linkTagsTable.userId, userId) ) ); if (linkIdRows.length === 0) return { items: [], hasMore: false, nextCursor: undefined }; @@ -796,7 +807,7 @@ export function createDrizzleLinksAdapter(db: DrizzleClient): LinksRepository { text: highlightsTable.text }) .from(highlightsTable) - .where(inArray(highlightsTable.linkId, linkIds)); + .where(and(inArray(highlightsTable.linkId, linkIds), eq(highlightsTable.userId, userId))); const highlightsByLinkId = new Map[]>(); for (const row of highlightRows) { From ab2fdb05e73023921351ffd1c524f50889de4967 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:55:58 +0700 Subject: [PATCH 4/9] feat(worker): scope article images per user, resume stalled jobs, drop per-image delay Store extracted article body images under user-scoped keys (user-/articles/...) and ownership-check them through the files route, so one user can never read another's extracted images. Block the legacy shared/articles/* prefix at the files route while leaving other shared uploads readable. Remove the unconditional 500ms sleep inside the per-image upload loop; it burned ~25s of the job budget on a 50-image article with no benefit. Make processing resumable: stamp processingStartedAt on the pending to processing flip, and reset a stale processing row back to pending on job start instead of returning skipped, so a hard-crashed worker no longer leaves a link stranded. Add behavior tests for user-scoped uploads, the removed delay, stale-processing recovery, and the legacy shared gate. --- .../server/src/routes/files/files.handlers.ts | 3 +- apps/server/src/routes/files/files.test.ts | 25 ++ ...content-extraction.worker.behavior.test.ts | 265 ++++++++++++++++++ .../src/workers/content-extraction.worker.ts | 38 ++- 4 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/workers/content-extraction.worker.behavior.test.ts diff --git a/apps/server/src/routes/files/files.handlers.ts b/apps/server/src/routes/files/files.handlers.ts index 6e140b3..39c25a2 100644 --- a/apps/server/src/routes/files/files.handlers.ts +++ b/apps/server/src/routes/files/files.handlers.ts @@ -19,6 +19,7 @@ async function serveFile(c: Context, key: string) { try { const isShared = storageService.isSharedFile(key); + const isLegacySharedArticle = key.startsWith('shared/articles/'); if (!user) { const response = errorResponse( @@ -30,7 +31,7 @@ async function serveFile(c: Context, key: string) { } else { const isUserOwned = storageService.isUserFile(key, user.id); - if (!isShared && !isUserOwned) { + if ((!isShared && !isUserOwned) || isLegacySharedArticle) { const response = errorResponse('Access denied to this file', HttpStatus.FORBIDDEN); return c.json(response, response.status); diff --git a/apps/server/src/routes/files/files.test.ts b/apps/server/src/routes/files/files.test.ts index 8b82577..335e317 100644 --- a/apps/server/src/routes/files/files.test.ts +++ b/apps/server/src/routes/files/files.test.ts @@ -31,6 +31,7 @@ let userId: string; let authCookie: string; let userFileKey: string; let sharedFileKey: string; +let legacySharedArticleKey: string; let testUser: UserWithoutPassword; beforeAll(async () => { @@ -51,6 +52,7 @@ beforeAll(async () => { userFileKey = `user-${userId}/uploads/test.txt`; sharedFileKey = 'shared/uploads/test.txt'; + legacySharedArticleKey = 'shared/articles/test.txt'; await fs.mkdir(path.join(STORAGE_ROOT, `user-${userId}/uploads`), { recursive: true @@ -58,8 +60,12 @@ beforeAll(async () => { await fs.mkdir(path.join(STORAGE_ROOT, 'shared/uploads'), { recursive: true }); + await fs.mkdir(path.join(STORAGE_ROOT, 'shared/articles'), { + recursive: true + }); await fs.writeFile(path.join(STORAGE_ROOT, userFileKey), FILE_CONTENT); await fs.writeFile(path.join(STORAGE_ROOT, sharedFileKey), FILE_CONTENT); + await fs.writeFile(path.join(STORAGE_ROOT, legacySharedArticleKey), FILE_CONTENT); }); afterAll(async () => { @@ -70,6 +76,9 @@ afterAll(async () => { await fs.rm(path.join(STORAGE_ROOT, 'shared/uploads/test.txt'), { force: true }); + await fs.rm(path.join(STORAGE_ROOT, 'shared/articles/test.txt'), { + force: true + }); }); function createFakeAuthRepository(user: UserWithoutPassword): AuthRepository { @@ -81,6 +90,14 @@ function createFakeAuthRepository(user: UserWithoutPassword): AuthRepository { avatar: user.avatar, settings: user.settings }), + createWithInitialRole: async () => ({ + id: user.id, + email: user.email, + name: user.name, + avatar: user.avatar, + settings: user.settings, + role: 'user' + }), findByEmail: async () => null, findById: async (id) => (id === user.id ? user : null), findByIdWithCredentials: async () => null, @@ -245,6 +262,14 @@ describe('GET /files/:key', () => { expect(response.status).toBe(HttpStatus.OK); }); + it('returns 403 for legacy shared article files', async () => { + const response = await client.files[':key{.*}'].$get( + { param: { key: legacySharedArticleKey } }, + { headers: { Cookie: authCookie } } + ); + expect(response.status).toBe(HttpStatus.FORBIDDEN); + }); + it("returns 403 for another user's file", async () => { const otherUserKey = `user-${OTHER_USER_ID}/uploads/secret.txt`; const response = await client.files[':key{.*}'].$get( diff --git a/apps/server/src/workers/content-extraction.worker.behavior.test.ts b/apps/server/src/workers/content-extraction.worker.behavior.test.ts new file mode 100644 index 0000000..16f5435 --- /dev/null +++ b/apps/server/src/workers/content-extraction.worker.behavior.test.ts @@ -0,0 +1,265 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { importWithEnv } from '@/tests/env.js'; + +let contentExtractionJobHandler: + | ((job: { + data: { + importSessionId?: string; + linkId: string; + url: string; + user: { id: string }; + }; + id: string; + updateProgress: ReturnType; + }) => Promise<{ status: string }>) + | undefined; + +const createWorkerMock = vi.hoisted(() => + vi.fn((name: string, jobHandler: unknown) => { + if (name === 'content-extraction') { + contentExtractionJobHandler = jobHandler as typeof contentExtractionJobHandler; + } + + return { on: vi.fn() }; + }) +); + +const enqueueContentExtractionMock = vi.hoisted(() => ({ + add: vi.fn(), + on: vi.fn() +})); + +const linksAdapterMock = vi.hoisted(() => ({ + findById: vi.fn(), + update: vi.fn() +})); + +const importSessionsAdapterMock = vi.hoisted(() => ({ + incrementExtractionCounts: vi.fn(), + findPendingLinksInSession: vi.fn(), + updateExtractionStatus: vi.fn() +})); + +const browserServiceMock = vi.hoisted(() => ({ + crawlPage: vi.fn() +})); + +const contentExtractionServiceMock = vi.hoisted(() => ({ + extractMetadata: vi.fn(), + extractReadableContent: vi.fn() +})); + +const markdownServiceMock = vi.hoisted(() => ({ + convertToMarkdown: vi.fn() +})); + +const storageServiceMock = vi.hoisted(() => ({ + uploadImageFromUrl: vi.fn() +})); + +vi.mock('@/db/index.js', () => ({ db: {} })); + +vi.mock('@/index.js', () => ({ getIsShuttingDown: vi.fn(() => false) })); + +vi.mock('@/lib/job-queue.js', () => ({ + createWorker: createWorkerMock, + createQueue: vi.fn(() => ({ + add: vi.fn(), + close: vi.fn(), + on: vi.fn(), + upsertJobScheduler: vi.fn() + })) +})); + +vi.mock('@/queues/content-extraction.queue.js', () => ({ + enqueueContentExtraction: enqueueContentExtractionMock +})); + +vi.mock('@/repositories/import-sessions.repository.js', () => ({ + createDrizzleImportSessionsAdapter: vi.fn(() => importSessionsAdapterMock) +})); + +vi.mock('@/repositories/links.repository.js', () => ({ + createDrizzleLinksAdapter: vi.fn(() => linksAdapterMock) +})); + +vi.mock('@/services/browser.service.js', () => ({ browserService: browserServiceMock })); +vi.mock('@/services/content-extraction.service.js', () => ({ + contentExtractionService: contentExtractionServiceMock +})); +vi.mock('@/services/markdown.service.js', () => ({ markdownService: markdownServiceMock })); +vi.mock('@/services/storage.service.js', () => ({ storageService: storageServiceMock })); + +await importWithEnv({ DEMO_MODE: 'false' }, async () => import('./content-extraction.worker.js')); + +beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(globalThis, 'setTimeout'); + + linksAdapterMock.findById.mockResolvedValue( + makeLink({ + processingStartedAt: null, + processingStatus: 'pending' + }) + ); + linksAdapterMock.update.mockResolvedValue(null); + browserServiceMock.crawlPage.mockResolvedValue({ + html: '
', + isPaywalled: false + }); + contentExtractionServiceMock.extractMetadata.mockResolvedValue({ + author: 'Author', + favicon: 'favicon.ico', + image: 'https://example.com/cover.png', + publishedDate: '2026-01-01', + title: 'Article Title' + }); + contentExtractionServiceMock.extractReadableContent.mockResolvedValue({ + author: 'Readable Author', + content: '
', + excerpt: 'Excerpt', + textContent: 'Text content', + title: 'Readable Title' + }); + markdownServiceMock.convertToMarkdown.mockReturnValue('Markdown content'); + storageServiceMock.uploadImageFromUrl + .mockResolvedValueOnce({ key: 'user-user-1/articles/cover.png', url: 'https://cdn/cover.png' }) + .mockResolvedValueOnce({ key: 'user-user-1/articles/body.png', url: 'https://cdn/body.png' }); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function makeLink(overrides: Record = {}) { + return { + author: null, + content: null, + coverImage: null, + createdAt: new Date().toISOString(), + errorMessage: null, + excerpt: null, + favicon: null, + highlights: [], + id: 'link-1', + importSessionId: null, + isArchived: false, + isFavorite: false, + isPaywalled: false, + isRead: false, + lastReadAt: null, + priority: 'none', + processingStartedAt: null, + processingStatus: 'pending', + publishedAt: null, + readingProgress: 0, + readingTime: 1, + tags: [], + textContent: null, + timeSpentReading: 0, + title: 'Seed Title', + updatedAt: new Date(), + url: 'https://example.com/article', + userId: 'user-1', + ...overrides + }; +} + +function job(overrides: Record = {}) { + return { + data: { + linkId: 'link-1', + url: 'https://example.com/article', + user: { id: 'user-1' }, + ...overrides + }, + id: 'job-1', + updateProgress: vi.fn() + }; +} + +describe('content extraction worker behavior', () => { + it('uploads article images under the current user and does not pause between images', async () => { + expect(contentExtractionJobHandler).toBeDefined(); + + const setTimeoutSpy = vi.mocked(globalThis.setTimeout); + + const result = await contentExtractionJobHandler!(job()); + + expect(result).toMatchObject({ status: 'success', imagesProcessed: 1, imagesFailed: 0 }); + expect(linksAdapterMock.update).toHaveBeenCalledWith( + 'link-1', + 'user-1', + expect.objectContaining({ + processingStatus: 'processing', + processingStartedAt: expect.any(Date) + }) + ); + expect(storageServiceMock.uploadImageFromUrl).toHaveBeenNthCalledWith( + 1, + 'https://example.com/cover.png', + { userId: 'user-1' } + ); + expect(storageServiceMock.uploadImageFromUrl).toHaveBeenNthCalledWith( + 2, + 'https://example.com/body.png', + { userId: 'user-1' } + ); + expect(setTimeoutSpy.mock.calls.some(([, delay]) => delay === 500)).toBe(false); + }); + + it('resumes stale processing jobs instead of skipping them', async () => { + linksAdapterMock.findById.mockResolvedValueOnce( + makeLink({ + processingStartedAt: new Date(Date.now() - 6 * 60 * 1000), + processingStatus: 'processing' + }) + ); + + expect(contentExtractionJobHandler).toBeDefined(); + + const result = await contentExtractionJobHandler!(job()); + + expect(result.status).toBe('success'); + expect( + linksAdapterMock.update.mock.calls.map( + ([, , updates]) => (updates as { processingStatus?: string }).processingStatus + ) + ).toEqual(['pending', 'processing', 'completed']); + expect(linksAdapterMock.update).toHaveBeenNthCalledWith( + 1, + 'link-1', + 'user-1', + expect.objectContaining({ processingStartedAt: null, processingStatus: 'pending' }) + ); + }); + + it('marks processing jobs as failed when extraction throws', async () => { + contentExtractionServiceMock.extractMetadata.mockRejectedValueOnce(new Error('boom')); + + expect(contentExtractionJobHandler).toBeDefined(); + + await expect(contentExtractionJobHandler!(job())).rejects.toThrow('boom'); + expect( + linksAdapterMock.update.mock.calls.map( + ([, , updates]) => (updates as { processingStatus?: string }).processingStatus + ) + ).toEqual(['processing', 'failed']); + expect(linksAdapterMock.update).toHaveBeenNthCalledWith( + 1, + 'link-1', + 'user-1', + expect.objectContaining({ + processingStartedAt: expect.any(Date), + processingStatus: 'processing' + }) + ); + expect(linksAdapterMock.update).toHaveBeenNthCalledWith( + 2, + 'link-1', + 'user-1', + expect.objectContaining({ processingStartedAt: null, processingStatus: 'failed' }) + ); + }); +}); diff --git a/apps/server/src/workers/content-extraction.worker.ts b/apps/server/src/workers/content-extraction.worker.ts index 1279653..5f17af3 100644 --- a/apps/server/src/workers/content-extraction.worker.ts +++ b/apps/server/src/workers/content-extraction.worker.ts @@ -24,6 +24,7 @@ const links = createDrizzleLinksAdapter(db); const importSessions = createDrizzleImportSessionsAdapter(db); const workerName = 'content-extraction-worker'; +const jobTimeoutMs = 5 * 60 * 1000; const isDataURI = (uri: string): boolean => uri.startsWith('data:'); @@ -86,8 +87,6 @@ async function contentExtractionJob(job: Job): Promise const { linkId, url: articleUrl, user } = job.data; - // overall job timeout: 5 minutes - const jobTimeoutMs = 5 * 60 * 1000; const jobController = new AbortController(); const jobTimeout = setTimeout(() => { jobController.abort(new Error('Job exceeded 5 minute timeout')); @@ -104,15 +103,35 @@ async function contentExtractionJob(job: Job): Promise return { error: 'Link not found', status: 'aborted' }; } - if (initialDoc.processingStatus !== 'pending') { + let currentDoc = initialDoc; + + if (currentDoc.processingStatus === 'processing') { + const startedAtMs = currentDoc.processingStartedAt?.getTime(); + const isStale = !startedAtMs || Date.now() - startedAtMs >= jobTimeoutMs; + + if (isStale) { + await links.update(linkId, user.id, { + processingStartedAt: null, + processingStatus: 'pending' + }); + currentDoc = { + ...currentDoc, + processingStartedAt: null, + processingStatus: 'pending' + }; + } + } + + if (currentDoc.processingStatus !== 'pending') { clearTimeout(jobTimeout); logger.warn(`Link with id ${linkId} is not pending`); - return { articleTitle: initialDoc.title as string, status: 'skipped' }; + return { articleTitle: currentDoc.title as string, status: 'skipped' }; } try { await links.update(linkId, user.id, { + processingStartedAt: new Date(), processingStatus: 'processing' }); } catch (error: unknown) { @@ -205,10 +224,9 @@ async function contentExtractionJob(job: Job): Promise `Processing image ${currentImageProgress} of ${images.length}: ${absoluteSrc}` ); - // rate limiting 500ms delay between image uploads to avoid anti-bot - await new Promise((resolve) => setTimeout(resolve, 500)); - - const uploadedImage = await storageService.uploadImageFromUrl(absoluteSrc); + const uploadedImage = await storageService.uploadImageFromUrl(absoluteSrc, { + userId: user.id + }); if (!uploadedImage?.url) { // upload failed, hide broken image @@ -249,6 +267,7 @@ async function contentExtractionJob(job: Job): Promise excerpt: readableContent.excerpt, favicon: metadata.favicon, isPaywalled, + processingStartedAt: null, processingStatus: 'completed' as const, publishedAt: metadata.publishedDate ? new Date(metadata.publishedDate) : null, readingTime, @@ -283,6 +302,7 @@ async function contentExtractionJob(job: Job): Promise content: readableContent.content || null, errorMessage: 'No content extracted from the article', isPaywalled, + processingStartedAt: null, processingStatus: 'completed' as const, textContent: readableContent.textContent || null, title: readableContent.title @@ -310,6 +330,7 @@ async function contentExtractionJob(job: Job): Promise await links.update(linkId, user.id, { errorMessage: 'No content extracted from the article', + processingStartedAt: null, processingStatus: 'failed' }); @@ -322,6 +343,7 @@ async function contentExtractionJob(job: Job): Promise await links.update(linkId, user.id, { content: 'No content extracted from the article', errorMessage: "Content couldn't be extracted. Link may be broken", + processingStartedAt: null, processingStatus: 'failed' }); From 1552e2f5ae42343572fad62fae94d50e7f66a450 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:56:16 +0700 Subject: [PATCH 5/9] feat(security): gate rate-limit proxy trust, sniff image bytes, assign first admin atomically Rate limiting keyed client identity on forwarded headers unconditionally, so a spoofable x-forwarded-for let a single client evade limits (or frame others). Trust x-forwarded-for/x-real-ip only behind an explicit BEHIND_PROXY flag or a TRUSTED_PROXIES allowlist, falling back to the socket peer IP otherwise. Replace the unused RATE_LIMIT_WINDOW_MS/MAX env vars, which no limiter read, with the new proxy-trust flags and document them for self-hosters. Validate uploaded images by sniffing the buffer magic bytes rather than trusting the client filename or Content-Type, and stop trusting response Content-Type when downloading article images. Make first-admin assignment atomic: the initial role check plus insert runs in a serializable transaction with retry, so concurrent first registrations settle on exactly one admin instead of racing to create several. Add tests for header trust in both modes, image-byte rejection, and concurrent first-user registration. --- README.md | 1 + apps/server/.env.example | 2 + apps/server/.env.test.example | 2 + apps/server/src/lib/env-config.ts | 7 ++- .../server/src/middlewares/rate-limit.test.ts | 27 +++++++++ apps/server/src/middlewares/rate-limit.ts | 48 ++++++++++++++- .../src/repositories/auth.repository.test.ts | 33 ++++++++++ .../src/repositories/auth.repository.ts | 55 +++++++++++++++++ apps/server/src/routes/auth/auth.handlers.ts | 11 +--- .../src/services/storage.service.test.ts | 27 +++++++++ apps/server/src/services/storage.service.ts | 60 +++++++++++++++---- apps/server/src/tests/in-memory/auth.ts | 33 +++++++++- docs/SELF_HOSTING.md | 5 +- 13 files changed, 285 insertions(+), 26 deletions(-) create mode 100644 apps/server/src/middlewares/rate-limit.test.ts create mode 100644 apps/server/src/repositories/auth.repository.test.ts create mode 100644 apps/server/src/services/storage.service.test.ts diff --git a/README.md b/README.md index 355e500..a9a5f04 100644 --- a/README.md +++ b/README.md @@ -377,4 +377,5 @@ STORAGE_SECRET_ACCESS_KEY= ## Operational Notes - Article extraction uses a Playwright-compatible Camoufox websocket configured by `BROWSER_URL`. +- Rate-limit IP trust stays on the socket peer by default; set `BEHIND_PROXY=true` or `TRUSTED_PROXIES` before trusting forwarded headers. - Local storage is available by default; S3-compatible storage is supported through server env vars. diff --git a/apps/server/.env.example b/apps/server/.env.example index 8b8dfce..e783960 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -30,6 +30,8 @@ REDIS_PORT=6379 # Rate limiting RATE_LIMIT_WINDOW_MS=60000 RATE_LIMIT_MAX=50 +BEHIND_PROXY=false +TRUSTED_PROXIES= # Body size limit (in bytes) BODY_SIZE_LIMIT=102400 diff --git a/apps/server/.env.test.example b/apps/server/.env.test.example index 9ba9970..d82b010 100644 --- a/apps/server/.env.test.example +++ b/apps/server/.env.test.example @@ -23,6 +23,8 @@ REDIS_PORT=6379 RATE_LIMIT_WINDOW_MS=60000 RATE_LIMIT_MAX=50 +BEHIND_PROXY=false +TRUSTED_PROXIES= BODY_SIZE_LIMIT=102400 PUBLIC_URL=http://localhost:3000 diff --git a/apps/server/src/lib/env-config.ts b/apps/server/src/lib/env-config.ts index cb2342a..504ec46 100644 --- a/apps/server/src/lib/env-config.ts +++ b/apps/server/src/lib/env-config.ts @@ -44,9 +44,12 @@ const envSchema = z REDIS_PORT: z.coerce.number().int().positive().default(6379), - RATE_LIMIT_WINDOW_MS: z.coerce.number().int().positive().default(60_000), + BEHIND_PROXY: z + .enum(['true', 'false']) + .default('false') + .transform((value) => value === 'true'), - RATE_LIMIT_MAX: z.coerce.number().int().positive().default(50), + TRUSTED_PROXIES: z.string().default(''), BODY_SIZE_LIMIT: z.coerce.number().int().positive().default(4_194_304), diff --git a/apps/server/src/middlewares/rate-limit.test.ts b/apps/server/src/middlewares/rate-limit.test.ts new file mode 100644 index 0000000..2efd17e --- /dev/null +++ b/apps/server/src/middlewares/rate-limit.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveRateLimitIpKey } from './rate-limit.js'; + +describe('resolveRateLimitIpKey', () => { + it('ignores spoofed forwarded headers when proxies are untrusted', () => { + expect( + resolveRateLimitIpKey({ + peerIp: '203.0.113.10', + forwardedFor: '198.51.100.23', + realIp: '198.51.100.24', + trustForwardedHeaders: false + }) + ).toBe('203.0.113.10'); + }); + + it('uses the forwarded client IP when proxies are trusted', () => { + expect( + resolveRateLimitIpKey({ + peerIp: '203.0.113.10', + forwardedFor: '198.51.100.23, 198.51.100.24', + realIp: '198.51.100.25', + trustForwardedHeaders: true + }) + ).toBe('198.51.100.23'); + }); +}); diff --git a/apps/server/src/middlewares/rate-limit.ts b/apps/server/src/middlewares/rate-limit.ts index 4693bd5..001cd87 100644 --- a/apps/server/src/middlewares/rate-limit.ts +++ b/apps/server/src/middlewares/rate-limit.ts @@ -1,3 +1,4 @@ +import { getConnInfo } from '@hono/node-server/conninfo'; import type { Context } from 'hono'; import type { RedisClient } from 'hono-rate-limiter'; import { RedisStore, rateLimiter } from 'hono-rate-limiter'; @@ -5,6 +6,7 @@ import { Redis } from 'ioredis'; import redisConfig from '@/config/redis.config.js'; +import { env } from '@/lib/env-config.js'; import type { AppBindings } from '@/lib/types.js'; const redisClient = new Redis(redisConfig); @@ -19,6 +21,29 @@ const redisAdapter: RedisClient = { }; const ONE_HOUR_WINDOW = 60 * 60 * 1000; +const trustedProxies = new Set( + env.TRUSTED_PROXIES.split(',') + .map((value) => value.trim()) + .filter(Boolean) +); + +export interface RateLimitIpKeyInput { + peerIp: string; + forwardedFor?: string | null; + realIp?: string | null; + trustForwardedHeaders: boolean; +} + +export function resolveRateLimitIpKey({ + peerIp, + forwardedFor, + realIp, + trustForwardedHeaders +}: RateLimitIpKeyInput): string { + if (!trustForwardedHeaders) return peerIp; + + return forwardedFor?.split(',')[0]?.trim() || realIp?.trim() || peerIp; +} function makeStore(prefix: string) { return new RedisStore({ @@ -27,10 +52,27 @@ function makeStore(prefix: string) { }); } +function getPeerIp(c: Context): string { + return getConnInfo(c).remote.address ?? 'unknown'; +} + +function isTrustedForwardedHeaders(c: Context): boolean { + if (env.BEHIND_PROXY) return true; + + const peerIp = getPeerIp(c); + return trustedProxies.has(peerIp); +} + function keyByIp(c: Context): string { - return ( - c.req.header('x-forwarded-for')?.split(',')[0]?.trim() || c.req.header('x-real-ip') || 'unknown' - ); + const peerIp = getPeerIp(c); + const trustForwardedHeaders = isTrustedForwardedHeaders(c); + + return resolveRateLimitIpKey({ + peerIp, + forwardedFor: c.req.header('x-forwarded-for'), + realIp: c.req.header('x-real-ip'), + trustForwardedHeaders + }); } function keyByUser(c: Context): string { diff --git a/apps/server/src/repositories/auth.repository.test.ts b/apps/server/src/repositories/auth.repository.test.ts new file mode 100644 index 0000000..907d00e --- /dev/null +++ b/apps/server/src/repositories/auth.repository.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; + +import { db } from '@/db/index.js'; + +import { passwordManager } from '@/lib/password-manager.js'; + +import { createDrizzleAuthAdapter } from './auth.repository.js'; + +describe('createWithInitialRole', () => { + it('assigns only one admin when first-user registrations race', async () => { + const auth = createDrizzleAuthAdapter(db); + const passwordHash = await passwordManager.hash('password123'); + + const [first, second] = await Promise.all([ + auth.createWithInitialRole({ + email: 'race-a@example.com', + passwordHash, + name: 'Race A', + settings: {} + }), + auth.createWithInitialRole({ + email: 'race-b@example.com', + passwordHash, + name: 'Race B', + settings: {} + }) + ]); + + expect([first.role, second.role].sort()).toEqual(['admin', 'user']); + await expect(auth.countActiveAdmins()).resolves.toBe(1); + await expect(auth.countUsers()).resolves.toBe(2); + }); +}); diff --git a/apps/server/src/repositories/auth.repository.ts b/apps/server/src/repositories/auth.repository.ts index 507d3a0..dbd1039 100644 --- a/apps/server/src/repositories/auth.repository.ts +++ b/apps/server/src/repositories/auth.repository.ts @@ -32,6 +32,36 @@ const publicUserColumns = { settings: usersTable.settings }; +const serializableRetryableCodes = new Set(['40001', '40P01']); + +function isRetryableTransactionError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + serializableRetryableCodes.has(String((error as { code?: string }).code)) + ); +} + +async function runSerializableTransaction( + db: DrizzleClient, + operation: (tx: DrizzleClient) => Promise +): Promise { + for (let attempt = 1; attempt <= 3; attempt++) { + try { + return await db.transaction(async (tx) => operation(tx as DrizzleClient), { + isolationLevel: 'serializable' + }); + } catch (error) { + if (!isRetryableTransactionError(error) || attempt === 3) { + throw error; + } + } + } + + throw new Error('Failed to complete serializable transaction'); +} + const userWithoutPasswordColumns = { id: usersTable.id, email: usersTable.email, @@ -46,6 +76,7 @@ const userWithoutPasswordColumns = { export interface AuthRepository { create(userData: CreateUser): Promise; + createWithInitialRole(userData: CreateUser): Promise; findByEmail(email: string): Promise; findById(id: string): Promise; // Returns full User including passwordHash — only for auth verification @@ -89,6 +120,30 @@ export function createDrizzleAuthAdapter(db: DrizzleClient): AuthRepository { return createdUser; }, + async createWithInitialRole(userData) { + return runSerializableTransaction(db, async (tx) => { + const [users] = await tx + .select({ count: count() }) + .from(usersTable) + .where(isNull(usersTable.deletedAt)); + + const role = Number(users?.count ?? 0) === 0 ? 'admin' : 'user'; + const [createdUser] = await tx + .insert(usersTable) + .values({ ...userData, role }) + .returning({ + ...publicUserColumns, + role: usersTable.role + }); + + if (!createdUser) { + throw new Error('Failed to create user'); + } + + return createdUser; + }); + }, + async findByEmail(email) { const user = await db.query.usersTable.findFirst({ where: eq(usersTable.email, email) diff --git a/apps/server/src/routes/auth/auth.handlers.ts b/apps/server/src/routes/auth/auth.handlers.ts index a549f1d..981954e 100644 --- a/apps/server/src/routes/auth/auth.handlers.ts +++ b/apps/server/src/routes/auth/auth.handlers.ts @@ -55,16 +55,11 @@ export const create: AppRouteHandler = async (c) => { const hashedPassword = await passwordManager.hash(password); - const userCount = await auth.countUsers(); - const isFirstUser = userCount === 0; - const role = isFirstUser ? 'admin' : 'user'; - - const createdUser = await auth.create({ + const createdUser = await auth.createWithInitialRole({ email, passwordHash: hashedPassword, name: name || null, - settings: defaultUserSettings, - ...(isFirstUser && { role: 'admin' as const }) + settings: defaultUserSettings }); const { tags } = c.get('repos'); @@ -84,7 +79,7 @@ export const create: AppRouteHandler = async (c) => { email: createdUser.email, name: createdUser.name, avatar: createdUser.avatar, - role, + role: createdUser.role, settings: createdUser.settings }, 'User created successfully', diff --git a/apps/server/src/services/storage.service.test.ts b/apps/server/src/services/storage.service.test.ts new file mode 100644 index 0000000..32f328c --- /dev/null +++ b/apps/server/src/services/storage.service.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { storageService } from './storage.service.js'; + +const pngSignature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +describe('storageService.uploadImage', () => { + it('rejects non-image bytes even when the filename looks like a png', async () => { + await expect( + storageService.uploadImage(Buffer.from('not-an-image'), 'fake.png', 'test-user', 'uploads') + ).rejects.toThrow('File is not a supported image format'); + }); + + it('accepts image bytes even when the filename extension is wrong', async () => { + const result = await storageService.uploadImage( + Buffer.concat([pngSignature, Buffer.from('fake image payload')]), + 'fake.txt', + 'test-user', + 'uploads' + ); + + expect(result.key).toContain('.png'); + expect(result.key).toContain('user-test-user/uploads/'); + + await storageService.deleteFile(result.key); + }); +}); diff --git a/apps/server/src/services/storage.service.ts b/apps/server/src/services/storage.service.ts index b512f2c..2fe1a8d 100644 --- a/apps/server/src/services/storage.service.ts +++ b/apps/server/src/services/storage.service.ts @@ -95,6 +95,45 @@ const sharedUtils = { return ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'].includes(extension.toLowerCase()); }, + detectImageFormat(buffer: Buffer): { extension: string; contentType: string } | null { + const pngSignature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (buffer.subarray(0, pngSignature.length).equals(pngSignature)) { + return { extension: '.png', contentType: 'image/png' }; + } + + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return { extension: '.jpg', contentType: 'image/jpeg' }; + } + + const gifSignature = buffer.subarray(0, 6).toString('ascii'); + if (gifSignature === 'GIF87a' || gifSignature === 'GIF89a') { + return { extension: '.gif', contentType: 'image/gif' }; + } + + if ( + buffer.length >= 12 && + buffer.subarray(0, 4).toString('ascii') === 'RIFF' && + buffer.subarray(8, 12).toString('ascii') === 'WEBP' + ) { + return { extension: '.webp', contentType: 'image/webp' }; + } + + const textPrefix = buffer.subarray(0, 512).toString('utf8').trimStart().toLowerCase(); + if ( + textPrefix.startsWith(' { - const ext = sharedUtils.getFileExtension(originalName); - if (!sharedUtils.isImageFile(ext)) { + const imageFormat = sharedUtils.detectImageFormat(buffer); + if (!imageFormat) { throw new Error('File is not a supported image format'); } - return this.uploadFile(buffer, originalName, userId, imageType); + + const normalizedName = sharedUtils.normalizeImageFilename(originalName, imageFormat.extension); + return this.uploadFile(buffer, normalizedName, userId, imageType); } async uploadImageFromUrl( @@ -232,11 +273,6 @@ class LocalStorageAdapter implements IStorageService { throw new Error(`Failed to download image: ${response.status} ${response.statusText}`); } - const contentType = response.headers.get('content-type'); - if (!contentType?.startsWith('image/')) { - throw new Error(`URL does not point to an image. Content-Type: ${contentType}`); - } - const contentLength = response.headers.get('content-length'); const maxSize = 10 * 1024 * 1024; if (contentLength && Number.parseInt(contentLength, 10) > maxSize) { @@ -392,11 +428,13 @@ class S3StorageAdapter implements IStorageService { userId?: string, imageType: ImageType = 'uploads' ): Promise { - const ext = sharedUtils.getFileExtension(originalName); - if (!sharedUtils.isImageFile(ext)) { + const imageFormat = sharedUtils.detectImageFormat(buffer); + if (!imageFormat) { throw new Error('File is not a supported image format'); } - return this.uploadFile(buffer, originalName, userId, imageType); + + const normalizedName = sharedUtils.normalizeImageFilename(originalName, imageFormat.extension); + return this.uploadFile(buffer, normalizedName, userId, imageType); } async uploadImageFromUrl( diff --git a/apps/server/src/tests/in-memory/auth.ts b/apps/server/src/tests/in-memory/auth.ts index 4fd9723..cb02d94 100644 --- a/apps/server/src/tests/in-memory/auth.ts +++ b/apps/server/src/tests/in-memory/auth.ts @@ -69,7 +69,38 @@ export function createInMemoryAuthAdapter(): AuthRepository { return publicUser; }, - countUsers: async () => usersByEmail.size, + createWithInitialRole: async (data) => { + const now = new Date(Date.now() + createdOffset++).toISOString(); + const activeUserCount = Array.from(usersById.values()).filter( + (user) => !user.deletedAt + ).length; + const role = data.role ?? (activeUserCount === 0 ? 'admin' : 'user'); + const user: User = { + ...data, + passwordHash: data.passwordHash, + id: crypto.randomUUID(), + role, + name: data.name ?? null, + avatar: data.avatar ?? null, + settings: {}, + deletedAt: null, + createdAt: now, + updatedAt: now + }; + usersByEmail.set(user.email, user); + usersById.set(user.id, user); + const { + passwordHash: _, + role: __, + deletedAt: ___, + createdAt: ____, + updatedAt: _____, + ...publicUser + } = user; + return { ...publicUser, role: user.role }; + }, + + countUsers: async () => Array.from(usersById.values()).filter((user) => !user.deletedAt).length, countActiveAdmins: async () => Array.from(usersById.values()).filter((user) => user.role === 'admin' && !user.deletedAt) diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index 627426e..1143e2f 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -102,6 +102,8 @@ CORS_ORIGINS=https://loreo.example.com JWT_SECRET= # minimum 32 characters PUBLIC_URL=https://loreo.example.com +BEHIND_PROXY=true +TRUSTED_PROXIES=127.0.0.1 ``` ## Service Details @@ -182,7 +184,8 @@ When using a reverse proxy: 1. Set `WEB_PUBLIC_PORT` to match the port your proxy forwards to 2. Set `PUBLIC_URL` to your HTTPS domain (e.g. `https://loreo.example.com`) 3. Set `CORS_ORIGINS` to the same HTTPS domain -4. The server port (`SERVER_PUBLIC_PORT`) does not need to be exposed through the proxy — nginx inside the web container handles API routing +4. Set `BEHIND_PROXY=true` or list the proxy's peer IP in `TRUSTED_PROXIES` so rate-limit headers are only trusted from your proxy +5. The server port (`SERVER_PUBLIC_PORT`) does not need to be exposed through the proxy — nginx inside the web container handles API routing ## Storage From b595ff5b1bf7683af24c3320a1331befee170a6c Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:56:41 +0700 Subject: [PATCH 6/9] feat(reader): persist progress on unload via keepalive and harden client retries Reading progress was saved only through the react-query PATCH mutation, which the browser can cancel during page unload, silently dropping the final position. Route the unload save through a dedicated fetch PATCH with keepalive: true and no mutation-cache work, while normal in-app navigation keeps the cached mutation path. Stop retrying non-idempotent mutations: react-query mutations no longer retry by default, and the HTTP client retries only on 429/5xx, excluding 401/403 so a failed auth fails immediately instead of cascading. Add reader hook tests for throttling, localStorage persistence, and the unload save contract, and paragraph-tracker tests with a mocked IntersectionObserver. --- .../src/features/articles/api/update-link.ts | 10 ++ .../hooks/use-paragraph-tracker.test.ts | 136 ++++++++++++++++++ .../reader/hooks/use-reading-session.test.ts | 121 ++++++++++++++++ .../reader/hooks/use-reading-session.ts | 27 +++- apps/web/src/lib/api-client.ts | 7 +- apps/web/src/lib/react-query.ts | 2 +- apps/web/src/pages/article-reader.tsx | 6 +- 7 files changed, 300 insertions(+), 9 deletions(-) create mode 100644 apps/web/src/features/reader/hooks/use-paragraph-tracker.test.ts create mode 100644 apps/web/src/features/reader/hooks/use-reading-session.test.ts diff --git a/apps/web/src/features/articles/api/update-link.ts b/apps/web/src/features/articles/api/update-link.ts index 1a28cc1..2f97aea 100644 --- a/apps/web/src/features/articles/api/update-link.ts +++ b/apps/web/src/features/articles/api/update-link.ts @@ -20,6 +20,16 @@ const updateLink = async ({ return response.json(); }; +export const updateLinkKeepalive = ({ + body, + linkId +}: { + body: UpdateLinkBody; + linkId: string; +}): void => { + void apiClient.patchKeepalive(`links/${linkId}`, body); +}; + type UseUpdateLinkOptions = { mutationConfig?: MutationConfig; }; diff --git a/apps/web/src/features/reader/hooks/use-paragraph-tracker.test.ts b/apps/web/src/features/reader/hooks/use-paragraph-tracker.test.ts new file mode 100644 index 0000000..e44e007 --- /dev/null +++ b/apps/web/src/features/reader/hooks/use-paragraph-tracker.test.ts @@ -0,0 +1,136 @@ +import { act, renderHook } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useParagraphTracker } from './use-paragraph-tracker'; + +class MockIntersectionObserver { + static instances: MockIntersectionObserver[] = []; + + private readonly callback: IntersectionObserverCallback; + + constructor(callback: IntersectionObserverCallback) { + this.callback = callback; + MockIntersectionObserver.instances.push(this); + } + + observe = vi.fn(); + disconnect = vi.fn(); + + trigger(entries: Partial[]) { + this.callback(entries as IntersectionObserverEntry[], this as unknown as IntersectionObserver); + } +} + +describe('useParagraphTracker', () => { + beforeEach(() => { + vi.useFakeTimers(); + MockIntersectionObserver.instances = []; + vi.stubGlobal('IntersectionObserver', MockIntersectionObserver); + }); + + it('observes reader paragraphs and positions the indicator on mount', () => { + const container = document.createElement('div'); + const content = document.createElement('div'); + const indicator = document.createElement('div'); + const paragraph = document.createElement('p'); + + vi.spyOn(container, 'getBoundingClientRect').mockReturnValue({ + bottom: 400, + height: 400, + left: 0, + right: 600, + toJSON: () => {}, + top: 0, + width: 600, + x: 0, + y: 0 + }); + vi.spyOn(paragraph, 'getBoundingClientRect').mockReturnValue({ + bottom: 180, + height: 100, + left: 0, + right: 600, + toJSON: () => {}, + top: 80, + width: 600, + x: 0, + y: 80 + }); + + content.append(paragraph); + container.append(content); + + const contentRef = { current: content }; + const containerRef = { current: container }; + const { result } = renderHook(() => useParagraphTracker(contentRef, containerRef)); + result.current.indicatorRef.current = indicator; + + act(() => { + vi.advanceTimersByTime(100); + }); + + expect(MockIntersectionObserver.instances[0]?.observe).toHaveBeenCalledWith(paragraph); + expect(MockIntersectionObserver.instances[0]?.disconnect).not.toHaveBeenCalled(); + }); + + it('updates the indicator when a later paragraph becomes visible', () => { + const container = document.createElement('div'); + const content = document.createElement('div'); + const indicator = document.createElement('div'); + const firstParagraph = document.createElement('p'); + const secondParagraph = document.createElement('p'); + + vi.spyOn(container, 'getBoundingClientRect').mockReturnValue({ + bottom: 600, + height: 600, + left: 0, + right: 600, + toJSON: () => {}, + top: 0, + width: 600, + x: 0, + y: 0 + }); + vi.spyOn(firstParagraph, 'getBoundingClientRect').mockReturnValue({ + bottom: 100, + height: 100, + left: 0, + right: 600, + toJSON: () => {}, + top: 0, + width: 600, + x: 0, + y: 0 + }); + vi.spyOn(secondParagraph, 'getBoundingClientRect').mockReturnValue({ + bottom: 320, + height: 120, + left: 0, + right: 600, + toJSON: () => {}, + top: 200, + width: 600, + x: 0, + y: 200 + }); + + content.append(firstParagraph, secondParagraph); + container.append(content); + + const { result } = renderHook(() => + useParagraphTracker({ current: content }, { current: container }) + ); + result.current.indicatorRef.current = indicator; + + const observer = MockIntersectionObserver.instances[0]; + expect(observer).toBeDefined(); + + act(() => { + observer?.trigger([{ intersectionRatio: 0.8, target: secondParagraph }]); + vi.advanceTimersByTime(300); + }); + + expect(indicator.style.transform).toBe('translateY(200px)'); + expect(indicator.style.height).toBe('120px'); + }); +}); diff --git a/apps/web/src/features/reader/hooks/use-reading-session.test.ts b/apps/web/src/features/reader/hooks/use-reading-session.test.ts new file mode 100644 index 0000000..4f0a563 --- /dev/null +++ b/apps/web/src/features/reader/hooks/use-reading-session.test.ts @@ -0,0 +1,121 @@ +import { act, renderHook } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useReadingSession } from './use-reading-session'; + +const subscribe = vi.hoisted(() => vi.fn(() => vi.fn())); + +vi.mock('@tanstack/react-router', () => ({ + useRouter: () => ({ subscribe }) +})); + +const link = { + readingProgress: 0, + timeSpentReading: 0, + isRead: false +}; + +function setScrollPosition(scrollY: number) { + Object.defineProperty(window, 'scrollY', { + configurable: true, + value: scrollY, + writable: true + }); +} + +describe('useReadingSession', () => { + beforeEach(() => { + vi.useFakeTimers(); + localStorage.clear(); + subscribe.mockClear(); + Object.defineProperty(document.documentElement, 'scrollHeight', { + configurable: true, + value: 1200 + }); + Object.defineProperty(window, 'innerHeight', { + configurable: true, + value: 200 + }); + setScrollPosition(0); + vi.spyOn(window, 'scrollTo').mockImplementation(() => {}); + }); + + it('throttles scroll progress updates and persists the latest position', () => { + const { result } = renderHook(() => + useReadingSession({ + linkId: 'link-1', + link, + onSaveProgress: vi.fn() + }) + ); + + setScrollPosition(500); + act(() => { + window.dispatchEvent(new Event('scroll')); + }); + + expect(result.current.progress).toBe(0); + + act(() => { + vi.advanceTimersByTime(150); + }); + expect(result.current.progress).toBe(0.5); + + act(() => { + vi.advanceTimersByTime(1000); + }); + expect(localStorage.getItem('reading-position-link-1')).toBe( + JSON.stringify({ progress: 0.5, scrollPosition: 500 }) + ); + }); + + it('restores a persisted reading position above the resume threshold', () => { + localStorage.setItem( + 'reading-position-link-2', + JSON.stringify({ progress: 0.4, scrollPosition: 640 }) + ); + + const { result } = renderHook(() => + useReadingSession({ + linkId: 'link-2', + link, + onSaveProgress: vi.fn() + }) + ); + + expect(result.current.restorablePosition).toEqual({ + progress: 0.4, + scrollPosition: 640 + }); + + act(() => result.current.restore()); + expect(window.scrollTo).toHaveBeenCalledWith({ behavior: 'smooth', top: 640 }); + expect(result.current.restorablePosition).toBeNull(); + }); + + it('uses the unload save callback when the page is leaving', () => { + const onSaveProgress = vi.fn(); + const onSaveProgressOnUnload = vi.fn(); + renderHook(() => + useReadingSession({ + linkId: 'link-3', + link, + onSaveProgress, + onSaveProgressOnUnload + }) + ); + + setScrollPosition(500); + act(() => { + window.dispatchEvent(new Event('scroll')); + vi.advanceTimersByTime(150); + window.dispatchEvent(new Event('beforeunload')); + }); + + expect(onSaveProgressOnUnload).toHaveBeenCalledWith({ + readingProgress: 50, + timeSpentReading: 0 + }); + expect(onSaveProgress).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/features/reader/hooks/use-reading-session.ts b/apps/web/src/features/reader/hooks/use-reading-session.ts index 978ac54..e26da97 100644 --- a/apps/web/src/features/reader/hooks/use-reading-session.ts +++ b/apps/web/src/features/reader/hooks/use-reading-session.ts @@ -1,6 +1,10 @@ import { useRouter } from '@tanstack/react-router'; import { useCallback, useEffect, useRef, useState } from 'react'; +type ProgressData = { readingProgress: number; timeSpentReading: number }; + +type SaveProgress = (data: ProgressData) => void; + interface UseReadingSessionOptions { linkId: string; link: @@ -10,7 +14,8 @@ interface UseReadingSessionOptions { isRead: boolean; } | undefined; - onSaveProgress: (data: { readingProgress: number; timeSpentReading: number }) => void; + onSaveProgress: SaveProgress; + onSaveProgressOnUnload?: SaveProgress; } interface ReadingSession { @@ -22,7 +27,8 @@ interface ReadingSession { export function useReadingSession({ linkId, link, - onSaveProgress + onSaveProgress, + onSaveProgressOnUnload }: UseReadingSessionOptions): ReadingSession { const router = useRouter(); const storageKey = `reading-position-${linkId}`; @@ -56,6 +62,7 @@ export function useReadingSession({ const startTimeRef = useRef(Date.now()); const linkRef = useRef(link); const onSaveProgressRef = useRef(onSaveProgress); + const onSaveProgressOnUnloadRef = useRef(onSaveProgressOnUnload); // keep refs current useEffect(() => { @@ -64,6 +71,9 @@ export function useReadingSession({ useEffect(() => { onSaveProgressRef.current = onSaveProgress; }, [onSaveProgress]); + useEffect(() => { + onSaveProgressOnUnloadRef.current = onSaveProgressOnUnload; + }, [onSaveProgressOnUnload]); // scroll progress tracking useEffect(() => { @@ -116,7 +126,7 @@ export function useReadingSession({ useEffect(() => { startTimeRef.current = Date.now(); - const saveProgress = () => { + const saveProgress = (isUnloading = false) => { const currentLink = linkRef.current; if (!currentLink) return; @@ -129,7 +139,11 @@ export function useReadingSession({ if (!isProgressGreater && !hasSpentTime && currentLink.isRead) return; - onSaveProgressRef.current({ + const save = isUnloading + ? (onSaveProgressOnUnloadRef.current ?? onSaveProgressRef.current) + : onSaveProgressRef.current; + + save({ readingProgress: currentProgress, timeSpentReading: currentLink.timeSpentReading + timeSpentMinutes }); @@ -140,11 +154,12 @@ export function useReadingSession({ saveProgress(); } }); - window.addEventListener('beforeunload', saveProgress); + const handleBeforeUnload = () => saveProgress(true); + window.addEventListener('beforeunload', handleBeforeUnload); return () => { unsub(); - window.removeEventListener('beforeunload', saveProgress); + window.removeEventListener('beforeunload', handleBeforeUnload); }; }, [router, linkId]); diff --git a/apps/web/src/lib/api-client.ts b/apps/web/src/lib/api-client.ts index 274bb72..f8aa656 100644 --- a/apps/web/src/lib/api-client.ts +++ b/apps/web/src/lib/api-client.ts @@ -52,7 +52,7 @@ export const instance = ky.create({ prefix: env.API_URL || globalThis.location.origin, retry: { limit: 1, - statusCodes: [401, 403, 429, 500, 502, 503, 504] + statusCodes: [429, 500, 502, 503, 504] } }); @@ -72,6 +72,11 @@ export const apiClient = { return instance.patch(url, payload); }, + patchKeepalive: (url: string, body: RequestBody = {}, headers: object = {}) => { + const payload = buildPayload(body, headers); + + return instance.patch(url, { ...payload, keepalive: true, retry: 0 }); + }, post: (url: string, body: RequestBody = {}, headers: object = {}) => { const payload = buildPayload(body, headers); diff --git a/apps/web/src/lib/react-query.ts b/apps/web/src/lib/react-query.ts index b1c4162..def750a 100644 --- a/apps/web/src/lib/react-query.ts +++ b/apps/web/src/lib/react-query.ts @@ -16,7 +16,7 @@ declare module '@tanstack/react-query' { const queryConfig = { mutations: { - retry: 1, + retry: 0, retryDelay: 1000 }, queries: { diff --git a/apps/web/src/pages/article-reader.tsx b/apps/web/src/pages/article-reader.tsx index b1e441a..5bcb30e 100644 --- a/apps/web/src/pages/article-reader.tsx +++ b/apps/web/src/pages/article-reader.tsx @@ -9,6 +9,7 @@ import Tag from '@/components/ui/tag'; import { useGetLink } from '@/features/articles/api/get-link'; import { useGetUpcomingArticles } from '@/features/articles/api/get-upcoming-articles'; +import { updateLinkKeepalive } from '@/features/articles/api/update-link'; import { ArticleCard } from '@/features/articles/components/article-card'; import ArticleTypography from '@/features/reader/components/article-typography'; import FloatingProgressIndicator from '@/features/reader/components/floating-progress-indicator'; @@ -57,7 +58,10 @@ function ArticleReaderPage() { const { progress, restorablePosition, restore } = useReadingSession({ linkId: id, link: article, - onSaveProgress: env.isDemo ? () => {} : (data) => actions.updateLink(id, data) + onSaveProgress: env.isDemo ? () => {} : (data) => actions.updateLink(id, data), + onSaveProgressOnUnload: env.isDemo + ? () => {} + : (data) => updateLinkKeepalive({ body: data, linkId: id }) }); const { shouldShowFloating } = useProgressIndicator(); From 260ce6c21ff7f69581bf67a5a3f286e246cd3ee6 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 10:57:02 +0700 Subject: [PATCH 7/9] chore: document url-guard layering and add dependency audit to ci The article-url-guard and browser-request-guard wrappers looked like duplicates but are intentionally separate: article navigation reports a typed failure to the extraction workflow, while browser subresource interception needs a boolean per request. Document that layering inline so the two are not collapsed by mistake. Add a `pnpm audit --prod --audit-level=high` step to CI. It runs non-blocking for now while the existing dependency baseline is remediated; flip it to gating once that baseline is clean. --- .github/workflows/ci.yml | 5 +++++ apps/server/src/lib/article-url-guard.ts | 3 +++ apps/server/src/lib/browser-request-guard.ts | 3 +++ 3 files changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 952db19..8a020d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,11 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + # Report dependency vulnerabilities without blocking until the baseline is remediated. + - name: Dependency audit + continue-on-error: true + run: pnpm audit --prod --audit-level=high + - name: Lint run: pnpm lint diff --git a/apps/server/src/lib/article-url-guard.ts b/apps/server/src/lib/article-url-guard.ts index 17c3ed5..74bed6a 100644 --- a/apps/server/src/lib/article-url-guard.ts +++ b/apps/server/src/lib/article-url-guard.ts @@ -1,5 +1,8 @@ import { isValidUrl } from './url-validator.js'; +// Keep this domain-specific assertion separate from browser-request-guard: +// article navigation reports a typed failure to the extraction workflow, while +// browser subresource interception needs a boolean decision per request. export async function assertSafeArticleUrl(url: string): Promise { if (!(await isValidUrl(url))) { throw new Error('Article URL is not allowed'); diff --git a/apps/server/src/lib/browser-request-guard.ts b/apps/server/src/lib/browser-request-guard.ts index 6739e5b..a6efc7d 100644 --- a/apps/server/src/lib/browser-request-guard.ts +++ b/apps/server/src/lib/browser-request-guard.ts @@ -1,5 +1,8 @@ import { isValidUrl } from './url-validator.js'; +// Intentional layering with article-url-guard: browser requests are evaluated +// per subresource and return a boolean for Playwright route handling, whereas +// article navigation uses an exception-oriented domain boundary. export async function isAllowedBrowserRequestUrl(url: string): Promise { return isValidUrl(url); } From 43c099527cabd59b16975c96c15e3ab7d9541e04 Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 11:47:38 +0700 Subject: [PATCH 8/9] feat(files): gate legacy shared article images behind an opt-in flag Existing deployments have article body images stored under the shared/articles/* prefix from before per-user storage. The hard 403 introduced for ownership safety would break those articles on upgrade. Keep the secure default (deny shared/articles/*) but add an ALLOW_LEGACY_SHARED_ARTICLES env flag that serves the legacy prefix while on, so older articles keep rendering during migration. New extractions always use the ownership-checked user-/articles/* path regardless of the flag; the two are separate key prefixes and code paths. Extract the access decision into an env-free predicate (isFileAccessForbidden) following the rate-limit convention so both flag states are unit-tested without touching process env. Document the flag for self-hosters. --- apps/server/.env.example | 5 ++ apps/server/.env.test.example | 1 + apps/server/src/lib/env-config.ts | 8 +++ .../server/src/routes/files/files.handlers.ts | 30 +++++++++- apps/server/src/routes/files/files.test.ts | 60 ++++++++++++++++++- docs/SELF_HOSTING.md | 2 + 6 files changed, 104 insertions(+), 2 deletions(-) diff --git a/apps/server/.env.example b/apps/server/.env.example index e783960..cc85cb8 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -45,6 +45,11 @@ PUBLIC_URL=http://localhost:3000 STORAGE_PROVIDER=local STORAGE_PATH=./data/storage +# Existing deployments only: serve article body images extracted before per-user storage +# (shared/articles/*). Leave false for fresh installs; set true to keep older articles rendering +# until their images are backfilled, then set false again. +ALLOW_LEGACY_SHARED_ARTICLES=false + # S3-compatible storage (optional; required only when STORAGE_PROVIDER=s3) S3_ENDPOINT= S3_REGION=auto diff --git a/apps/server/.env.test.example b/apps/server/.env.test.example index d82b010..388cccb 100644 --- a/apps/server/.env.test.example +++ b/apps/server/.env.test.example @@ -30,6 +30,7 @@ BODY_SIZE_LIMIT=102400 PUBLIC_URL=http://localhost:3000 STORAGE_PROVIDER=local STORAGE_PATH=./data/storage-test +ALLOW_LEGACY_SHARED_ARTICLES=false S3_ENDPOINT= S3_REGION=auto diff --git a/apps/server/src/lib/env-config.ts b/apps/server/src/lib/env-config.ts index 504ec46..d430e47 100644 --- a/apps/server/src/lib/env-config.ts +++ b/apps/server/src/lib/env-config.ts @@ -63,6 +63,14 @@ const envSchema = z STORAGE_PROVIDER: z.enum(['local', 'local-docker', 's3']).default('local'), + // Serve pre-user-scoping article body images (shared/articles/*) extracted before the + // ownership-checked storage scheme. Leave off for fresh deployments; enable for existing + // installations until their legacy assets are backfilled, then turn off again. + ALLOW_LEGACY_SHARED_ARTICLES: z + .enum(['true', 'false']) + .default('false') + .transform((value) => value === 'true'), + // S3-compatible storage (AWS S3, Cloudflare R2, MinIO, etc.) // S3_ENDPOINT: omit for AWS S3, set to custom URL for R2/MinIO S3_ENDPOINT: z.string().default(''), diff --git a/apps/server/src/routes/files/files.handlers.ts b/apps/server/src/routes/files/files.handlers.ts index 39c25a2..e8de0a8 100644 --- a/apps/server/src/routes/files/files.handlers.ts +++ b/apps/server/src/routes/files/files.handlers.ts @@ -1,5 +1,6 @@ import type { Context } from 'hono'; +import { env } from '@/lib/env-config.js'; import { logger } from '@/lib/logger.js'; import { errorResponse, HttpStatus } from '@/lib/response.js'; import type { AppBindings, AppRouteHandler } from '@/lib/types.js'; @@ -8,6 +9,26 @@ import { storageService } from '@/services/storage.service.js'; import type { GetFileRoute } from './files.routes.js'; +export interface FileAccessInput { + isShared: boolean; + isUserOwned: boolean; + isLegacySharedArticle: boolean; + legacyArticlesAllowed: boolean; +} + +// Pure access-policy predicate (kept env-free) so the legacy-article flag behaviour is unit-testable +// without touching the process env. New user-scoped article keys are always ownership-checked; +// this only controls whether the pre-user-scoping shared/articles/* prefix stays readable. +export function isFileAccessForbidden({ + isShared, + isUserOwned, + isLegacySharedArticle, + legacyArticlesAllowed +}: FileAccessInput): boolean { + if (!isShared && !isUserOwned) return true; + return isLegacySharedArticle && !legacyArticlesAllowed; +} + const getFileHandler: AppRouteHandler = async (c) => { const { key } = c.req.valid('param'); @@ -31,7 +52,14 @@ async function serveFile(c: Context, key: string) { } else { const isUserOwned = storageService.isUserFile(key, user.id); - if ((!isShared && !isUserOwned) || isLegacySharedArticle) { + if ( + isFileAccessForbidden({ + isShared, + isUserOwned, + isLegacySharedArticle, + legacyArticlesAllowed: env.ALLOW_LEGACY_SHARED_ARTICLES + }) + ) { const response = errorResponse('Access denied to this file', HttpStatus.FORBIDDEN); return c.json(response, response.status); diff --git a/apps/server/src/routes/files/files.test.ts b/apps/server/src/routes/files/files.test.ts index 335e317..17b3cfe 100644 --- a/apps/server/src/routes/files/files.test.ts +++ b/apps/server/src/routes/files/files.test.ts @@ -18,6 +18,7 @@ import { storageService } from '@/services/storage.service.js'; import type { UserWithoutPassword } from '@/types/auth.js'; +import { isFileAccessForbidden } from './files.handlers.js'; import router from './files.index.js'; const STORAGE_ROOT = path.resolve(process.cwd(), 'data/storage'); @@ -262,7 +263,7 @@ describe('GET /files/:key', () => { expect(response.status).toBe(HttpStatus.OK); }); - it('returns 403 for legacy shared article files', async () => { + it('returns 403 for legacy shared article files by default (flag off)', async () => { const response = await client.files[':key{.*}'].$get( { param: { key: legacySharedArticleKey } }, { headers: { Cookie: authCookie } } @@ -323,3 +324,60 @@ describe('GET /files/:key', () => { }); }); }); + +describe('isFileAccessForbidden', () => { + it('denies legacy shared article keys by default (flag off)', () => { + expect( + isFileAccessForbidden({ + isShared: true, + isUserOwned: false, + isLegacySharedArticle: true, + legacyArticlesAllowed: false + }) + ).toBe(true); + }); + + it('allows legacy shared article keys when the migration flag is on', () => { + expect( + isFileAccessForbidden({ + isShared: true, + isUserOwned: false, + isLegacySharedArticle: true, + legacyArticlesAllowed: true + }) + ).toBe(false); + }); + + it('denies keys that are neither shared nor user-owned', () => { + expect( + isFileAccessForbidden({ + isShared: false, + isUserOwned: false, + isLegacySharedArticle: false, + legacyArticlesAllowed: false + }) + ).toBe(true); + }); + + it('allows user-owned keys regardless of the flag', () => { + expect( + isFileAccessForbidden({ + isShared: false, + isUserOwned: true, + isLegacySharedArticle: false, + legacyArticlesAllowed: false + }) + ).toBe(false); + }); + + it('allows non-article shared keys regardless of the flag', () => { + expect( + isFileAccessForbidden({ + isShared: true, + isUserOwned: false, + isLegacySharedArticle: false, + legacyArticlesAllowed: false + }) + ).toBe(false); + }); +}); diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index 1143e2f..068d50b 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -126,6 +126,8 @@ The API server handles authentication, article management, RSS feed subscription The server uses `STORAGE_PROVIDER: local-docker` by default, storing uploaded files in the `storage_data` volume. See the Storage section for S3 configuration. +Article body images extracted before per-user storage are served from the `shared/articles/*` prefix. New installations deny that prefix by default; **existing deployments** that still reference those images should set `ALLOW_LEGACY_SHARED_ARTICLES=true` so older articles keep rendering (new images always use the ownership-checked per-user path). Backfill the legacy assets and unset the flag when ready. + ### Web The web container is portable: it does not require rebuilding when your deployment URL or server port changes. The browser calls the API on the same origin (through nginx's reverse proxy), and the nginx template resolves `${API_UPSTREAM}` at container startup. From bea3d4f60d987e5406cd9a7cc9c4996fe87bf6fa Mon Sep 17 00:00:00 2001 From: Muhammad Fadhil Date: Thu, 30 Jul 2026 11:53:32 +0700 Subject: [PATCH 9/9] docs: add unreleased changelog entries for the platform hardening pass --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b21404..eaf31d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Store extracted article body images under per-user, ownership-checked paths so users cannot read each other's images; add `ALLOW_LEGACY_SHARED_ARTICLES` to keep older articles rendering during migration. +- Scope highlight and tag enrichment to the requesting user across home, upcoming, and search. +- Validate image uploads by sniffing file content instead of trusting filename or Content-Type. +- Gate rate-limit client identification on forwarded headers behind `BEHIND_PROXY`/`TRUSTED_PROXIES`; default to the direct connection IP. +- Assign the first admin account atomically so concurrent initial sign-ups settle on exactly one admin. +- Save reading progress when closing the tab via a keepalive request instead of dropping the final update. +- Stop retrying non-idempotent actions such as link create/delete, and drop auth errors from HTTP retry cascades. +- Resume stalled article extraction and reset stuck processing rows instead of skipping retries. +- Align the create-link tag shape between web and server, guarded by a contract test that fails on drift. +- Trim full text content from list and search responses. +- Parallelize home-suggestion reads and collapse short/long reading counts into a single query. +- Declare the generated search index column and indexes in the schema so schema sync cannot drop them. +- Add a production dependency audit step to CI. + ## [v0.2.0] - 2026-07-29 - Add user-curated RSS and Atom subscriptions with staged Review, Saved, and Dismissed collections.