From 62dc32d5c77268f464a3db6f2662883c3f2f8e38 Mon Sep 17 00:00:00 2001 From: Facundo Farias Date: Sun, 25 Oct 2026 09:34:48 +0100 Subject: [PATCH 1/2] fix(seo): neutralize UGC link spam and deindex user pages Anonymous public docs are being used as a dofollow link farm (~11.8k docs, ~9.7k gambling-related, ramping since late July). Hotfix: - Render markdown and comment links with rel="nofollow ugc noopener noreferrer" so user content can never pass link equity. - Send X-Robots-Tag: noindex, nofollow on /explore, /public/*, /share/* and /c/*; set the same in page metadata. - Drop user docs and /explore from the sitemap; only verified-owner docs are submitted. - Limit the explore page + API to account-owned docs while the spam backlog is cleaned up. Tests cover the robots header and the explore ownership filter. --- next.config.ts | 13 ++++++++++- src/__tests__/api/doc-access.test.ts | 25 ++++++++++++++++++++ src/app/api/v1/explore/route.ts | 4 +++- src/app/explore/page.tsx | 6 ++++- src/app/public/[seoSlug]/page.tsx | 5 ++-- src/app/share/[slug]/page.tsx | 5 ++-- src/app/sitemap.ts | 11 +++++++-- src/components/CommentMarkdown.tsx | 34 +++++++++++++++++++++++++++- src/components/MarkdownPreview.tsx | 22 ++++++++++++++++++ 9 files changed, 115 insertions(+), 10 deletions(-) diff --git a/next.config.ts b/next.config.ts index 366ba5c..55ee0e6 100644 --- a/next.config.ts +++ b/next.config.ts @@ -30,6 +30,11 @@ const CONTENT_SECURITY_POLICY = [ "upgrade-insecure-requests", ].join("; "); +// User-generated surfaces are excluded from search while the spam backlog is +// cleaned up. Sent as a header (not just page metadata) so it applies to +// already-cached/raw responses and takes effect on the next crawl. +const NOINDEX_HEADERS = [{ key: "X-Robots-Tag", value: "noindex, nofollow" }]; + const nextConfig: NextConfig = { output: "standalone", async headers() { @@ -40,7 +45,13 @@ const nextConfig: NextConfig = { { key: "Content-Security-Policy", value: CONTENT_SECURITY_POLICY } ); } - return [{ source: "/:path*", headers }]; + return [ + { source: "/:path*", headers }, + { source: "/explore", headers: NOINDEX_HEADERS }, + { source: "/public/:path*", headers: NOINDEX_HEADERS }, + { source: "/share/:path*", headers: NOINDEX_HEADERS }, + { source: "/c/:path*", headers: NOINDEX_HEADERS }, + ]; }, }; diff --git a/src/__tests__/api/doc-access.test.ts b/src/__tests__/api/doc-access.test.ts index c907cca..e9d3486 100644 --- a/src/__tests__/api/doc-access.test.ts +++ b/src/__tests__/api/doc-access.test.ts @@ -156,6 +156,31 @@ describe("GET /docs/:slug access paths", () => { expect(data.views_count).toBeDefined(); }); + it("deindexes user-content pages via X-Robots-Tag", async () => { + const { doc } = await createDoc({ visibility: "public" }); + + const share = await fetch(`${BASE_URL}/share/${doc.slug}`); + expect(share.headers.get("x-robots-tag")).toContain("noindex"); + + const explore = await fetch(`${BASE_URL}/explore`); + expect(explore.headers.get("x-robots-tag")).toContain("noindex"); + }); + + it("only lists account-owned docs in the explore feed", async () => { + const user = await prisma.user.create({ + data: { email: `feed-${generateSlug()}@example.com` }, + }); + const owned = await createDoc({ visibility: "public", userId: user.id }); + const anonymous = await createDoc({ visibility: "public" }); + + const res = await fetch(`${BASE_URL}/api/v1/explore`); + const data = await res.json(); + const slugs = data.docs.map((d: { slug: string }) => d.slug); + + expect(slugs).toContain(owned.doc.slug); + expect(slugs).not.toContain(anonymous.doc.slug); + }); + it("does not expose owner-only fields to share-token readers", async () => { const shareToken = generateShareToken(); const { doc } = await createDoc({ shareToken, meta: { agent: "claude" } }); diff --git a/src/app/api/v1/explore/route.ts b/src/app/api/v1/explore/route.ts index d80695b..fc518c5 100644 --- a/src/app/api/v1/explore/route.ts +++ b/src/app/api/v1/explore/route.ts @@ -7,7 +7,9 @@ export async function GET(request: NextRequest) { const cursor = request.nextUrl.searchParams.get("cursor"); const docs = await prisma.doc.findMany({ - where: { visibility: "public" }, + // Mirrors the /explore page: only account-owned docs while anonymous + // creation is gated against the current spam wave. + where: { visibility: "public", userId: { not: null } }, orderBy: { createdAt: "desc" }, take: PAGE_SIZE + 1, ...(cursor && { diff --git a/src/app/explore/page.tsx b/src/app/explore/page.tsx index b8d51bd..227c54f 100644 --- a/src/app/explore/page.tsx +++ b/src/app/explore/page.tsx @@ -12,6 +12,8 @@ export const metadata = { description: "Browse public markdown documents shared by humans and AI agents.", }, + // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up. + robots: { index: false, follow: false }, }; export const dynamic = "force-dynamic"; @@ -20,7 +22,9 @@ const PAGE_SIZE = 20; export default async function ExplorePage() { const docs = await prisma.doc.findMany({ - where: { visibility: "public" }, + // Only account-owned docs until anonymous creation is gated — the feed is + // currently dominated by automated spam. + where: { visibility: "public", userId: { not: null } }, orderBy: { createdAt: "desc" }, take: PAGE_SIZE + 1, select: { diff --git a/src/app/public/[seoSlug]/page.tsx b/src/app/public/[seoSlug]/page.tsx index fef29bc..aca703a 100644 --- a/src/app/public/[seoSlug]/page.tsx +++ b/src/app/public/[seoSlug]/page.tsx @@ -72,9 +72,10 @@ export async function generateMetadata({ params }: Props): Promise { title, description, }, + // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up. robots: { - index: true, - follow: true, + index: false, + follow: false, }, }; } diff --git a/src/app/share/[slug]/page.tsx b/src/app/share/[slug]/page.tsx index def364d..0e8a418 100644 --- a/src/app/share/[slug]/page.tsx +++ b/src/app/share/[slug]/page.tsx @@ -41,9 +41,10 @@ export async function generateMetadata({ params }: Props) { title, description, }, + // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up. robots: { - index: doc.visibility === "public", - follow: true, + index: false, + follow: false, }, alternates: doc.visibility === "public" && doc.seoSlug ? { canonical: `/public/${doc.seoSlug}` } diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts index 6a22ecd..e95d8fd 100644 --- a/src/app/sitemap.ts +++ b/src/app/sitemap.ts @@ -10,7 +10,6 @@ export default async function sitemap(): Promise { { url: BASE_URL, lastModified: new Date(), changeFrequency: "weekly", priority: 1.0 }, { url: `${BASE_URL}/docs`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 }, { url: `${BASE_URL}/about`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.5 }, - { url: `${BASE_URL}/explore`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 }, { url: `${BASE_URL}/example`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.6 }, { url: `${BASE_URL}/agents`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 }, { url: `${BASE_URL}/cli`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 }, @@ -26,8 +25,16 @@ export default async function sitemap(): Promise { { url: `${BASE_URL}/vs/stackedit`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 }, ]; + // Only verified-owner docs are submitted: anonymous public docs are the + // target of an ongoing spam campaign and stay out of the index until the + // backlog is cleaned up. const publicDocs = await prisma.doc.findMany({ - where: { visibility: "public", seoSlug: { not: null } }, + where: { + visibility: "public", + seoSlug: { not: null }, + userId: { not: null }, + user: { is: { emailVerifiedAt: { not: null } } }, + }, select: { seoSlug: true, updatedAt: true }, orderBy: { updatedAt: "desc" }, take: 1000, diff --git a/src/components/CommentMarkdown.tsx b/src/components/CommentMarkdown.tsx index 318ff6f..82cd797 100644 --- a/src/components/CommentMarkdown.tsx +++ b/src/components/CommentMarkdown.tsx @@ -1,3 +1,4 @@ +import type { ReactNode } from "react"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; @@ -44,10 +45,41 @@ function normalizeMarkdown(text: string): string { return result.trim(); } +const components = { + a({ + href, + children, + title, + }: { + href?: string; + children?: ReactNode; + title?: string; + }) { + const isExternal = typeof href === "string" && /^https?:\/\//i.test(href); + if (!isExternal) { + return ( + + {children} + + ); + } + return ( + + {children} + + ); + }, +}; + export default function CommentMarkdown({ content }: { content: string }) { return (
- + {normalizeMarkdown(content)}
diff --git a/src/components/MarkdownPreview.tsx b/src/components/MarkdownPreview.tsx index 64cb1da..5ffeddf 100644 --- a/src/components/MarkdownPreview.tsx +++ b/src/components/MarkdownPreview.tsx @@ -43,6 +43,28 @@ function textContent(node: ReactNode): string { } const components: Components = { + a({ href, children, title }) { + // User-authored content is untrusted: never hand it dofollow link equity. + // External links get ugc + nofollow; in-page anchors stay plain. + const isExternal = typeof href === "string" && /^https?:\/\//i.test(href); + if (!isExternal) { + return ( + + {children} + + ); + } + return ( + + {children} + + ); + }, h2({ children }) { const text = textContent(children); const id = slugify(text); From ee840ee871dad982a51f2a22afb15bfb7d13be18 Mon Sep 17 00:00:00 2001 From: Facundo Farias Date: Sun, 25 Oct 2026 09:37:47 +0100 Subject: [PATCH 2/2] test(seo): allow cold page compile in robots header test --- src/__tests__/api/doc-access.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/api/doc-access.test.ts b/src/__tests__/api/doc-access.test.ts index e9d3486..3924644 100644 --- a/src/__tests__/api/doc-access.test.ts +++ b/src/__tests__/api/doc-access.test.ts @@ -164,7 +164,7 @@ describe("GET /docs/:slug access paths", () => { const explore = await fetch(`${BASE_URL}/explore`); expect(explore.headers.get("x-robots-tag")).toContain("noindex"); - }); + }, 20_000); it("only lists account-owned docs in the explore feed", async () => { const user = await prisma.user.create({