Hotfix: neutralize UGC link spam and deindex user pages

next.config.ts

@@ -30,6 +30,11 @@ const CONTENT_SECURITY_POLICY = [
   "upgrade-insecure-requests",
 ].join("; ");
 
+// User-generated surfaces are excluded from search while the spam backlog is
+// cleaned up. Sent as a header (not just page metadata) so it applies to
+// already-cached/raw responses and takes effect on the next crawl.
+const NOINDEX_HEADERS = [{ key: "X-Robots-Tag", value: "noindex, nofollow" }];
+
 const nextConfig: NextConfig = {
   output: "standalone",
   async headers() {
@@ -40,7 +45,13 @@ const nextConfig: NextConfig = {
         { key: "Content-Security-Policy", value: CONTENT_SECURITY_POLICY }
       );
     }
-    return [{ source: "/:path*", headers }];
+    return [
+      { source: "/:path*", headers },
+      { source: "/explore", headers: NOINDEX_HEADERS },
+      { source: "/public/:path*", headers: NOINDEX_HEADERS },
+      { source: "/share/:path*", headers: NOINDEX_HEADERS },
+      { source: "/c/:path*", headers: NOINDEX_HEADERS },
+    ];
   },
 };
 

src/__tests__/api/doc-access.test.ts

@@ -156,6 +156,31 @@ describe("GET /docs/:slug access paths", () => {
     expect(data.views_count).toBeDefined();
   });
 
+  it("deindexes user-content pages via X-Robots-Tag", async () => {
+    const { doc } = await createDoc({ visibility: "public" });
+
+    const share = await fetch(`${BASE_URL}/share/${doc.slug}`);
+    expect(share.headers.get("x-robots-tag")).toContain("noindex");
+
+    const explore = await fetch(`${BASE_URL}/explore`);
+    expect(explore.headers.get("x-robots-tag")).toContain("noindex");
+  }, 20_000);
+
+  it("only lists account-owned docs in the explore feed", async () => {
+    const user = await prisma.user.create({
+      data: { email: `feed-${generateSlug()}@example.com` },
+    });
+    const owned = await createDoc({ visibility: "public", userId: user.id });
+    const anonymous = await createDoc({ visibility: "public" });
+
+    const res = await fetch(`${BASE_URL}/api/v1/explore`);
+    const data = await res.json();
+    const slugs = data.docs.map((d: { slug: string }) => d.slug);
+
+    expect(slugs).toContain(owned.doc.slug);
+    expect(slugs).not.toContain(anonymous.doc.slug);
+  });
+
   it("does not expose owner-only fields to share-token readers", async () => {
     const shareToken = generateShareToken();
     const { doc } = await createDoc({ shareToken, meta: { agent: "claude" } });

src/app/api/v1/explore/route.ts

@@ -7,7 +7,9 @@ export async function GET(request: NextRequest) {
   const cursor = request.nextUrl.searchParams.get("cursor");
 
   const docs = await prisma.doc.findMany({
-    where: { visibility: "public" },
+    // Mirrors the /explore page: only account-owned docs while anonymous
+    // creation is gated against the current spam wave.
+    where: { visibility: "public", userId: { not: null } },
     orderBy: { createdAt: "desc" },
     take: PAGE_SIZE + 1,
     ...(cursor && {

src/app/explore/page.tsx

@@ -12,6 +12,8 @@ export const metadata = {
     description:
       "Browse public markdown documents shared by humans and AI agents.",
   },
+  // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up.
+  robots: { index: false, follow: false },
 };
 
 export const dynamic = "force-dynamic";
@@ -20,7 +22,9 @@ const PAGE_SIZE = 20;
 
 export default async function ExplorePage() {
   const docs = await prisma.doc.findMany({
-    where: { visibility: "public" },
+    // Only account-owned docs until anonymous creation is gated — the feed is
+    // currently dominated by automated spam.
+    where: { visibility: "public", userId: { not: null } },
     orderBy: { createdAt: "desc" },
     take: PAGE_SIZE + 1,
     select: {

src/app/public/[seoSlug]/page.tsx

@@ -72,9 +72,10 @@ export async function generateMetadata({ params }: Props): Promise<Metadata> {
       title,
       description,
     },
+    // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up.
     robots: {
-      index: true,
-      follow: true,
+      index: false,
+      follow: false,
     },
   };
 }

src/app/share/[slug]/page.tsx

@@ -41,9 +41,10 @@ export async function generateMetadata({ params }: Props) {
       title,
       description,
     },
+    // Temporarily deindexed while the anonymous-doc spam backlog is cleaned up.
     robots: {
-      index: doc.visibility === "public",
-      follow: true,
+      index: false,
+      follow: false,
     },
     alternates: doc.visibility === "public" && doc.seoSlug
       ? { canonical: `/public/${doc.seoSlug}` }

src/app/sitemap.ts

@@ -10,7 +10,6 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     { url: BASE_URL, lastModified: new Date(), changeFrequency: "weekly", priority: 1.0 },
     { url: `${BASE_URL}/docs`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 },
     { url: `${BASE_URL}/about`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.5 },
-    { url: `${BASE_URL}/explore`, lastModified: new Date(), changeFrequency: "daily", priority: 0.8 },
     { url: `${BASE_URL}/example`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.6 },
     { url: `${BASE_URL}/agents`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 },
     { url: `${BASE_URL}/cli`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 },
@@ -26,8 +25,16 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     { url: `${BASE_URL}/vs/stackedit`, lastModified: new Date(), changeFrequency: "monthly", priority: 0.7 },
   ];
 
+  // Only verified-owner docs are submitted: anonymous public docs are the
+  // target of an ongoing spam campaign and stay out of the index until the
+  // backlog is cleaned up.
   const publicDocs = await prisma.doc.findMany({
-    where: { visibility: "public", seoSlug: { not: null } },
+    where: {
+      visibility: "public",
+      seoSlug: { not: null },
+      userId: { not: null },
+      user: { is: { emailVerifiedAt: { not: null } } },
+    },
     select: { seoSlug: true, updatedAt: true },
     orderBy: { updatedAt: "desc" },
     take: 1000,

src/components/CommentMarkdown.tsx

@@ -1,3 +1,4 @@
+import type { ReactNode } from "react";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 
@@ -44,10 +45,41 @@ function normalizeMarkdown(text: string): string {
   return result.trim();
 }
 
+const components = {
+  a({
+    href,
+    children,
+    title,
+  }: {
+    href?: string;
+    children?: ReactNode;
+    title?: string;
+  }) {
+    const isExternal = typeof href === "string" && /^https?:\/\//i.test(href);
+    if (!isExternal) {
+      return (
+        <a href={href} title={title}>
+          {children}
+        </a>
+      );
+    }
+    return (
+      <a
+        href={href}
+        title={title}
+        target="_blank"
+        rel="nofollow ugc noopener noreferrer"
+      >
+        {children}
+      </a>
+    );
+  },
+};
+
 export default function CommentMarkdown({ content }: { content: string }) {
   return (
     <div className="comment-markdown">
-      <ReactMarkdown remarkPlugins={[remarkGfm]}>
+      <ReactMarkdown remarkPlugins={[remarkGfm]} components={components}>
         {normalizeMarkdown(content)}
       </ReactMarkdown>
     </div>

src/components/MarkdownPreview.tsx

@@ -43,6 +43,28 @@ function textContent(node: ReactNode): string {
 }
 
 const components: Components = {
+  a({ href, children, title }) {
+    // User-authored content is untrusted: never hand it dofollow link equity.
+    // External links get ugc + nofollow; in-page anchors stay plain.
+    const isExternal = typeof href === "string" && /^https?:\/\//i.test(href);
+    if (!isExternal) {
+      return (
+        <a href={href} title={title}>
+          {children}
+        </a>
+      );
+    }
+    return (
+      <a
+        href={href}
+        title={title}
+        target="_blank"
+        rel="nofollow ugc noopener noreferrer"
+      >
+        {children}
+      </a>
+    );
+  },
   h2({ children }) {
     const text = textContent(children);
     const id = slugify(text);