add markdown building and llms.txt

deploy.php

@@ -79,6 +79,8 @@
     run('apt update && apt install brotli -y');
     within('{{release_or_current_path}}/docs/_build/html', function () {
         run('find . -name "*.html" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');
+        run('find . -name "*.html.md" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');
+        run('find . -name "llms.txt" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');
         run('find . -name "*.css" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');
         run('find . -name "*.js" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');
         run('find . -name "*.svg" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;');

docs/_templates/breadcrumbs.html

@@ -30,6 +30,9 @@
     {%- endblock %}
     {%- block breadcrumbs_aside %}
       <li class="wy-breadcrumbs-aside">
+        {%- if markdown_page_url %}
+          <a href="{{ markdown_page_url|e }}" type="text/markdown"><span class="fa fa-file-alt"></span> Markdown</a>
+        {%- endif %}
         {%- if hasdoc(pagename) and display_vcs_links %}
           {%- if display_github %}
             {%- if check_meta and 'github_url' in meta %}

docs/_templates/layout.html

@@ -65,6 +65,14 @@
     <link rel="canonical" href="{{ pageurl|e }}" />
   {%- endif -%}
 
+  {#- LLM-FRIENDLY MARKDOWN VERSION #}
+  {%- if markdown_page_url %}
+    <link rel="alternate" type="text/markdown" href="{{ markdown_page_url|e }}" />
+  {%- endif -%}
+  {%- if llms_txt_url %}
+    <link rel="describedby" type="text/plain" href="{{ llms_txt_url|e }}" />
+  {%- endif -%}
+
   {#- JAVASCRIPTS #}
   {%- block scripts %}
   <!--[if lt IE 9]>

docs/conf.py

@@ -49,6 +49,9 @@
     "hypernode.sphinx.extensions.updated_at",
     "hypernode.sphinx.extensions.meta_robots",
     "hypernode.sphinx.extensions.github_actions_logging",
+    "hypernode.sphinx.extensions.markdown_pages",
+    "hypernode.sphinx.extensions.markdown_pages_writer",
+    "hypernode.sphinx.extensions.llms_txt",
     "sphinx_docsearch",
 ]
 

docs/hypernode-platform/botstopper/how-to-use-botstopper.md

@@ -68,7 +68,7 @@
 | `moderate`   | Blocks AI training crawlers and unknown AI bots. Allows documented AI search bots and user-triggered AI clients. |
 | `permissive` | Allows documented AI bots. Blocks unknown AI-style bots.                                                         |
 
-Use `aggressive` if you want the strictest AI blocking. Use `moderate` if you want to block AI training while keeping documented AI search and user tools working. Use `permissive` if you only want to block unclear or undocumented AI crawlers.
+See [How to Choose a Botstopper AI Policy](./how-to-choose-a-botstopper-ai-policy.md) for a detailed comparison of what each policy blocks and how verified AI bots are treated.
 
 Some AI crawlers also require `robots.txt` rules before they respect your opt-out. Botstopper blocks requests at the webserver layer, but `robots.txt` is still useful for crawlers that require policy signals there. See the [Magento 1 robots.txt](../../ecommerce-applications/magento-1/how-to-create-a-robots-txt-for-your-magento-1-shop.md) or [Magento 2 robots.txt](../../ecommerce-applications/magento-2/how-to-create-a-robots-txt-for-magento-2-x.md) articles if you need to configure one.
 

hypernode/sphinx/extensions/llms_txt.py

@@ -0,0 +1,179 @@
+import re
+from typing import List, Tuple
+
+import sphinx.addnodes as addnodes
+
+from hypernode.sphinx.extensions.markdown_tree import site_url_for, title_of
+
+DESCRIPTION = (
+    "Documentation for Hypernode: the managed hosting platform for Magento, "
+    "Shopware, Laravel and other web applications."
+)
+
+
+def write_llms_txt(app, exception) -> None:
+    env = app.builder.env
+    config = app.config
+
+    sections, parents = _sections(env)
+    in_sections = {docname for _, entries in sections for docname in entries}
+
+    base_url = (config.html_baseurl or "").rstrip("/")
+    intro = (
+        "Hypernode is a managed hosting platform for webshops and web applications,"
+        " maintained by Team.blue. This file is a curated map of the documentation"
+        " at {}. Every page is also available as clean markdown by appending"
+        " `.html.md` to its URL.".format(base_url if base_url else "this site")
+    )
+    lines: List[str] = [
+        "# Hypernode Documentation",
+        "",
+        "> " + DESCRIPTION,
+        "",
+        intro,
+        "",
+    ]
+
+    for caption, entries in sections:
+        lines.append("## " + caption)
+        lines.append("")
+        lines.extend(_link_list(env, config, entries))
+        lines.append("")
+
+    optional = _optional(env, in_sections | set(parents))
+    if optional:
+        lines.append("## Optional")
+        lines.append("")
+        lines.extend(_link_list(env, config, optional))
+        lines.append("")
+
+    with open(str(app.outdir) + "/llms.txt", "w", encoding="utf-8") as f:
+        f.write("\n".join(lines).rstrip() + "\n")
+
+
+def _sections(env) -> Tuple[List[Tuple[str, List[str]]], List[str]]:
+    """
+    Build H2 sections from the top-level index toctrees. Category pages
+    (e.g. hypernode-platform/php) are flattened into their articles; the
+    flattened-away pure stub pages are returned separately so they can be
+    excluded elsewhere.
+    """
+    toc = env.tocs.get("index")
+    if toc is None:
+        return [], []
+
+    sections: List[Tuple[str, List[str]]] = []
+    parents: List[str] = []
+    for node in toc.findall(addnodes.toctree):
+        caption = node.get("caption") or "Documentation"
+        entries: List[str] = []
+        for _, entry in node.attributes["entries"]:
+            if entry == "self":
+                continue
+            flattened = _flatten(env, entry)
+            if entry not in flattened:
+                parents.append(entry)
+            entries.extend(flattened)
+        if entries:
+            sections.append((caption, entries))
+    return sections, parents
+
+
+def _flatten(env, docname: str, depth: int = 0) -> List[str]:
+    """
+    Flatten a category page into its article docnames. Pages with real
+    content of their own (chapter landing pages) are kept in the list
+    before their children; pure toctree stubs are dropped.
+    """
+    toc = env.tocs.get(docname)
+    if toc is None or depth > 5:
+        return [docname]
+
+    child_entries: List[str] = []
+    for node in toc.findall(addnodes.toctree):
+        for _, entry in node.attributes["entries"]:
+            if entry == "self":
+                continue
+            child_entries.extend(_flatten(env, entry, depth + 1))
+
+    if not child_entries:
+        return [docname]
+    if _wordcount(env, docname) < 5:
+        return child_entries
+    return [docname] + child_entries
+
+
+def _wordcount(env, docname: str) -> int:
+    metadata = env.metadata.get(docname, {})
+    return metadata.get("wordcount", {}).get("words", 0)
+
+
+def _link_list(env, config, docnames: List[str]) -> List[str]:
+    lines: List[str] = []
+    for docname in docnames:
+        title = title_of(env, docname)
+        url = site_url_for(config, docname)
+        summary = _summary(env, docname)
+        if summary:
+            lines.append("- [{}]({}): {}".format(title, url, summary))
+        else:
+            lines.append("- [{}]({})".format(title, url))
+    return lines
+
+
+def _summary(env, docname: str) -> str:
+    """
+    Use the article's meta description (the SEO summary every article has)
+    as the link notes in llms.txt; fall back to child titles for category
+    pages that end up in the list.
+    """
+    metadata = env.metadata.get(docname, {})
+    description = metadata.get("description")
+    if description:
+        return _clean_summary(description)
+
+    toc = env.tocs.get(docname)
+    if toc is not None:
+        children = []
+        for node in toc.findall(addnodes.toctree):
+            for _, entry in node.attributes["entries"]:
+                if entry != "self":
+                    children.append(title_of(env, entry))
+        if children:
+            preview = ", ".join(children[:5])
+            if len(children) > 5:
+                return "Articles: {}, and more.".format(preview)
+            return "Articles: {}.".format(preview)
+    return ""
+
+
+def _clean_summary(description: str) -> str:
+    description = re.sub(r"\s+", " ", description).strip()
+    description = re.sub(r"\s*\|\s*Hypernode\s*$", "", description)
+    return description
+
+
+def _optional(env, excluded) -> List[str]:
+    """
+    Pages not reachable from the main navigation end up in Optional.
+    Orphaned toctree stubs (nearly empty) and flattened category pages
+    are skipped.
+    """
+    optional: List[str] = []
+    for docname in sorted(env.found_docs):
+        if docname in ("index", "404") or docname in excluded:
+            continue
+        metadata = env.metadata.get(docname, {})
+        if metadata.get("wordcount", {}).get("words", 0) < 5:
+            continue
+        optional.append(docname)
+    return optional
+
+
+def setup(app):
+    app.connect("build-finished", write_llms_txt)
+
+    return {
+        "version": "0.1",
+        "parallel_write_safe": True,
+    }

hypernode/sphinx/extensions/markdown_content.py

@@ -0,0 +1,110 @@
+import re
+from posixpath import normpath
+from typing import List, Tuple
+
+FRONTMATTER_RE = re.compile(r"\A---\s*\n.*?\n---\s*\n", re.DOTALL)
+SOURCE_COMMENT_RE = re.compile(r"^<!-- source: .+ -->\s*$", re.MULTILINE)
+
+# Any MyST directive or fenced code block, matched fence-to-fence. The
+# info string (rest of the opening fence line) tells them apart: a
+# directive starts with "{name}". Backreference (?P=fence) makes nested
+# shorter fences inside the body harmless.
+FENCE_RE = re.compile(
+    r"^(?P<fence>`{3,}|~{3,})(?P<info>[^\n]*)\n(?P<body>.*?)^(?P=fence)[^\n]*$",
+    re.DOTALL | re.MULTILINE,
+)
+
+DIRECTIVE_LABELS = {
+    "note": "Note",
+    "tip": "Tip",
+    "warning": "Warning",
+    "important": "Important",
+    "caution": "Caution",
+}
+
+LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(([^)\s]+)\)")
+
+
+def convert(source, docname, images, titles, links):
+    """
+    Convert a MyST source file into LLM-friendly plain markdown:
+    strip frontmatter and source comments, turn MyST directives into
+    readable markdown, and rewrite internal links to absolute URLs.
+    Fenced code blocks are kept verbatim and shielded from rewriting.
+    """
+    source = FRONTMATTER_RE.sub("", source, count=1)
+    source = SOURCE_COMMENT_RE.sub("", source)
+
+    blocks: List[Tuple[str, str]] = []
+
+    def shield(text: str) -> str:
+        placeholder = "\x00PROTECTED{}\x00".format(len(blocks))
+        blocks.append((placeholder, text))
+        return placeholder
+
+    def handle_fence(match) -> str:
+        info = match.group("info").strip()
+        if not info.startswith("{"):
+            return shield(match.group(0))
+
+        name = info[1:].partition("}")[0]
+        if name == "toctree":
+            return ""
+
+        label = (
+            DIRECTIVE_LABELS.get(name) or info[1 + len(name) + 1 :].strip() or "Note"
+        )
+        body = FENCE_RE.sub(lambda m: shield(m.group(0)), match.group("body"))
+        quoted = "\n".join("> " + line for line in body.strip("\n").splitlines())
+        return "**{}:**\n\n{}".format(label, quoted)
+
+    source = FENCE_RE.sub(handle_fence, source)
+    source = _rewrite_links(source, docname, images, titles, links)
+
+    for placeholder, block in blocks:
+        source = source.replace(placeholder, block)
+
+    return source.strip() + "\n"
+
+
+def _rewrite_links(source, docname, images, titles, links):
+    def replace(match):
+        bang, text, target = match.group(1), match.group(2), match.group(3)
+
+        if target.startswith(("http://", "https://", "mailto:", "#")):
+            return match.group(0)
+
+        path, _, anchor = target.partition("#")
+        anchor = "#" + anchor if anchor else ""
+
+        if not bang:
+            stem = path[: -len(".md")] if path.endswith(".md") else path
+            for candidate in (stem, stem + ".md"):
+                docname_candidate = _resolve_target(docname, candidate)
+                if docname_candidate in links:
+                    return "[{}]({}{})".format(text, links[docname_candidate], anchor)
+            if stem in ("index", "index/index"):
+                return "[{}]({})".format(text, links.get("index", path))
+
+        resolved = _resolve_target(docname, path)
+        image = images.get(resolved)
+        if image is not None:
+            return "![{}]({})".format(text, image)
+
+        return match.group(0)
+
+    return LINK_RE.sub(replace, source)
+
+
+def _resolve_target(docname, target):
+    """
+    Resolve a relative target like ../php/foo.md against the directory of
+    the page containing the link. Returns the normalized path without
+    leading slash, or the input unchanged when it cannot be resolved.
+    """
+    if target.startswith("/"):
+        return normpath(target[1:])
+    base = docname.rpartition("/")[0]
+    if not base or base == docname:
+        return normpath(target)
+    return normpath("{}/{}".format(base, target))

hypernode/sphinx/extensions/markdown_pages.py

@@ -0,0 +1,48 @@
+from posixpath import dirname
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+MARKDOWN_SUFFIX = ".html.md"
+
+# The homepage's markdown counterpart is llms.txt itself, and the 404 page
+# is generated by the notfound extension; neither gets a .html.md file.
+NO_MARKDOWN_PAGES = {"index", "404"}
+
+
+def get_markdown_page_url(pagename: str) -> str:
+    """
+    The markdown version of a page is served next to its html file:
+    docs/foo.md -> foo.html.md. So from the rendered page, the markdown
+    version is always a sibling file with the same basename.
+    """
+    return pagename.rpartition("/")[2] + MARKDOWN_SUFFIX
+
+
+def get_llms_txt_url(pagename: str) -> str:
+    """
+    Path from the rendered page to the llms.txt file at the site root.
+    """
+    base = dirname(pagename)
+    if not base:
+        return "llms.txt"
+    depth = len(base.split("/"))
+    return "../" * depth + "llms.txt"
+
+
+def page_context_handler(app: "Sphinx", pagename: str, templatename, context, doctree):
+    context["markdown_page_url"] = (
+        get_markdown_page_url(pagename) if pagename not in NO_MARKDOWN_PAGES else None
+    )
+    context["llms_txt_url"] = get_llms_txt_url(pagename)
+
+
+def setup(app: "Sphinx"):
+    app.connect("html-page-context", page_context_handler)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }