diff --git a/deploy.php b/deploy.php index 11deadd5..de6fa4a3 100644 --- a/deploy.php +++ b/deploy.php @@ -79,6 +79,8 @@ run('apt update && apt install brotli -y'); within('{{release_or_current_path}}/docs/_build/html', function () { run('find . -name "*.html" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); + run('find . -name "*.html.md" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); + run('find . -name "llms.txt" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); run('find . -name "*.css" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); run('find . -name "*.js" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); run('find . -name "*.svg" -type f -exec brotli -f -q {{brotli_compression_level}} {} \\;'); diff --git a/docs/_templates/breadcrumbs.html b/docs/_templates/breadcrumbs.html index 6a57cb5b..51018edb 100644 --- a/docs/_templates/breadcrumbs.html +++ b/docs/_templates/breadcrumbs.html @@ -30,6 +30,9 @@ {%- endblock %} {%- block breadcrumbs_aside %}
  • + {%- if markdown_page_url %} + Markdown + {%- endif %} {%- if hasdoc(pagename) and display_vcs_links %} {%- if display_github %} {%- if check_meta and 'github_url' in meta %} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index fa0396cf..b580cbca 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -65,6 +65,14 @@ {%- endif -%} + {#- LLM-FRIENDLY MARKDOWN VERSION #} + {%- if markdown_page_url %} + + {%- endif -%} + {%- if llms_txt_url %} + + {%- endif -%} + {#- JAVASCRIPTS #} {%- block scripts %} \s*$", re.MULTILINE) + +# Any MyST directive or fenced code block, matched fence-to-fence. The +# info string (rest of the opening fence line) tells them apart: a +# directive starts with "{name}". Backreference (?P=fence) makes nested +# shorter fences inside the body harmless. +FENCE_RE = re.compile( + r"^(?P`{3,}|~{3,})(?P[^\n]*)\n(?P.*?)^(?P=fence)[^\n]*$", + re.DOTALL | re.MULTILINE, +) + +DIRECTIVE_LABELS = { + "note": "Note", + "tip": "Tip", + "warning": "Warning", + "important": "Important", + "caution": "Caution", +} + +LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(([^)\s]+)\)") + + +def convert(source, docname, images, titles, links): + """ + Convert a MyST source file into LLM-friendly plain markdown: + strip frontmatter and source comments, turn MyST directives into + readable markdown, and rewrite internal links to absolute URLs. + Fenced code blocks are kept verbatim and shielded from rewriting. + """ + source = FRONTMATTER_RE.sub("", source, count=1) + source = SOURCE_COMMENT_RE.sub("", source) + + blocks: List[Tuple[str, str]] = [] + + def shield(text: str) -> str: + placeholder = "\x00PROTECTED{}\x00".format(len(blocks)) + blocks.append((placeholder, text)) + return placeholder + + def handle_fence(match) -> str: + info = match.group("info").strip() + if not info.startswith("{"): + return shield(match.group(0)) + + name = info[1:].partition("}")[0] + if name == "toctree": + return "" + + label = ( + DIRECTIVE_LABELS.get(name) or info[1 + len(name) + 1 :].strip() or "Note" + ) + body = FENCE_RE.sub(lambda m: shield(m.group(0)), match.group("body")) + quoted = "\n".join("> " + line for line in body.strip("\n").splitlines()) + return "**{}:**\n\n{}".format(label, quoted) + + source = FENCE_RE.sub(handle_fence, source) + source = _rewrite_links(source, docname, images, titles, links) + + for placeholder, block in blocks: + source = source.replace(placeholder, block) + + return source.strip() + "\n" + + +def _rewrite_links(source, docname, images, titles, links): + def replace(match): + bang, text, target = match.group(1), match.group(2), match.group(3) + + if target.startswith(("http://", "https://", "mailto:", "#")): + return match.group(0) + + path, _, anchor = target.partition("#") + anchor = "#" + anchor if anchor else "" + + if not bang: + stem = path[: -len(".md")] if path.endswith(".md") else path + for candidate in (stem, stem + ".md"): + docname_candidate = _resolve_target(docname, candidate) + if docname_candidate in links: + return "[{}]({}{})".format(text, links[docname_candidate], anchor) + if stem in ("index", "index/index"): + return "[{}]({})".format(text, links.get("index", path)) + + resolved = _resolve_target(docname, path) + image = images.get(resolved) + if image is not None: + return "![{}]({})".format(text, image) + + return match.group(0) + + return LINK_RE.sub(replace, source) + + +def _resolve_target(docname, target): + """ + Resolve a relative target like ../php/foo.md against the directory of + the page containing the link. Returns the normalized path without + leading slash, or the input unchanged when it cannot be resolved. + """ + if target.startswith("/"): + return normpath(target[1:]) + base = docname.rpartition("/")[0] + if not base or base == docname: + return normpath(target) + return normpath("{}/{}".format(base, target)) diff --git a/hypernode/sphinx/extensions/markdown_pages.py b/hypernode/sphinx/extensions/markdown_pages.py new file mode 100644 index 00000000..97f1fca8 --- /dev/null +++ b/hypernode/sphinx/extensions/markdown_pages.py @@ -0,0 +1,48 @@ +from posixpath import dirname +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sphinx.application import Sphinx + +MARKDOWN_SUFFIX = ".html.md" + +# The homepage's markdown counterpart is llms.txt itself, and the 404 page +# is generated by the notfound extension; neither gets a .html.md file. +NO_MARKDOWN_PAGES = {"index", "404"} + + +def get_markdown_page_url(pagename: str) -> str: + """ + The markdown version of a page is served next to its html file: + docs/foo.md -> foo.html.md. So from the rendered page, the markdown + version is always a sibling file with the same basename. + """ + return pagename.rpartition("/")[2] + MARKDOWN_SUFFIX + + +def get_llms_txt_url(pagename: str) -> str: + """ + Path from the rendered page to the llms.txt file at the site root. + """ + base = dirname(pagename) + if not base: + return "llms.txt" + depth = len(base.split("/")) + return "../" * depth + "llms.txt" + + +def page_context_handler(app: "Sphinx", pagename: str, templatename, context, doctree): + context["markdown_page_url"] = ( + get_markdown_page_url(pagename) if pagename not in NO_MARKDOWN_PAGES else None + ) + context["llms_txt_url"] = get_llms_txt_url(pagename) + + +def setup(app: "Sphinx"): + app.connect("html-page-context", page_context_handler) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/hypernode/sphinx/extensions/markdown_pages_writer.py b/hypernode/sphinx/extensions/markdown_pages_writer.py new file mode 100644 index 00000000..033e0598 --- /dev/null +++ b/hypernode/sphinx/extensions/markdown_pages_writer.py @@ -0,0 +1,63 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Dict + +from hypernode.sphinx.extensions.markdown_content import convert +from hypernode.sphinx.extensions.markdown_tree import ( + collect_tree, + site_url_for, + title_of, +) + +if TYPE_CHECKING: + from sphinx.application import Sphinx + +SKIP_DOCS = {"index", "404"} + + +def write_markdown_pages(app: "Sphinx", exception) -> None: + env = app.builder.env + config = app.config + + docnames = collect_tree(env, "index") + if config.markdown_include_orphans: + known = set(docnames) | set(SKIP_DOCS) + docnames.extend(doc for doc in sorted(env.found_docs) if doc not in known) + + titles: Dict[str, str] = {} + links: Dict[str, str] = {} + for docname in ["index"] + docnames: + titles[docname] = title_of(env, docname) + links[docname] = site_url_for(config, docname) + + # Source image paths (e.g. about-hypernode/billing/_res/cancel.png) map + # to their published URL (https://docs.hypernode.com/_images/cancel.png). + image_urls: Dict[str, str] = {} + for source_key, (_, published_name) in env.images.items(): + image_urls[source_key] = "{}/_images/{}".format( + (config.html_baseurl or "").rstrip("/"), published_name + ) + + for docname in docnames: + source_file = Path(app.srcdir) / (docname + ".md") + if not source_file.exists(): + continue + output = convert( + source_file.read_text(encoding="utf-8"), + docname, + image_urls, + titles, + links, + ) + output_path = Path(app.outdir) / (docname + ".html.md") + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(output, encoding="utf-8") + + +def setup(app: "Sphinx"): + app.add_config_value("markdown_include_orphans", True, "html") + app.connect("build-finished", write_markdown_pages) + + return { + "version": "0.1", + "parallel_write_safe": True, + } diff --git a/hypernode/sphinx/extensions/markdown_tree.py b/hypernode/sphinx/extensions/markdown_tree.py new file mode 100644 index 00000000..3e4adbfe --- /dev/null +++ b/hypernode/sphinx/extensions/markdown_tree.py @@ -0,0 +1,57 @@ +from typing import List + +import sphinx.addnodes as addnodes + + +def collect_tree(env, root_docname: str) -> List[str]: + """ + Collect all docnames reachable from root_docname's toctrees, in + navigation order. Does not include root_docname itself unless it is + also referenced from a child toctree (e.g. the site index). + """ + result: List[str] = [] + + def walk(docname: str) -> None: + toc = env.tocs.get(docname) + if toc is None: + return + for node in toc.findall(addnodes.toctree): + for _, entry in node.attributes["entries"]: + if entry in ("self", "*"): + continue + if entry in result: + continue + result.append(entry) + walk(entry) + + walk(root_docname) + return result + + +def title_of(env, docname: str) -> str: + title = env.titles.get(docname) + return title.astext().strip() if title else docname + + +def site_url_for(config, docname: str) -> str: + base_url = (config.html_baseurl or "").rstrip("/") + if docname == "index": + return base_url + "/" + if docname.endswith("/index"): + return base_url + "/" + docname[: -len("/index")] + "/" + return base_url + "/" + docname + ".html" + + +def image_url_for(images, target: str) -> str: + """ + Resolve an image reference (e.g. _res/foo.png) to its published URL + (e.g. https://docs.hypernode.com/_images/foo.png) using the Sphinx + environment's image collection, falling back to the bare filename. + """ + from posixpath import basename + + entry = images.get(target) + if entry: + # FilenameUniqDict maps source path -> (set of docnames, published name) + return basename(entry[1]) + return basename(target) diff --git a/tests/sphinx/extensions/test_markdown_content.py b/tests/sphinx/extensions/test_markdown_content.py new file mode 100644 index 00000000..ff51ebab --- /dev/null +++ b/tests/sphinx/extensions/test_markdown_content.py @@ -0,0 +1,160 @@ +from hypernode.sphinx.extensions.markdown_content import _resolve_target, convert +from tests.testcase import HypernodeTestCase + +IMAGES = { + "about-hypernode/billing/_res/cancel.png": "https://docs.hypernode.com/_images/cancel.png", +} + +TITLES = { + "hypernode-platform/ftp/how-to-configure-ftp-sftp-on-hypernode": "How to Configure FTP/SFTP on Hypernode", + "index": "Welcome to Hypernode Docs", +} + +LINKS = { + "hypernode-platform/ftp/how-to-configure-ftp-sftp-on-hypernode": "https://docs.hypernode.com/hypernode-platform/ftp/how-to-configure-ftp-sftp-on-hypernode.html", + "index": "https://docs.hypernode.com/", +} + + +class TestConvert(HypernodeTestCase): + def test_strips_frontmatter(self): + source = "---\ntitle: foo\n---\n\n# Hello\n\nBody text." + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertNotIn("title: foo", result) + self.assertIn("# Hello", result) + + def test_strips_source_comment(self): + source = "\n\n# Hello" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertNotIn("source:", result) + self.assertIn("# Hello", result) + + def test_drops_toctree_directive(self): + source = "# PHP\n\n```{toctree}\n---\ncaption: x\n---\nphp/*\n```\n\nBody" + + result = convert(source, "php", IMAGES, TITLES, LINKS) + + self.assertNotIn("toctree", result) + self.assertIn("# PHP", result) + self.assertIn("Body", result) + + def test_converts_note_directive_to_blockquote(self): + source = "Intro\n\n```{note}\nSome note text.\n```\n\nOutro" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn("**Note:**", result) + self.assertIn("> Some note text.", result) + self.assertNotIn("```{note}", result) + + def test_converts_tip_directive_with_custom_label(self): + source = "```{tip}\nHandy tip.\n```" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn("**Tip:**", result) + + def test_converts_unknown_directive_with_title_argument(self): + source = "```{admonition} Watch Out\nBody text.\n```" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn("**Watch Out:**", result) + self.assertIn("> Body text.", result) + + def test_converts_nested_directive_with_inner_code_fence(self): + source = "````{note}\nText with code:\n```yaml\nkey: value\n```\n````" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn("**Note:**", result) + self.assertIn("```yaml", result) + self.assertIn("key: value", result) + self.assertNotIn("````", result) + + def test_keeps_fenced_code_block_verbatim(self): + source = 'Before\n\n```bash\necho "[link](relative.md)"\n```\n\nAfter' + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn('echo "[link](relative.md)"', result) + self.assertIn("```bash", result) + + def test_rewrites_internal_md_link_to_absolute_url(self): + source = "See [the FTP article](../../hypernode-platform/ftp/how-to-configure-ftp-sftp-on-hypernode.md)." + + result = convert( + source, "about-hypernode/billing/how-to-cancel", IMAGES, TITLES, LINKS + ) + + self.assertIn( + "]({})".format( + "https://docs.hypernode.com/hypernode-platform/ftp/how-to-configure-ftp-sftp-on-hypernode.html" + ), + result, + ) + self.assertNotIn(".md)", result) + + def test_rewrites_index_link_to_site_root(self): + source = "Check [our docs](../../index.md)." + + result = convert( + source, "getting-started/how-to-order/some-article", IMAGES, TITLES, LINKS + ) + + self.assertIn("](https://docs.hypernode.com/)", result) + + def test_keeps_external_links(self): + source = "[Hypernode](https://www.hypernode.com/)" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertEqual("[Hypernode](https://www.hypernode.com/)", result.strip()) + + def test_rewrites_image_to_published_url(self): + source = "![Cancellation date](_res/cancel.png)" + + result = convert( + source, "about-hypernode/billing/how-to-cancel", IMAGES, TITLES, LINKS + ) + + self.assertEqual( + "![Cancellation date](https://docs.hypernode.com/_images/cancel.png)", + result.strip(), + ) + + def test_does_not_touch_code_blocks_containing_directive_like_text(self): + source = "Example:\n\n```text\n```{note}\n```\n\nDone" + + result = convert(source, "foo", IMAGES, TITLES, LINKS) + + self.assertIn("```{note}", result) + + +class TestResolveTarget(HypernodeTestCase): + def test_resolves_relative_path_against_docname_directory(self): + result = _resolve_target( + "about-hypernode/billing/how-to-cancel", + "../../hypernode-platform/ftp/article.md", + ) + + self.assertEqual("hypernode-platform/ftp/article.md", result) + + def test_resolves_sibling_path(self): + result = _resolve_target("hypernode-platform/php/article", "other-article.md") + + self.assertEqual("hypernode-platform/php/other-article.md", result) + + def test_resolves_absolute_path(self): + result = _resolve_target("any/page", "/hypernode-platform/ssh/article.md") + + self.assertEqual("hypernode-platform/ssh/article.md", result) + + def test_resolves_path_in_root(self): + result = _resolve_target("hypernode-platform/php/article", "index.md") + + self.assertEqual("hypernode-platform/php/index.md", result) diff --git a/tests/sphinx/extensions/test_markdown_pages.py b/tests/sphinx/extensions/test_markdown_pages.py new file mode 100644 index 00000000..126566e3 --- /dev/null +++ b/tests/sphinx/extensions/test_markdown_pages.py @@ -0,0 +1,67 @@ +from unittest.mock import Mock + +from hypernode.sphinx.extensions.markdown_pages import ( + get_llms_txt_url, + get_markdown_page_url, + page_context_handler, + setup, +) +from tests.testcase import HypernodeTestCase + + +class TestGetMarkdownPageUrl(HypernodeTestCase): + def test_returns_sibling_markdown_url_for_nested_page(self): + self.assertEqual( + "how-to-flush-the-php-opcache.html.md", + get_markdown_page_url( + "hypernode-platform/php/how-to-flush-the-php-opcache" + ), + ) + + def test_returns_sibling_markdown_url_for_top_level_page(self): + self.assertEqual("foo.html.md", get_markdown_page_url("foo")) + + +class TestGetLlmsTxtUrl(HypernodeTestCase): + def test_returns_llms_txt_for_root_page(self): + self.assertEqual("llms.txt", get_llms_txt_url("index")) + + def test_returns_llms_txt_for_top_level_page(self): + self.assertEqual("llms.txt", get_llms_txt_url("foo")) + + def test_returns_relative_path_to_root_for_nested_page(self): + self.assertEqual( + "../../llms.txt", + get_llms_txt_url("hypernode-platform/php/how-to-flush-the-php-opcache"), + ) + + +class TestPageContextHandler(HypernodeTestCase): + def setUp(self) -> None: + self.context: dict = {} + + def test_sets_markdown_and_llms_txt_urls(self): + page_context_handler(Mock(), "hypernode-platform/php", "", self.context, None) + + self.assertEqual("php.html.md", self.context["markdown_page_url"]) + self.assertEqual("../llms.txt", self.context["llms_txt_url"]) + + def test_sets_urls_for_root_page(self): + page_context_handler(Mock(), "index", "", self.context, None) + + self.assertIsNone(self.context["markdown_page_url"]) + self.assertEqual("llms.txt", self.context["llms_txt_url"]) + + def test_sets_no_markdown_url_for_404_page(self): + page_context_handler(Mock(), "404", "", self.context, None) + + self.assertIsNone(self.context["markdown_page_url"]) + + +class TestSetup(HypernodeTestCase): + def test_setup_connects_page_context_handler(self): + app = Mock() + + setup(app) + + app.connect.assert_called_once_with("html-page-context", page_context_handler)