diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5ad39b5..8cd37ed 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,7 +1,8 @@ name: Docs -# The Astro + Starlight site in docs/: hand-written pages, the executed tutorials and the API -# reference. Built on every push and pull request; deployed to GitHub Pages from devel. +# The Astro + Starlight site in docs/: hand-written pages, the executed notebooks and the API +# reference. Built on every push and pull request with the shared workflow of +# max-models/starlight-theme; deployed to GitHub Pages from devel. on: push: @@ -20,65 +21,17 @@ concurrency: cancel-in-progress: true jobs: - build: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@v7 - - - uses: astral-sh/setup-uv@v7 - with: - python-version: "3.12" - enable-cache: true - - - uses: actions/setup-node@v7 - with: - node-version: 22 - cache: npm - cache-dependency-path: docs/package-lock.json - - - name: Install the package with the docs extra - run: | - uv sync --extra docs - uv run python -m ipykernel install --user --name python3 - - - name: Execute and convert the tutorials - run: | - uv run python docs/tools/notebooks.py execute - uv run python docs/tools/notebooks.py convert --strict - uv run python docs/tools/examples.py - - - name: Build the site - working-directory: docs - run: | - npm ci - npm run check:mdx - DOCS_PYTHON="$(pwd)/../.venv/bin/python" npm run build - - - name: Smoke test the built site - working-directory: docs - run: | - for page in /index.html /getting-started/installation/index.html /tutorials/index.html /tutorials/example-tutorial/index.html /full-examples/index.html /full-examples/sine-wave/index.html /api/app/index.html /api/app/main/index.html; do - test -f "dist$page" || { echo "missing dist$page"; exit 1; } - done - test -f dist/tutorials/example-tutorial/example_tutorial.ipynb - test -f dist/full-examples/sine-wave/plot-1.png - grep -q 'href="/template-python/' dist/index.html || { echo "links do not use the base path"; exit 1; } - echo "all pages built with the /template-python base path" - - - uses: actions/upload-pages-artifact@v5 - with: - path: docs/dist - - deploy: - needs: build - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/devel' - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - id: deployment - uses: actions/deploy-pages@v5 + docs: + uses: max-models/starlight-theme/.github/workflows/docs.yml@v0.1.0 + with: + pre-build: uv run python docs/tools/examples.py + smoke-paths: | + getting-started/installation/index.html + tutorials/index.html + tutorials/example-tutorial/index.html + full-examples/index.html + full-examples/sine-wave/index.html + api/app/index.html + api/app/main/index.html + tutorials/example-tutorial/example_tutorial.ipynb + full-examples/sine-wave/plot-1.png diff --git a/.github/workflows/test_tutorials.yml b/.github/workflows/test_tutorials.yml index 9389789..a6dbc68 100644 --- a/.github/workflows/test_tutorials.yml +++ b/.github/workflows/test_tutorials.yml @@ -28,10 +28,19 @@ jobs: python-version: ${{ matrix.python-version }} enable-cache: true + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version: 22 + cache: npm + cache-dependency-path: docs/package-lock.json + - name: Install project run: | uv sync --extra docs uv run python -m ipykernel install --user --name python3 + cd docs && npm ci - name: Execute tutorials - run: uv run python docs/tools/notebooks.py execute --force + working-directory: docs + run: DOCS_PYTHON="$(pwd)/../.venv/bin/python" npx starlight-notebooks execute --force diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba45282..2ff09ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,9 +43,11 @@ pages: script: - apt-get update && apt-get install -y python3 python3-pip python3-venv - pip install uv && uv sync --extra docs - - uv run python docs/tools/notebooks.py execute - - uv run python docs/tools/notebooks.py convert - - cd docs && npm ci && DOCS_PYTHON="$(pwd)/../.venv/bin/python" npm run build + - export DOCS_PYTHON="$CI_PROJECT_DIR/.venv/bin/python" + - cd docs && npm ci + - npx starlight-notebooks execute + - npx starlight-notebooks convert + - npx astro build - mv dist ../public artifacts: paths: diff --git a/Makefile b/Makefile index 17cf667..ece9999 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Development environment (uv), README (Quarto) and the documentation site (docs/, Astro + Starlight). -# Notebook tool: docs/tools/notebooks.py. +# Notebook tool: starlight-notebooks, from @max-models/starlight-theme (docs/node_modules). PYTHON ?= .venv/bin/python -NOTEBOOKS = $(PYTHON) docs/tools/notebooks.py +NOTEBOOKS = cd docs && $(DOCS_ENV) npx starlight-notebooks DOCS_ENV = DOCS_PYTHON=$(shell $(PYTHON) -c "import sys; print(sys.executable)") FORCE ?= diff --git a/docs/.gitignore b/docs/.gitignore index 26a959f..4e58606 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -2,7 +2,7 @@ dist/ .astro/ node_modules/ -# generated by docs/tools/notebooks.py (regenerated in CI) +# generated by starlight-notebooks (regenerated in CI) .cache/ public/tutorials/ src/content/docs/tutorials/*.mdx diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 9f1f357..2a189da 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -1,94 +1,42 @@ // @ts-check -import { defineConfig } from 'astro/config'; -import { unified } from '@astrojs/markdown-remark'; -import starlight from '@astrojs/starlight'; -import starlightLinksValidator from 'starlight-links-validator'; -import starlightPydocs, { pydocsSidebarGroup } from 'starlight-pydocs'; -import rehypeKatex from 'rehype-katex'; -import remarkMath from 'remark-math'; +// The shared max-models docs setup (@max-models/starlight-theme): served at +// https://max-models.github.io/template-python/, with math, the API reference and a link check. +import { defineDocs } from '@max-models/starlight-theme'; -// The site is served at https://max-models.github.io/template-python/ by GitHub Pages. -const base = '/template-python'; - -// The Python package documented under "API reference" (the directory name in src/). -const pythonPackage = 'app'; - -export default defineConfig({ - site: 'https://max-models.github.io', - base, - markdown: { - // $...$ and $$...$$ math in Markdown and MDX pages, rendered with KaTeX at build time - processor: unified({ remarkPlugins: [remarkMath], rehypePlugins: [rehypeKatex] }), - }, - integrations: [ - starlight({ - title: 'template-python', - description: 'Template repository for Python projects.', - customCss: ['katex/dist/katex.min.css', './src/styles/custom.css'], - components: { - Header: './src/components/Header.astro', - }, - social: [ - { icon: 'github', label: 'GitHub', href: 'https://github.com/max-models/template-python' }, - ], - editLink: { baseUrl: 'https://github.com/max-models/template-python/edit/devel/docs/' }, - lastUpdated: true, - plugins: [ - starlightPydocs({ - packages: [ - { - name: pythonPackage, - label: 'All modules', - search: ['../src'], - docstringStyle: 'google', - members: { - exclude: [`${pythonPackage}.tests`, `${pythonPackage}.tests.*`], - }, - sourceLink: { - host: 'github', - repo: 'max-models/template-python', - ref: 'devel', - root: '..', - }, - sidebar: { collapsed: false }, - }, - ], - // The interpreter griffe runs in (set by `make docs-*`); falls back to uvx / python. - runner: process.env.DOCS_PYTHON ? { python: process.env.DOCS_PYTHON } : undefined, - inventories: ['python', { url: 'https://numpy.org/doc/stable/objects.inv' }], - }), - ...(process.env.DOCS_VALIDATE_LINKS === 'false' - ? [] - : [starlightLinksValidator({ errorOnLocalLinks: false, exclude: [`${base}/api/**`, '/api/**'] })]), +export default defineDocs({ + repo: 'template-python', + title: 'template-python', + description: 'Template repository for Python projects.', + pythonPackage: 'app', + nav: [ + { label: 'Getting started', link: '/getting-started/installation/' }, + { label: 'Tutorials', link: '/tutorials/' }, + { label: 'Full examples', link: '/full-examples/' }, + { label: 'Development', link: '/development/docs/' }, + { label: 'API', link: '/api/app/' }, + ], + sidebar: [ + { + label: 'Getting started', + items: [ + { label: 'Installation', slug: 'getting-started/installation' }, + { label: 'Quickstart', slug: 'getting-started/quickstart' }, ], - sidebar: [ - { - label: 'Getting started', - items: [ - { label: 'Installation', slug: 'getting-started/installation' }, - { label: 'Quickstart', slug: 'getting-started/quickstart' }, - ], - }, - { - label: 'Tutorials', - items: [{ autogenerate: { directory: 'tutorials' } }], - }, - { - label: 'Full examples', - items: [{ autogenerate: { directory: 'full-examples' } }], - }, - { - label: 'Development', - items: [ - { label: 'Building the docs', slug: 'development/docs' }, - { label: 'Publishing', slug: 'development/publishing' }, - ], - }, - { - label: 'API reference', - items: [pydocsSidebarGroup], - }, + }, + { + label: 'Tutorials', + items: [{ autogenerate: { directory: 'tutorials' } }], + }, + { + label: 'Full examples', + items: [{ autogenerate: { directory: 'full-examples' } }], + }, + { + label: 'Development', + items: [ + { label: 'Building the docs', slug: 'development/docs' }, + { label: 'Publishing', slug: 'development/publishing' }, ], - }), + }, ], }); diff --git a/docs/package-lock.json b/docs/package-lock.json index 3750ffc..7820246 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -8,18 +8,10 @@ "name": "template-python-docs", "version": "0.0.1", "dependencies": { - "@astrojs/markdown-remark": "^7.3.1", - "@astrojs/starlight": "^0.42.5", - "astro": "^7.3.5", - "katex": "^0.19.0", - "rehype-katex": "^7.0.1", - "remark-math": "^6.0.0", - "sharp": "^0.35.3", - "starlight-links-validator": "^0.26.0", - "starlight-pydocs": "0.2.1" - }, - "devDependencies": { - "@mdx-js/mdx": "^3.1.0" + "@astrojs/starlight": "^0.42.6", + "@max-models/starlight-theme": "github:max-models/starlight-theme#v0.1.0", + "astro": "^7.3.8", + "sharp": "^0.35.3" }, "engines": { "node": ">=22.12.0" @@ -237,12 +229,12 @@ } }, "node_modules/@astrojs/markdown-remark": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/@astrojs/markdown-remark/-/markdown-remark-7.3.1.tgz", - "integrity": "sha512-QCjsjZQl0+Cgb91+T3Ip9hyUe1r5aLm4udi584rML9YO3syupbAYE+jWpAxYg+z4lFLNUugPrylFxZ+8qFTClg==", + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/@astrojs/markdown-remark/-/markdown-remark-7.3.2.tgz", + "integrity": "sha512-ubszw8SVEI4OHR5Y+0jTbkmzG8xatrW47iAkQNqH7wUM1bky7DOnr51FGgoUdXt/fmKpUl7sRRX9talmU64Y+g==", "license": "MIT", "dependencies": { - "@astrojs/internal-helpers": "0.11.0", + "@astrojs/internal-helpers": "0.12.0", "@astrojs/prism": "4.0.2", "@mdx-js/mdx": "^3.1.1", "acorn": "^8.16.0", @@ -266,6 +258,22 @@ "vfile": "^6.0.3" } }, + "node_modules/@astrojs/markdown-remark/node_modules/@astrojs/internal-helpers": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.12.0.tgz", + "integrity": "sha512-pKPuwCDtO0L7zgmt1JGd9X4Aes1+0SX3IEBSqBNPHsScYdPMlfmed3Y7DTbXyF7E+8ylnb563o/0OzGQPOFehw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.4", + "@types/mdast": "^4.0.4", + "picomatch": "^4.0.4", + "retext-smartypants": "^6.2.0", + "shiki": "^4.0.2", + "smol-toml": "^1.8.0", + "unified": "^11.0.5", + "yaml": "^2.9.1" + } + }, "node_modules/@astrojs/markdown-satteri": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/@astrojs/markdown-satteri/-/markdown-satteri-0.4.2.tgz", @@ -325,9 +333,9 @@ } }, "node_modules/@astrojs/starlight": { - "version": "0.42.5", - "resolved": "https://registry.npmjs.org/@astrojs/starlight/-/starlight-0.42.5.tgz", - "integrity": "sha512-S6hjUexk9rrKtjJF3WR8T9dM1woNR7X2XU8e3wj4OpRT4X15AXk/mzPjWe/Qd5+P8whHj2+ves+k06qirBzimg==", + "version": "0.42.6", + "resolved": "https://registry.npmjs.org/@astrojs/starlight/-/starlight-0.42.6.tgz", + "integrity": "sha512-Cfs7GdgzgNRCauDxQVsFUVL1thrGHK/rQdbv/SURv/xhGBXx+j5ypOHW9ENPraQxbIzUC8KAnJV7L9mvEVRuyQ==", "license": "MIT", "dependencies": { "@astrojs/markdown-satteri": "^0.4.0", @@ -675,6 +683,7 @@ "os": [ "aix" ], + "peer": true, "engines": { "node": ">=18" } @@ -691,6 +700,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -707,6 +717,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -723,6 +734,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=18" } @@ -739,6 +751,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=18" } @@ -755,6 +768,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=18" } @@ -771,6 +785,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -787,6 +802,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -803,6 +819,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -819,6 +836,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -835,6 +853,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -851,6 +870,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -867,6 +887,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -883,6 +904,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -899,6 +921,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -915,6 +938,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -931,6 +955,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=18" } @@ -947,6 +972,7 @@ "os": [ "netbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -963,6 +989,7 @@ "os": [ "netbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -979,6 +1006,7 @@ "os": [ "openbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -995,6 +1023,7 @@ "os": [ "openbsd" ], + "peer": true, "engines": { "node": ">=18" } @@ -1011,6 +1040,7 @@ "os": [ "openharmony" ], + "peer": true, "engines": { "node": ">=18" } @@ -1027,6 +1057,7 @@ "os": [ "sunos" ], + "peer": true, "engines": { "node": ">=18" } @@ -1043,6 +1074,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -1059,6 +1091,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -1075,6 +1108,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=18" } @@ -1630,6 +1664,34 @@ "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "license": "MIT" }, + "node_modules/@max-models/starlight-theme": { + "version": "0.1.0", + "resolved": "git+ssh://git@github.com/max-models/starlight-theme.git#20ca97ef936a213fa127766e54ec4a50392ec16b", + "license": "MIT", + "workspaces": [ + "example/docs" + ], + "dependencies": { + "@astrojs/markdown-remark": "^7.3.2", + "@mdx-js/mdx": "^3.1.1", + "katex": "^0.19.0", + "rehype-katex": "^7.0.1", + "remark-math": "^6.0.0", + "starlight-links-validator": "^0.27.0", + "starlight-pydocs": "0.3.0" + }, + "bin": { + "starlight-check-mdx": "bin/check-mdx.mjs", + "starlight-notebooks": "bin/notebooks.mjs" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "@astrojs/starlight": ">=0.42.6", + "astro": ">=7.3.8" + } + }, "node_modules/@mdx-js/mdx": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.1.tgz", @@ -2240,12 +2302,6 @@ "undici-types": ">=7.24.0 <7.24.7" } }, - "node_modules/@types/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ==", - "license": "MIT" - }, "node_modules/@types/sax": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", @@ -2289,9 +2345,9 @@ } }, "node_modules/am-i-vibing": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/am-i-vibing/-/am-i-vibing-0.4.0.tgz", - "integrity": "sha512-MxT4XZL7pzLHpuvhDKdMaQHMGGkJDLluKBLsbstn+8wv9sWcFT6h+0ve9qkml95amVTZtZV83gQe2hY+ojgHLg==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/am-i-vibing/-/am-i-vibing-0.5.0.tgz", + "integrity": "sha512-KLcsDhqf1XRrJmJr+9I/C0EX2K+sGrAGZFgkmd84c8TJO9zJBu+/kFInjazu1KfJWFhWDKdRWkpBzEs7s35DVA==", "license": "MIT", "dependencies": { "process-ancestry": "^0.1.0" @@ -2381,64 +2437,63 @@ } }, "node_modules/astro": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/astro/-/astro-7.3.5.tgz", - "integrity": "sha512-1p/ernaq/RuESbM5NMjWzbO6SiebMGrGcYlGjMfBhzzgM+n/X7AUmVyw8BqW2QYCdivpsHwJBcD1fqmlTZjZHg==", + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/astro/-/astro-7.3.8.tgz", + "integrity": "sha512-LHzDHSaV+YzZiR3EFndDYTLLeNeuMtP1qX1FLcCNUUHOCqTTIWPczy2UV3zpA5VqvJe5cWZolqM9Z8JFqR2i9w==", "license": "MIT", "dependencies": { - "@astrojs/compiler-rs": "^0.5.0", - "@astrojs/internal-helpers": "0.11.0", - "@astrojs/markdown-satteri": "0.4.2", + "@astrojs/compiler-rs": "^0.5.1", + "@astrojs/internal-helpers": "0.12.0", + "@astrojs/markdown-satteri": "0.4.3", "@astrojs/telemetry": "3.3.3", - "@capsizecss/unpack": "^4.0.0", - "@clack/prompts": "^1.1.0", + "@capsizecss/unpack": "^4.0.1", + "@clack/prompts": "^1.8.1", "@oslojs/encoding": "^1.1.0", - "am-i-vibing": "^0.4.0", + "am-i-vibing": "^0.5.0", "aria-query": "^5.3.2", "axobject-query": "^4.1.0", "ci-info": "^4.4.0", "clsx": "^2.1.1", "common-ancestor-path": "^2.0.0", "cookie": "^2.0.1", - "devalue": "^5.8.1", + "devalue": "^5.9.4", "diff": "^9.0.0", "dset": "^3.1.4", - "es-module-lexer": "^2.0.0", - "esbuild": "^0.28.0", + "es-module-lexer": "^3.0.2", "find-proc": "0.2.0", "flattie": "^1.1.1", "fontace": "~0.4.1", - "get-tsconfig": "5.0.0-beta.4", + "get-tsconfig": "5.0.0-beta.6", "github-slugger": "^2.0.0", "html-escaper": "3.0.3", "http-cache-semantics": "^4.2.0", - "js-yaml": "^4.3.2", "jsonc-parser": "^3.3.1", - "magic-string": "^1.0.0", - "magicast": "^0.5.2", + "magic-string": "^1.4.2", + "magicast": "^0.5.5", "mrmime": "^2.0.1", "neotraverse": "^1.0.1", "obug": "^3.0.0", - "p-limit": "^7.3.0", - "p-queue": "^9.1.0", - "package-manager-detector": "^1.6.0", + "p-limit": "^7.3.3", + "p-queue": "^9.3.3", + "package-manager-detector": "^1.8.0", "piccolore": "^0.1.3", - "picomatch": "^4.0.4", - "shiki": "^4.0.2", - "smol-toml": "^1.8.0", + "picomatch": "^4.0.7", + "rolldown": "^1.2.11", + "shiki": "^4.4.3", + "smol-toml": "^1.9.0", "svgo": "^4.1.0", - "tinyclip": "^1.0.1", - "tinyexec": "^1.0.4", - "tinyglobby": "^0.2.15", - "ultrahtml": "^1.6.0", + "tinyclip": "^1.0.3", + "tinyexec": "^1.3.1", + "tinyglobby": "^0.2.17", + "ultrahtml": "^1.7.0", "unifont": "~0.7.5", "unstorage": "^1.17.5", - "verkit": "^0.4.0", - "vite": "^8.0.13", - "vitefu": "^1.1.2", + "verkit": "^0.5.0", + "vite": "^8.3.1", + "vitefu": "^1.1.3", "xxhash-wasm": "^1.1.0", "yargs-parser": "^22.0.0", - "zod": "^4.5.4" + "zod": "^4.6.5" }, "bin": { "astro": "bin/astro.mjs" @@ -2477,6 +2532,40 @@ "astro": "^4.0.0-beta || ^5.0.0-beta || ^3.3.0 || ^6.0.0-beta || ^7.0.0" } }, + "node_modules/astro/node_modules/@astrojs/internal-helpers": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.12.0.tgz", + "integrity": "sha512-pKPuwCDtO0L7zgmt1JGd9X4Aes1+0SX3IEBSqBNPHsScYdPMlfmed3Y7DTbXyF7E+8ylnb563o/0OzGQPOFehw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.4", + "@types/mdast": "^4.0.4", + "picomatch": "^4.0.4", + "retext-smartypants": "^6.2.0", + "shiki": "^4.0.2", + "smol-toml": "^1.8.0", + "unified": "^11.0.5", + "yaml": "^2.9.1" + } + }, + "node_modules/astro/node_modules/@astrojs/markdown-satteri": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@astrojs/markdown-satteri/-/markdown-satteri-0.4.3.tgz", + "integrity": "sha512-PFbEHPnwUlLvpXd7RXrrKM+j12akF4bG+DSuNuoN+3eiwjOwp4HaPi+QQdbkBV7QEYXmzyKca50OJsmGxhNIUA==", + "license": "MIT", + "dependencies": { + "@astrojs/internal-helpers": "0.12.0", + "@astrojs/prism": "4.0.2", + "github-slugger": "^2.0.0", + "satteri": "^0.10.3" + } + }, + "node_modules/astro/node_modules/es-module-lexer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-3.0.3.tgz", + "integrity": "sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==", + "license": "MIT" + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -3029,6 +3118,8 @@ "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -3286,9 +3377,9 @@ } }, "node_modules/get-tsconfig": { - "version": "5.0.0-beta.4", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.4.tgz", - "integrity": "sha512-7nF7C9fIPFEMHgEMEfgIlO9wDdZ8CyHw27rWciFZfHvHDReIiPhsYuzPRXsfvBCqFy1l8RRyyWV7QLM+ZhUJsQ==", + "version": "5.0.0-beta.6", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.6.tgz", + "integrity": "sha512-X6fBC0pmImC70gvX2zm56go9hx0MyoGVdG0tUCkg/D+Xnh5TJsOZ7iDbOdI3PvmtrDxnu1YdDufpK2QJX1Meqw==", "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -6395,13 +6486,12 @@ } }, "node_modules/starlight-links-validator": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/starlight-links-validator/-/starlight-links-validator-0.26.0.tgz", - "integrity": "sha512-b3JlzP5bczSDMQp10ST/Gldd6RNzcwpWfTrszzGPcqnuav2W31+MjyX92IXH+VCcG2pG9G1qyDcEcNuMRYi2ig==", + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/starlight-links-validator/-/starlight-links-validator-0.27.0.tgz", + "integrity": "sha512-UvPl8L9hfWR+yWbLOEtljLmpJCGcM6AVseVy5IO9dmqdOjW4R+Y8FaYyVA5axCC8RvxM5xU7M1WTerpwU8rYbw==", "license": "MIT", "dependencies": { "@astrojs/markdown-satteri": "^0.4.0", - "@types/picomatch": "^4.0.2", "github-slugger": "^2.0.0", "hast-util-from-html": "^2.0.3", "is-absolute-url": "^5.0.0", @@ -6413,19 +6503,19 @@ "unist-util-visit": "^5.1.0", "yaml": "^2.8.3" }, - "engines": { - "node": ">=22.12.0" - }, "peerDependencies": { "@astrojs/starlight": ">=0.42.0", "astro": ">=7.2.10" } }, "node_modules/starlight-pydocs": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/starlight-pydocs/-/starlight-pydocs-0.2.1.tgz", - "integrity": "sha512-JU//aeAmEEBdXwgminAlqlCaMUUVA9OIqHGgmNehZh/VDg0ZknlFmVIq86CucnKKV5gtPps5qHwkSfatEy4n3g==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/starlight-pydocs/-/starlight-pydocs-0.3.0.tgz", + "integrity": "sha512-2RVc8fhDvc9ECBt91W3G1OLXUqj0wNF4FCdk7qGj0v0zP6ngl+0jeH9C0cNbFDG/x1/E6r+VZyrAYciCDB8pcA==", "license": "MIT", + "dependencies": { + "shiki": "^4.0.2" + }, "engines": { "node": ">=22.12.0" }, @@ -6934,9 +7024,9 @@ "license": "MIT" }, "node_modules/verkit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/verkit/-/verkit-0.4.1.tgz", - "integrity": "sha512-mMVzj0TXExtVdlDEq+Mzp0eyOuyznNpFobNM3uAqe3HVm/Ps2c+u17BligMkZjZCA6EiXpoon8iRTer3iu40SQ==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/verkit/-/verkit-0.5.0.tgz", + "integrity": "sha512-u4J1hr6Y+C/hit34GwL3eIcUf3wngT8TlYUjK7tk0jxydKLAq3sYvifjdTDA6U3KCKRcQCQDGf/srjxfZF3l7g==", "license": "MIT", "engines": { "node": ">=18.12.0" diff --git a/docs/package.json b/docs/package.json index c4be530..4cee1d0 100644 --- a/docs/package.json +++ b/docs/package.json @@ -11,21 +11,14 @@ "start": "astro dev", "build": "astro build", "preview": "astro preview", - "check:mdx": "node scripts/check-mdx.mjs", + "check:mdx": "starlight-check-mdx", + "notebooks": "starlight-notebooks", "astro": "astro" }, "dependencies": { - "@astrojs/markdown-remark": "^7.3.1", - "@astrojs/starlight": "^0.42.5", - "astro": "^7.3.5", - "katex": "^0.19.0", - "rehype-katex": "^7.0.1", - "remark-math": "^6.0.0", - "sharp": "^0.35.3", - "starlight-links-validator": "^0.26.0", - "starlight-pydocs": "0.2.1" - }, - "devDependencies": { - "@mdx-js/mdx": "^3.1.0" + "@astrojs/starlight": "^0.42.6", + "@max-models/starlight-theme": "github:max-models/starlight-theme#v0.1.0", + "astro": "^7.3.8", + "sharp": "^0.35.3" } } diff --git a/docs/scripts/check-mdx.mjs b/docs/scripts/check-mdx.mjs deleted file mode 100644 index 7a94582..0000000 --- a/docs/scripts/check-mdx.mjs +++ /dev/null @@ -1,37 +0,0 @@ -// Compile every MDX page under src/content/docs with the site's markdown plugins and report the -// files that do not parse, with the line and the message. Runs in seconds, so a broken notebook -// conversion is found before the (slow) site build. -import { readdirSync, readFileSync, statSync } from 'node:fs'; -import { join, relative } from 'node:path'; -import { compile } from '@mdx-js/mdx'; -import remarkMath from 'remark-math'; -import rehypeKatex from 'rehype-katex'; - -const root = new URL('..', import.meta.url).pathname; -const content = join(root, 'src', 'content', 'docs'); - -function* walk(directory) { - for (const name of readdirSync(directory)) { - const path = join(directory, name); - if (statSync(path).isDirectory()) yield* walk(path); - else if (name.endsWith('.mdx')) yield path; - } -} - -let failures = 0; -let count = 0; -for (const file of walk(content)) { - count += 1; - const source = readFileSync(file, 'utf8').replace(/^---\n[\s\S]*?\n---\n/, (front) => - '\n'.repeat(front.split('\n').length - 1) - ); - try { - await compile(source, { remarkPlugins: [remarkMath], rehypePlugins: [rehypeKatex] }); - } catch (error) { - failures += 1; - const where = error.line ? `:${error.line}:${error.column ?? 0}` : ''; - console.error(`${relative(root, file)}${where}: ${error.reason ?? error.message}`); - } -} -console.log(`[check-mdx] ${count - failures} of ${count} MDX files compile`); -if (failures) process.exit(1); diff --git a/docs/src/components/Header.astro b/docs/src/components/Header.astro deleted file mode 100644 index 6cf8e6e..0000000 --- a/docs/src/components/Header.astro +++ /dev/null @@ -1,109 +0,0 @@ ---- -// A top nav bar with the site title, a row of section links, then search, socials and theme. -// Starlight's default Header has no link row, so this replaces it. -import config from 'virtual:starlight/user-config'; -import LanguageSelect from 'virtual:starlight/components/LanguageSelect'; -import Search from 'virtual:starlight/components/Search'; -import SiteTitle from 'virtual:starlight/components/SiteTitle'; -import SocialIcons from 'virtual:starlight/components/SocialIcons'; -import ThemeSelect from 'virtual:starlight/components/ThemeSelect'; -import { withBase } from '../utils/base'; - -const shouldRenderSearch = - config.pagefind || config.components.Search !== '@astrojs/starlight/components/Search.astro'; - -const navLinks = [ - { label: 'Getting started', href: withBase('/getting-started/installation/'), section: withBase('/getting-started/') }, - { label: 'Tutorials', href: withBase('/tutorials/'), section: withBase('/tutorials/') }, - { label: 'Full examples', href: withBase('/full-examples/'), section: withBase('/full-examples/') }, - { label: 'Development', href: withBase('/development/docs/'), section: withBase('/development/') }, - { label: 'API', href: withBase('/api/app/'), section: withBase('/api/') }, -]; - -const { pathname } = Astro.url; -const isCurrent = (section: string) => pathname.startsWith(section); ---- - -
-
- -
- -
- {shouldRenderSearch && } -
-
- - - -
-
- - diff --git a/docs/src/components/NotebookHeader.astro b/docs/src/components/NotebookHeader.astro deleted file mode 100644 index 1206b9d..0000000 --- a/docs/src/components/NotebookHeader.astro +++ /dev/null @@ -1,78 +0,0 @@ ---- -// The strip at the top of a tutorial page: download the notebook and open it on GitHub -// (written by tools/notebooks.py). -import { withBase } from '../utils/base'; - -export interface Props { - /** Site-root path of the downloadable notebook under public/, e.g. "/tutorials/example-tutorial/example_tutorial.ipynb". */ - download: string; - /** Repository path of the notebook, e.g. "tutorials/example_tutorial.ipynb". */ - source: string; - /** "executed" (outputs from a run at build time) or "not run". */ - status?: 'executed' | 'not run'; -} -const { download, source, status = 'executed' } = Astro.props; -const filename = download.split('/').at(-1) ?? 'tutorial.ipynb'; -const github = `https://github.com/max-models/template-python/blob/devel/${source}`; -const notes = { - executed: 'Run when the site was built.', - 'not run': 'Not run for this build: the outputs are missing.', -}; ---- - -
- Download notebook - View on GitHub - {notes[status]} -
- - - - diff --git a/docs/src/components/NotebookImage.astro b/docs/src/components/NotebookImage.astro deleted file mode 100644 index dd8a448..0000000 --- a/docs/src/components/NotebookImage.astro +++ /dev/null @@ -1,23 +0,0 @@ ---- -// A static image output of a notebook cell, stored under public/ (written by tools/notebooks.py). -export interface Props { - /** Site-root path of the image, e.g. "/tutorials/example-tutorial/1.png". */ - src: string; - alt?: string; -} -const { src, alt = 'Notebook output' } = Astro.props; -const resolved = import.meta.env.BASE_URL.replace(/\/$/, '') + src; ---- - -{alt} - - diff --git a/docs/src/components/RawHtml.astro b/docs/src/components/RawHtml.astro deleted file mode 100644 index 6096ecc..0000000 --- a/docs/src/components/RawHtml.astro +++ /dev/null @@ -1,16 +0,0 @@ ---- -// Rich HTML output of a notebook cell (pandas tables, etc.), inserted as is. -export interface Props { - html: string; -} -const { html } = Astro.props; ---- - -
- - diff --git a/docs/src/content.config.ts b/docs/src/content.config.ts index d9ee8c9..08161a2 100644 --- a/docs/src/content.config.ts +++ b/docs/src/content.config.ts @@ -1,7 +1 @@ -import { defineCollection } from 'astro:content'; -import { docsLoader } from '@astrojs/starlight/loaders'; -import { docsSchema } from '@astrojs/starlight/schema'; - -export const collections = { - docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), -}; +export { collections } from '@max-models/starlight-theme/content'; diff --git a/docs/src/content/docs/development/docs.md b/docs/src/content/docs/development/docs.md index 5195f2a..a915c87 100644 --- a/docs/src/content/docs/development/docs.md +++ b/docs/src/content/docs/development/docs.md @@ -7,13 +7,19 @@ The site in `docs/` is built with [Astro](https://astro.build/) and [Starlight](https://starlight.astro.build/). It has three kinds of pages: - **Hand-written pages** in `docs/src/content/docs/`, as Markdown or MDX. -- **Tutorials**, converted from the Jupyter notebooks in `tutorials/` by `docs/tools/notebooks.py`. +- **Tutorials**, converted from the Jupyter notebooks in `tutorials/` by `starlight-notebooks`. Each notebook is executed and its outputs (text, images, tables) are written into an MDX page. - **Full examples**, built from Python scripts in `examples/` by `docs/tools/examples.py`. Each page shows the module docstring, the complete source file, and its Matplotlib plots. - **API reference**, generated from the docstrings by [starlight-pydocs](https://ewels.github.io/starlight-pydocs/), one page per module. +The configuration (`defineDocs` in `docs/astro.config.mjs`), the look, the header, the notebook +tool and the build workflow come from the shared +[starlight-theme](https://github.com/max-models/starlight-theme) of the max-models projects; +change them there, and update the version in `docs/package.json` and +`.github/workflows/docs.yml` to pick up a new release. + ## Requirements - Node 22 or newer. @@ -43,6 +49,7 @@ paragraph its description. Cell tags control the output: | ------------------ | ---------------------------------------- | | `remove-cell` | the cell is left out of the page | | `hide-input` | the code is hidden, the outputs are kept | +| `fold-input` | the code is in a closed "Show code" box | | `remove-output` | the outputs are left out | | `raises-exception` | an error output is shown, not a failure | diff --git a/docs/src/content/docs/tutorials/example-tutorial.mdx b/docs/src/content/docs/tutorials/example-tutorial.mdx index 5234fcf..96c4764 100644 --- a/docs/src/content/docs/tutorials/example-tutorial.mdx +++ b/docs/src/content/docs/tutorials/example-tutorial.mdx @@ -5,7 +5,7 @@ sidebar: order: 2 --- -import NotebookHeader from '../../../components/NotebookHeader.astro'; +import NotebookHeader from '@max-models/starlight-theme/components/NotebookHeader.astro'; diff --git a/docs/src/styles/custom.css b/docs/src/styles/custom.css deleted file mode 100644 index fe5b85d..0000000 --- a/docs/src/styles/custom.css +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Palette: a warm paper/ink swap with a teal accent (shared with the other max-models sites). - * Replace the colours below to brand your own project. - */ - -:root { - /* dark mode */ - --sl-color-accent-low: #0d3a38; - --sl-color-accent: #63eee6; - --sl-color-accent-high: #b6f5f1; - --sl-color-white: #eeeadd; - --sl-color-text: #eeeadd; - --sl-color-text-accent: #63eee6; - --sl-color-text-invert: #07131d; - --sl-color-bg: #07131d; - --sl-color-bg-nav: #07131d; - --sl-color-bg-sidebar: #07131d; - --sl-color-bg-inline-code: #0b1a25; - --sl-color-bg-accent: #63eee6; - --sl-color-hairline-light: rgba(238, 234, 221, 0.2); - --sl-color-hairline: rgba(238, 234, 221, 0.14); - --sl-color-hairline-shade: #07131d; - --sl-color-gray-2: #a9b7bb; - --sl-color-gray-3: #8a9a9e; - --sl-color-gray-4: #6b7c80; - - --sl-font: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; -} - -:root[data-theme='light'] { - --sl-color-accent-low: #d9efee; - --sl-color-accent: #087f7b; - --sl-color-accent-high: #05504d; - --sl-color-white: #10222d; - --sl-color-text: #10222d; - --sl-color-text-accent: #087f7b; - --sl-color-text-invert: #f5f2e8; - --sl-color-bg: #f5f2e8; - --sl-color-bg-nav: #f5f2e8; - --sl-color-bg-sidebar: #f5f2e8; - --sl-color-bg-inline-code: #e8e5dc; - --sl-color-bg-accent: #087f7b; - --sl-color-hairline-light: rgba(16, 34, 45, 0.2); - --sl-color-hairline: rgba(16, 34, 45, 0.12); - --sl-color-hairline-shade: #e8e5dc; - --sl-color-gray-2: #52646b; - --sl-color-gray-3: #6c7d83; - --sl-color-gray-4: #8b9aa0; -} - -/* API reference: a hidden Markdown heading above an block feeds "On this page". */ -.sl-markdown-content .sl-heading-wrapper:has(+ section.pyd-member) { - position: absolute; - width: 1px; - height: 1px; - overflow: hidden; - clip-path: inset(50%); - white-space: nowrap; -} - -/* rectangular, low-radius buttons */ -.sl-link-button { - border-radius: 0.25rem; -} - -/* display math scrolls instead of overflowing on narrow screens */ -.sl-markdown-content .katex-display { - overflow-x: auto; - overflow-y: hidden; -} - -/* - * Let the text column fill the space between the sidebar and the "On this page" column. Starlight - * caps it at --sl-content-width (45rem); at 100% the TOC keeps its width and the content takes the rest. - */ -@media (min-width: 72rem) { - :root[data-has-sidebar] { - --sl-content-width: 100%; - } -} diff --git a/docs/src/utils/base.ts b/docs/src/utils/base.ts deleted file mode 100644 index b50bc80..0000000 --- a/docs/src/utils/base.ts +++ /dev/null @@ -1,5 +0,0 @@ -/** Prefix a site-root-relative path ("/tutorials/foo/") with the site's base path. */ -export function withBase(path: string): string { - if (/^[a-z]+:\/\//.test(path)) return path; - return import.meta.env.BASE_URL.replace(/\/$/, '') + (path.startsWith('/') ? path : `/${path}`); -} diff --git a/docs/tools/examples.py b/docs/tools/examples.py index 91cc092..6c6dda4 100644 --- a/docs/tools/examples.py +++ b/docs/tools/examples.py @@ -8,11 +8,17 @@ import re import runpy import shutil +import sys from pathlib import Path -from notebooks import sanitize_markdown - DOCS = Path(__file__).resolve().parents[1] + +# The MDX escaping of the theme's notebook tool (installed with the npm packages) +sys.path.insert( + 0, str(DOCS / "node_modules" / "@max-models" / "starlight-theme" / "tools") +) +from notebooks import sanitize_markdown # noqa: E402 + ROOT = DOCS.parent SOURCES = ROOT / "examples" CONTENT = DOCS / "src" / "content" / "docs" / "full-examples" diff --git a/docs/tools/notebooks.py b/docs/tools/notebooks.py deleted file mode 100644 index 3030a71..0000000 --- a/docs/tools/notebooks.py +++ /dev/null @@ -1,424 +0,0 @@ -"""Execute the notebooks in ``tutorials/`` and convert them into Starlight MDX pages. - -``execute`` - Runs each notebook with nbclient into ``docs/.cache/notebooks/.ipynb``. A notebook is - rerun only when the hash of its cells changes (``--force`` reruns all of them). -``convert`` - Writes ``docs/src/content/docs/tutorials/.mdx`` from the executed copy, the images into - ``docs/public/tutorials//`` and a downloadable copy of the notebook next to them. - Notebooks that have not been executed are converted without outputs (``--strict`` fails on - them, and on notebooks whose run raised). -``clean`` - Removes the cache, the generated pages and the public assets. - -Usage: python docs/tools/notebooks.py [options] -""" - -from __future__ import annotations - -import argparse -import base64 -import hashlib -import json -import re -import shutil -import sys -from pathlib import Path -from typing import Any - -DOCS = Path(__file__).resolve().parents[1] -REPO = DOCS.parent -NOTEBOOKS = REPO / "tutorials" -CACHE = DOCS / ".cache" / "notebooks" -CONTENT = DOCS / "src" / "content" / "docs" / "tutorials" -PUBLIC = DOCS / "public" / "tutorials" -COMPONENTS = "../../../components" - - -def slug_of(path: Path) -> str: - return re.sub(r"[^a-z0-9]+", "-", path.stem.lower()).strip("-") - - -def notebooks() -> list[Path]: - return sorted( - NOTEBOOKS.glob("*.ipynb"), key=lambda path: (path.stem != "index", path.name) - ) - - -# Execution ------------------------------------------------------------------------------- # - - -def cell_hash(path: Path) -> str: - nb = json.loads(path.read_text()) - cells = [ - { - "type": c["cell_type"], - "source": c["source"], - "tags": c.get("metadata", {}).get("tags", []), - } - for c in nb["cells"] - ] - return hashlib.sha256(json.dumps(cells, sort_keys=True).encode()).hexdigest()[:16] - - -def execute_notebook(path: Path, target: Path, timeout: int) -> None: - import nbformat - from nbclient import NotebookClient - - nb = nbformat.read(path, as_version=4) - client = NotebookClient( - nb, - timeout=timeout, - kernel_name="python3", - resources={"metadata": {"path": str(path.parent)}}, - ) - error = None - try: - client.execute() - except Exception as exc: # the partial outputs are still written for the page - error = f"{type(exc).__name__}: {exc}" - target.parent.mkdir(parents=True, exist_ok=True) - nbformat.write(nb, target) - error_file = target.with_suffix(".error") - if error: - error_file.write_text(error) - else: - error_file.unlink(missing_ok=True) - - -def cmd_execute(args: argparse.Namespace) -> None: - failed = [] - for path in notebooks(): - slug = slug_of(path) - target = CACHE / f"{slug}.ipynb" - key_file = CACHE / f"{slug}.key" - key = cell_hash(path) - if ( - not args.force - and target.exists() - and key_file.exists() - and key_file.read_text() == key - ): - print(f"[notebooks] {slug}: up to date") - continue - print(f"[notebooks] {slug}: executing", flush=True) - execute_notebook(path, target, args.timeout) - key_file.write_text(key) - error_file = target.with_suffix(".error") - if error_file.exists(): - failed.append(slug) - print(f"[notebooks] {slug}: FAILED: {error_file.read_text()}") - if failed: - print(f"[notebooks] {len(failed)} failed: {', '.join(failed)}") - sys.exit(1) - - -# Conversion: markdown ---------------------------------------------------------------------- # - -# Regions whose contents MDX must not touch: code fences, inline code and math. -_PROTECTED = re.compile( - r"(?P^(?P`{3,}|~{3,})[^\n]*\n.*?^(?P=ticks)[ \t]*$)" - r"|(?P\$\$.+?\$\$)" - r"|(?P\\\[.+?\\\])" - r"|(?P(?P`+).+?(?P=bt))" - r"|(?P(?\\\(.+?\\\))", - re.DOTALL | re.MULTILINE, -) -_VOID_TAGS = re.compile( - r"<(br|hr|img|input|meta|link|source)\b([^>]*?)\s*/?>", re.IGNORECASE -) -_TAG = re.compile(r"]*)?/?>") -_ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") - - -def _escape_plain(text: str) -> str: - return ( - text.replace("{", "\\{") - .replace("}", "\\}") - .replace("<", "<") - .replace(">", ">") - ) - - -def _escape_text(text: str) -> str: - """Escape what MDX would parse in plain markdown text: braces, stray <, comments.""" - text = re.sub(r"", "", text, flags=re.DOTALL) - text = _VOID_TAGS.sub(lambda m: f"<{m.group(1)}{m.group(2)} />", text) - out = [] - position = 0 - for match in _TAG.finditer(text): - out.append(_escape_plain(text[position : match.start()])) - out.append( - re.sub(r'\sstyle="[^"]*"', "", match.group(0)) - ) # JSX wants style objects - position = match.end() - out.append(_escape_plain(text[position:])) - return "".join(out) - - -def sanitize_markdown(text: str) -> str: - """Make notebook markdown valid MDX, keeping code and math as they are.""" - out = [] - position = 0 - for match in _PROTECTED.finditer(text): - out.append(_escape_text(text[position : match.start()])) - chunk = match.group(0) - if match.group("bracket"): - out.append("\n$$\n" + chunk[2:-2].strip() + "\n$$\n") - elif match.group("paren"): - out.append("$" + chunk[2:-2].strip() + "$") - else: - out.append(chunk) - position = match.end() - out.append(_escape_text(text[position:])) - return "".join(out) - - -def _first_heading(markdown: str) -> tuple[str | None, str]: - """Split off the first ``# Title`` line; return (title, the rest).""" - lines = markdown.splitlines() - for i, line in enumerate(lines): - if line.strip().startswith("# "): - return line.strip()[2:].strip(), "\n".join(lines[:i] + lines[i + 1 :]) - if line.strip(): - break - return None, markdown - - -def _plain(text: str) -> str: - """Markdown to plain text for a title or description.""" - text = re.sub(r"\$\$.*?\$\$", "", text, flags=re.DOTALL) - text = re.sub(r"\$([^$]+)\$", r"\1", text) - text = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", text) - text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) - text = re.sub(r"[`*_#>{}]", "", text) - return re.sub(r"\s+", " ", text).strip() - - -def _description(markdown: str) -> str: - for paragraph in re.split(r"\n\s*\n", markdown): - stripped = paragraph.strip() - if not stripped or stripped.startswith(("#", "|", "-", "*", "```", "$$", "<")): - continue - plain = _plain(stripped) - if len(plain) > 20: - return plain[:200].rsplit(" ", 1)[0] + "…" if len(plain) > 200 else plain - return "" - - -def _yaml_string(value: str) -> str: - return json.dumps(value, ensure_ascii=False) - - -# Conversion: outputs ----------------------------------------------------------------------- # - - -class Page: - def __init__(self, slug: str) -> None: - self.slug = slug - self.public = PUBLIC / slug - self.components: set[str] = set() - self.errors: list[str] = [] - self.assets = 0 - - def asset(self, suffix: str) -> tuple[Path, str]: - self.assets += 1 - self.public.mkdir(parents=True, exist_ok=True) - name = f"{self.assets}{suffix}" - return self.public / name, f"/tutorials/{self.slug}/{name}" - - -def _fence(text: str, language: str = "text") -> str: - text = _ANSI.sub("", text).rstrip("\n") - if not text.strip(): - return "" - lines = text.splitlines() - if len(lines) > 200: - lines = ( - lines[:100] + [f"... ({len(lines) - 200} lines skipped) ..."] + lines[-100:] - ) - text = "\n".join(lines) - ticks = "```" - while ticks in text: - ticks += "`" - return f"{ticks}{language}\n{text}\n{ticks}" - - -def _text(value: Any) -> str: - return "".join(value) if isinstance(value, list) else str(value) - - -def merge_streams(outputs: list[dict]) -> list[dict]: - """Join consecutive stream outputs of the same stream (one print per line otherwise).""" - merged: list[dict] = [] - for output in outputs: - previous = merged[-1] if merged else None - if ( - output.get("output_type") == "stream" - and previous is not None - and previous.get("output_type") == "stream" - and previous.get("name") == output.get("name") - ): - previous["text"] = _text(previous["text"]) + _text(output.get("text", "")) - else: - merged.append(dict(output)) - return merged - - -def convert_output(output: dict, page: Page, tags: list[str], index: int) -> str: - kind = output.get("output_type") - if kind == "stream": - return _fence(_text(output.get("text", ""))) - if kind == "error": - if "raises-exception" not in tags: - page.errors.append( - f"cell {index}: {output.get('ename')}: {output.get('evalue')}" - ) - return _fence(_ANSI.sub("", "\n".join(output.get("traceback", [])))) - data = output.get("data", {}) - if "image/png" in data or "image/jpeg" in data: - mime = "image/png" if "image/png" in data else "image/jpeg" - path, url = page.asset(".png" if mime == "image/png" else ".jpg") - path.write_bytes(base64.b64decode(_text(data[mime]))) - page.components.add("NotebookImage") - return f'' - if "image/svg+xml" in data: - page.components.add("RawHtml") - return f"" - if "text/html" in data: - page.components.add("RawHtml") - return f"" - if "text/markdown" in data: - return sanitize_markdown(_text(data["text/markdown"])) - if "text/latex" in data: - return "\n$$\n" + _text(data["text/latex"]).strip().strip("$") + "\n$$\n" - if "text/plain" in data: - return _fence(_text(data["text/plain"])) - return "" - - -# Conversion: pages ------------------------------------------------------------------------- # - - -def convert_notebook(source: Path, executed: Path | None, order: int) -> Page: - import nbformat - - slug = slug_of(source) - nb = nbformat.read(executed or source, as_version=4) - page = Page(slug) - shutil.rmtree(page.public, ignore_errors=True) - title = None - description = "" - body = [] - for index, cell in enumerate(nb.cells): - tags = list(cell.metadata.get("tags", [])) - if "remove-cell" in tags: - continue - if cell.cell_type == "markdown": - text = cell.source - if title is None: - heading, text = _first_heading(text) - if heading: - title = _plain(heading) - if not description: - description = _description(text) - for name, bundle in cell.get("attachments", {}).items(): - for mime, payload in bundle.items(): - if mime.startswith("image/"): - path, url = page.asset(Path(name).suffix or ".png") - path.write_bytes(base64.b64decode(_text(payload))) - text = text.replace(f"attachment:{name}", url) - body.append(sanitize_markdown(text).strip()) - elif cell.cell_type == "code": - if cell.source.strip() and "hide-input" not in tags: - body.append(_fence(cell.source, "python")) - if "remove-output" in tags: - continue - for output in merge_streams(cell.get("outputs", [])): - converted = convert_output(output, page, tags, index) - if converted: - body.append(converted) - title = title or source.stem.replace("_", " ").capitalize() - imports = [f"import NotebookHeader from '{COMPONENTS}/NotebookHeader.astro';"] - imports += [ - f"import {name} from '{COMPONENTS}/{name}.astro';" - for name in sorted(page.components) - ] - page.public.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, page.public / source.name) - status = "executed" if executed else "not run" - frontmatter = [ - "---", - f"title: {_yaml_string(title)}", - f"description: {_yaml_string(description or title)}", - "sidebar:", - f" order: {order}", - "---", - ] - header = ( - f'' - ) - text = ( - "\n".join(frontmatter) + "\n\n" + "\n".join(imports) + "\n\n" + header + "\n\n" - ) - text += "\n\n".join(part for part in body if part) + "\n" - CONTENT.mkdir(parents=True, exist_ok=True) - (CONTENT / f"{slug}.mdx").write_text(text) - return page - - -def cmd_convert(args: argparse.Namespace) -> None: - for stale in CONTENT.glob("*.mdx"): - stale.unlink() - problems = [] - for order, source in enumerate(notebooks(), start=1): - slug = slug_of(source) - executed = CACHE / f"{slug}.ipynb" - error_file = executed.with_suffix(".error") - page = convert_notebook(source, executed if executed.exists() else None, order) - state = "executed" if executed.exists() else "not run" - if error_file.exists(): - problems.append(f"{slug}: run failed: {error_file.read_text().strip()}") - elif not executed.exists(): - problems.append(f"{slug}: not executed") - problems += [f"{slug}: {error}" for error in page.errors] - print( - f"[notebooks] {slug}: {state}, {page.assets} assets -> {CONTENT / slug}.mdx" - ) - for problem in problems: - print(f"[notebooks] warning: {problem}") - if args.strict and problems: - sys.exit(1) - - -def cmd_clean(args: argparse.Namespace) -> None: - shutil.rmtree(CACHE, ignore_errors=True) - shutil.rmtree(PUBLIC, ignore_errors=True) - for page in CONTENT.glob("*.mdx"): - page.unlink() - - -def main(argv: list[str] | None = None) -> None: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - sub = parser.add_subparsers(dest="command", required=True) - execute = sub.add_parser("execute", help="run the notebooks whose cells changed") - execute.add_argument("--force", action="store_true", help="rerun every notebook") - execute.add_argument("--timeout", type=int, default=600, help="seconds per cell") - execute.set_defaults(func=cmd_execute) - convert = sub.add_parser("convert", help="write the MDX pages and assets") - convert.add_argument( - "--strict", action="store_true", help="fail on missing or failed runs" - ) - convert.set_defaults(func=cmd_convert) - clean = sub.add_parser("clean", help="remove the cache and generated files") - clean.set_defaults(func=cmd_clean) - args = parser.parse_args(argv) - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/setup_project.sh b/setup_project.sh index 72aa431..b5541c4 100755 --- a/setup_project.sh +++ b/setup_project.sh @@ -60,7 +60,7 @@ replace "${PACKAGE}\\.main\\.main" "${APPNAME_UNDERSCORE}.main.main" replace "/api/${PACKAGE}/" "/api/${APPNAME_UNDERSCORE}/" replace "src/${PACKAGE}/" "src/${APPNAME_UNDERSCORE}/" replace "source = \[ \"${PACKAGE}\" \]" "source = [ \"${APPNAME_UNDERSCORE}\" ]" -replace "^const pythonPackage = '${PACKAGE}';" "const pythonPackage = '${APPNAME_UNDERSCORE}';" +replace "pythonPackage: '${PACKAGE}'," "pythonPackage: '${APPNAME_UNDERSCORE}'," # 4. Remove the workflow that tests this script rm -f ".github/workflows/test_setup_script.yml" @@ -72,5 +72,5 @@ echo " Moved src/${PACKAGE} → src/${APPNAME_UNDERSCORE}" echo " Updated the imports, entry point, docs and API reference" echo " Removed .github/workflows/test_setup_script.yml" echo -echo "Next: update the GitHub links in pyproject.toml, docs/astro.config.mjs and" -echo "docs/src/components/NotebookHeader.astro, then remove this script with: rm setup_project.sh" +echo "Next: update the GitHub links in pyproject.toml and docs/astro.config.mjs (repo:)," +echo "then remove this script with: rm setup_project.sh"