From a569da180eb5784e3a949f48e265817288ada775 Mon Sep 17 00:00:00 2001 From: olaru Date: Tue, 15 Sep 2026 17:22:30 +0300 Subject: [PATCH] fix for tldr stripping --- README.md | 6 +- bin/create-uploads-dir | 198 ++++++++++++++++-- src/App/src/Service/ArticleBodyCleaner.php | 2 +- .../App/Service/ArticleBodyCleanerTest.php | 24 +++ 4 files changed, 213 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f233cf1f..ced835c8 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Copy an existing set of these three files in the same category as a starting poi If the article body uses images (via `asset('uploads/article/' ~ article.id ~ '/filename.png')` in the `.html.twig`), just drop the image file anywhere under `public/uploads` - `bin/create-uploads-dir` (step 4) finds it by filename and copies it to the right place. No manual path/folder creation needed. +Markdown articles reference images with a literal path instead, e.g. `![](/uploads/article/{post-id}/filename.png)` - `{post-id}` there is just a placeholder for whatever UUID the `Post` has when you write the file. `bin/create-uploads-dir` also scans `.md` files: it resolves the real `Post` by slug and, if the UUID hardcoded in the file doesn't match the post's actual current UUID (which happens whenever fixtures assign it a new one, e.g. in a fresh environment), rewrites the file to the real UUID and copies the image into the correct directory. + ## 3. At deploy - run in this order ```shell @@ -61,7 +63,7 @@ php bin/create-uploads-dir ``` - `bin/doctrine-fixtures` loads `articles_cleaned.json` into the database, creating the `Post` entity (with its database-generated UUID) for the new article. -- `bin/create-uploads-dir` must run *after* it - it resolves the post by slug to get that UUID, creates `public/uploads/article/{post-id}/`, and copies each image referenced in the `.html.twig` there from wherever it already lives under `public/uploads`. +- `bin/create-uploads-dir` must run *after* it - it resolves the post by slug to get that UUID, creates `public/uploads/article/{post-id}/`, and copies each image referenced in the `.html.twig` there from wherever it already lives under `public/uploads`. It does the same for `.md` files, additionally correcting the UUID hardcoded in the file if it no longer matches the post's real one. ## 4. Regenerate the public artifacts - any order @@ -91,7 +93,7 @@ Steps to edit an existing article (change its status, text, or both) and get the - `public/md-articles/{category-slug}/{article-slug}.md` - `src/Blog/templates/page/blog-resource/{category-slug}/{article-slug}.html.twig` - `src/Blog/templates/page/JSON-LD/{category-slug}/{article-slug}.jsonld.twig` (only if it has hardcoded text outside of `article.*`/`meta.*` variables — most of its fields pull straight from the database and update automatically) -3. **Re-run the same commands as step 3 and step 4 above** (`bin/doctrine-fixtures`, then `bin/generate-feed` / `bin/sitemap` / `bin/generate-llms-full`) so the database and the generated artifacts reflect the change. `bin/create-uploads-dir` only needs to run again if you added a new image. +3. **Re-run the same commands as step 3 and step 4 above** (`bin/doctrine-fixtures`, then `bin/generate-feed` / `bin/sitemap` / `bin/generate-llms-full`) so the database and the generated artifacts reflect the change. `bin/create-uploads-dir` only needs to run again if you added a new image - it's also safe (and cheap) to run any time you suspect a `.md` file's hardcoded UUID has drifted from the post's real one. ## How to move an article to a different category diff --git a/bin/create-uploads-dir b/bin/create-uploads-dir index d60d844c..660afd94 100755 --- a/bin/create-uploads-dir +++ b/bin/create-uploads-dir @@ -11,7 +11,8 @@ chdir(__DIR__ . '/../'); require 'vendor/autoload.php'; $templatesDir = 'src/Blog/templates/page/blog-resource'; -$uploadsDir = 'public/uploads'; +$publicDir = 'public'; +$uploadsDir = $publicDir . '/uploads'; $articleDir = $uploadsDir . '/article'; $limit = null; @@ -33,18 +34,20 @@ $entityManager = $container->get(EntityManager::class); $postRepository = $entityManager->getRepository(Post::class); /** - * Index every file currently under public/uploads (excluding the - * uploads/article destination tree) by basename, so we can locate the - * source file for each image referenced from a template. + * Index every file anywhere under public/ by basename, so we can locate the + * source file for each image referenced from a template or markdown + * article - including images already sitting in some other article's + * uploads/article/{uuid}/ directory (e.g. left over from before a post's + * UUID changed), not just loose files dropped outside uploads/article/. * * @return array */ -function indexUploadSources(string $uploadsDir, string $articleDir): array +function indexUploadSources(string $publicDir): array { $index = []; $iterator = new RecursiveIteratorIterator( - new RecursiveDirectoryIterator($uploadsDir, FilesystemIterator::SKIP_DOTS) + new RecursiveDirectoryIterator($publicDir, FilesystemIterator::SKIP_DOTS) ); foreach ($iterator as $file) { @@ -52,21 +55,49 @@ function indexUploadSources(string $uploadsDir, string $articleDir): array continue; } - $path = $file->getPathname(); - if (str_starts_with($path, $articleDir . '/')) { - continue; - } - $basename = $file->getFilename(); if (! isset($index[$basename])) { - $index[$basename] = $path; + $index[$basename] = $file->getPathname(); } } return $index; } -$sourceIndex = indexUploadSources($uploadsDir, $articleDir); +/** + * Extract every (uuid, filename) pair referenced via an + * "uploads/article/{uuid}/{filename}" path in Markdown article content - + * these are hardcoded literally in .md files, unlike the dynamic + * `~ article.id ~` expression used in .html.twig templates. + * + * @return list + */ +function extractArticleImageRefs(string $contents): array +{ + $pattern = '/uploads\/article\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/([^)\]\s"\']+)/i'; + + preg_match_all($pattern, $contents, $matches, PREG_SET_ORDER); + + $seen = []; + $refs = []; + + foreach ($matches as $match) { + $uuid = strtolower($match[1]); + $filename = $match[2]; + $key = $uuid . '/' . $filename; + + if (isset($seen[$key])) { + continue; + } + + $seen[$key] = true; + $refs[] = ['uuid' => $uuid, 'filename' => $filename]; + } + + return $refs; +} + +$sourceIndex = indexUploadSources($publicDir); // Matches the filename in: asset('uploads/article/' ~ article.id ~ '/filename.ext') $pattern = '/~\s*article\.id\s*~\s*\'\/([^\']+)\'/'; @@ -140,8 +171,137 @@ foreach ($templateIterator as $file) { } } +/** + * Markdown articles (public/md-articles/{category-slug}/{article-slug}.md) + * embed uploads/article/{uuid}/{filename} paths as literal text, unlike + * .html.twig templates whose `~ article.id ~` is resolved dynamically. Since + * a Post's UUID (uuid7) is generated fresh the first time its slug is + * fixture-inserted in a given environment, a UUID hardcoded in a .md file + * drifts from the Post's real UUID as soon as that slug is (re)seeded + * elsewhere. This pass detects that drift, rewrites the file to the real + * UUID, and makes sure the image is available under the corrected path. + */ +$mdArticlesDir = 'public/md-articles'; + +$mdFilesProcessed = 0; +$uuidsFixed = 0; +$mdDirsCreated = 0; +$mdImagesCopied = 0; +$mdImagesMissing = 0; + +if (is_dir($mdArticlesDir)) { + $mdIterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($mdArticlesDir, FilesystemIterator::SKIP_DOTS) + ); + + foreach ($mdIterator as $file) { + if (! $file->isFile() || $file->getExtension() !== 'md') { + continue; + } + + $path = $file->getPathname(); + + // Only handle files one level under a category directory, i.e. + // public/md-articles/{category-slug}/{article-slug}.md - skips + // stray top-level files such as md-articles/index.md. + $relative = ltrim(substr($path, strlen($mdArticlesDir)), '/'); + if (substr_count($relative, '/') !== 1) { + continue; + } + + $slug = preg_replace('/\.md$/', '', $file->getFilename()); + $contents = file_get_contents($path); + if ($contents === false) { + continue; + } + + $refs = extractArticleImageRefs($contents); + if ($refs === []) { + continue; + } + + $post = $postRepository->findOneBy(['slug' => $slug]); + if ($post === null) { + printf("No Post found for slug '%s' (%s), skipping%s", $slug, $path, PHP_EOL); + continue; + } + + $mdFilesProcessed++; + + $realUuid = $post->getId()->toString(); + $staleUuids = []; + + foreach ($refs as $ref) { + $refUuid = $ref['uuid']; + $filename = $ref['filename']; + + if ($refUuid !== $realUuid) { + $staleUuids[$refUuid] = true; + } + + $targetDir = $articleDir . '/' . $realUuid; + $targetPath = $targetDir . '/' . $filename; + + if (file_exists($targetPath)) { + continue; + } + + if (! is_dir($targetDir)) { + if (! mkdir($targetDir, 0775, true) && ! is_dir($targetDir)) { + fwrite(STDERR, sprintf("Failed to create directory '%s'%s", $targetDir, PHP_EOL)); + continue; + } + $mdDirsCreated++; + } + + $sourcePath = null; + if ($refUuid !== $realUuid) { + $stalePath = $articleDir . '/' . $refUuid . '/' . $filename; + if (file_exists($stalePath)) { + $sourcePath = $stalePath; + } + } + + if ($sourcePath === null && isset($sourceIndex[$filename])) { + $sourcePath = $sourceIndex[$filename]; + } + + if ($sourcePath === null) { + printf("Source image '%s' not found for %s%s", $filename, $path, PHP_EOL); + $mdImagesMissing++; + continue; + } + + if (! copy($sourcePath, $targetPath)) { + fwrite(STDERR, sprintf("Failed to copy '%s' to '%s'%s", $sourcePath, $targetPath, PHP_EOL)); + continue; + } + + $mdImagesCopied++; + } + + if ($staleUuids !== []) { + $updated = $contents; + foreach (array_keys($staleUuids) as $staleUuid) { + $updated = str_replace($staleUuid, $realUuid, $updated); + } + + if ($updated !== $contents) { + if (file_put_contents($path, $updated) === false) { + fwrite(STDERR, sprintf("Failed to update '%s'%s", $path, PHP_EOL)); + } else { + $uuidsFixed++; + printf("Corrected UUID in '%s'%s", $path, PHP_EOL); + } + } + } + } +} + printf( - "Done. %d template%s processed, %d director%s created, %d image%s copied, %d missing.%s", + "Done. %d twig template%s processed, %d director%s created, %d image%s copied, %d missing.%s" + . " %d markdown file%s processed, %d UUID%s corrected, %d director%s created," + . " %d image%s copied, %d missing.%s", $filesProcessed, $filesProcessed === 1 ? '' : 's', $dirsCreated, @@ -149,5 +309,15 @@ printf( $imagesCopied, $imagesCopied === 1 ? '' : 's', $imagesMissing, + PHP_EOL, + $mdFilesProcessed, + $mdFilesProcessed === 1 ? '' : 's', + $uuidsFixed, + $uuidsFixed === 1 ? '' : 's', + $mdDirsCreated, + $mdDirsCreated === 1 ? 'y' : 'ies', + $mdImagesCopied, + $mdImagesCopied === 1 ? '' : 's', + $mdImagesMissing, PHP_EOL ); diff --git a/src/App/src/Service/ArticleBodyCleaner.php b/src/App/src/Service/ArticleBodyCleaner.php index 2eccbccf..452e9b21 100644 --- a/src/App/src/Service/ArticleBodyCleaner.php +++ b/src/App/src/Service/ArticleBodyCleaner.php @@ -17,7 +17,7 @@ final class ArticleBodyCleaner public static function clean(string $body): string { $body = (string) preg_replace('/\A#[ \t][^\n]*\n/', '', $body, 1); - $body = (string) preg_replace('/^## TL;DR\s*$.*?(?=^## |\z)/ms', '', $body, 1); + $body = (string) preg_replace('/^## TL;DR\s*$\n\s*\n.*?(?=\n\s*\n|^## |\z)/ms', '', $body, 1); return trim($body); } diff --git a/test/Unit/App/Service/ArticleBodyCleanerTest.php b/test/Unit/App/Service/ArticleBodyCleanerTest.php index 7abc7b3d..c1e991cb 100644 --- a/test/Unit/App/Service/ArticleBodyCleanerTest.php +++ b/test/Unit/App/Service/ArticleBodyCleanerTest.php @@ -22,4 +22,28 @@ public function testCleanOnlyStripsTheTitleWhenThereIsNoTlDrSection(): void $this->assertSame("## First Section\n\nThe real content.", ArticleBodyCleaner::clean($body)); } + + public function testCleanStripsTlDrWithNoLeadingTitleWhenFollowedBySection(): void + { + $body = "## TL;DR\n\nA short summary.\n\n## First Section\n\nThe real content."; + + $this->assertSame("## First Section\n\nThe real content.", ArticleBodyCleaner::clean($body)); + } + + public function testCleanStripsTlDrWithNoLeadingTitleAndNoFollowingHeading(): void + { + $body = "## TL;DR\n\nA short summary.\n\nJust plain continuing content with no further heading."; + + $this->assertSame( + 'Just plain continuing content with no further heading.', + ArticleBodyCleaner::clean($body) + ); + } + + public function testCleanReturnsEmptyStringWhenTlDrIsTheEntireBody(): void + { + $body = "## TL;DR\n\nOnly a summary, nothing else."; + + $this->assertSame('', ArticleBodyCleaner::clean($body)); + } }