From 0451b31b84d2132b6f5cb91b76cb17e9c406ec7a Mon Sep 17 00:00:00 2001 From: David Arcos Date: Thu, 15 Oct 2026 09:59:38 +0200 Subject: [PATCH 1/5] Add the text rules the archive import shares: canon, slug, and titlematch Three modules under bin/, one rule each, so that the import scripts that follow cannot each carry a copy that drifts. bin/canon.py normalizes text read from the Meetup export: one deterministic transform in code, which never touches a letter or an accent. A model told not to change the typography was measured against the export: it changed the apostrophe anyway and corrupted an accented letter in the same pass, so the rule cannot live in a prompt. The module also strips Markdown and a leading time from a schedule label, which is what comparing a source string against an extracted one needs. It has a --selftest. bin/slug.py makes an address out of a title: the file name of a meetup and the id of a talk, which is the last segment of the URL of its page. It drops the accents, which canon() must never do to a stored string. bin/titlematch.py recognises a title inside another string that carries it: the caption of a photo that quotes the talk it shows, or the title of a video upload that names the talk it recorded. The two titles are two honest records of one session at two moments, so the rule is tolerant and documented, and it lives here once. --- bin/canon.py | 286 ++++++++++++++++++++++++++++++++++++++++++++++ bin/slug.py | 46 ++++++++ bin/titlematch.py | 228 ++++++++++++++++++++++++++++++++++++ 3 files changed, 560 insertions(+) create mode 100644 bin/canon.py create mode 100644 bin/slug.py create mode 100644 bin/titlematch.py diff --git a/bin/canon.py b/bin/canon.py new file mode 100644 index 000000000..b2b4b169f --- /dev/null +++ b/bin/canon.py @@ -0,0 +1,286 @@ +"""Canonical text form for the Meetup archive import. + +The decision this module encodes: text imported from the Meetup export is +stored normalized and consistent, not as a facsimile of Meetup. The +normalization has to be deterministic and in code, with one definition, and +it cannot depend on a model behaving well. A model told NOT to normalize +the typography was measured against the Meetup export: it changed the +apostrophe anyway, and in the same pass it corrupted an accented letter +that it had copied correctly without the rule. Code cannot do that, +because it never touches a letter. + +canon() is the normalizer. strip_md() and prep() support comparing a source +string against an extracted one: the extractor runs text through Markdown, +and a source string never was, so the two cannot be compared until the +Markdown is gone from both. strip_time() removes a schedule label's leading +time, because the time lives in the event's own start and end fields and +does not belong in the label text too. + +The one rule canon() never breaks: it does not touch a letter or an accent. +Everywhere else, "normalize the typography" is safe to read as "make it +closer to ASCII", and a model treats it that way. Here it is not: the data +covers Catalan, Spanish, and French, so NFKD-then-strip-accents or any other +latinizing transform would silently corrupt "Història" into "Historia" or +"Historia" into something that no longer matches its own source. The map +below stays within the set of marks and spaces that normal keyboard input +also maps to one of these forms; it never reaches into the letters. +""" + +import argparse +import re +import sys +import unicodedata + +# Every substitution below is built from its code point, not typed as the +# raw character. Several of these marks look alike, or look like nothing at +# all (a hair space renders as a gap the width of a typo), and a source file +# that carries them as raw bytes hides that from a reviewer: the same trap +# bin/check-html-safety guards against in built HTML. Spelling out the code +# point also keeps these tables in the ASCII range, so a diff of one never +# carries an invisible or look-alike character by accident. The rest of the +# file is not ASCII, and cannot be: the selftest table below holds a raw +# curly apostrophe and raw accented letters, because those are what it +# tests. + +# Quotation marks and primes that stand in for a straight quote on a +# keyboard that supports typing one directly, or that a word processor +# "smart quotes" feature produces from one. Catalan text uses the left and +# right single forms for its apostrophe (d'una), so both single and double +# families are mapped, not just the double ones a generic cleanup usually +# targets. +QUOTES = { + 0x2018: "'", 0x2019: "'", 0x201A: "'", 0x201B: "'", # single quotes + 0x201C: '"', 0x201D: '"', 0x201E: '"', # double quotes + 0x00AB: '"', 0x00BB: '"', # guillemets, used as quotes in French text + 0x2032: "'", 0x2033: '"', # prime and double prime +} + +# Dashes and the mathematical minus sign that a source pastes in place of a +# plain hyphen: Meetup descriptions carry an en dash, an em dash, and the +# Unicode minus sign interchangeably with "-". All of them collapse to the +# one ASCII hyphen, so a comparison never fails over which dash a human +# picked. +DASHES = { + 0x2013: "-", # en dash + 0x2014: "-", # em dash + 0x2212: "-", # minus sign + 0x2010: "-", # hyphen (the dedicated Unicode character, not "-") + 0x2011: "-", # non-breaking hyphen +} + +# A single ellipsis character, which prep() has to treat as the same text as +# three periods: a source may use either, and the extractor's model tends to +# write the three-period form regardless of which one the source used. +ELLIPSIS = {0x2026: "..."} + +# Non-breaking, figure, narrow, and thin spaces. Meetup's export carries these +# where a human just pressed the space bar on a system that substituted one, +# or where an editor inserted a non-breaking space before a unit or a +# punctuation mark. None of them is a letter, so all of them fold to the +# plain space. +SPACES = { + 0x00A0: " ", # no-break space + 0x2007: " ", # figure space + 0x202F: " ", # narrow no-break space + 0x2009: " ", # thin space + 0x200A: " ", # hair space +} + +_CANON_MAP = {} +for _table in (QUOTES, DASHES, ELLIPSIS, SPACES): + _CANON_MAP.update({chr(codepoint): replacement for codepoint, replacement in _table.items()}) +_CANON_TABLE = str.maketrans(_CANON_MAP) + +_HORIZONTAL_WHITESPACE = re.compile(r"[ \t]+") + + +def canon(text): + """Return the acceptable typographic form of text. + + NFC first, so a letter with a combining accent and the same letter with + a precomposed accent compare equal. Then the character substitutions + above: curly and angle quotes to a straight quote, every dash and the + minus sign to a hyphen, a one-character ellipsis to three periods, and + the non-ASCII spaces to a plain one. Runs of plain spaces and tabs + collapse to one space. + + This never changes a letter or an accent. That boundary is the entire + point of the function: the archive's source text is Catalan, Spanish, and + French, and a transform that also "normalizes" accents would turn + "Història" into "Historia" or "Historia" into "História". Either breaks + a comparison against the source and nobody would notice until a word + read wrong on the published page. + """ + text = unicodedata.normalize("NFC", text or "") + text = text.translate(_CANON_TABLE) + return _HORIZONTAL_WHITESPACE.sub(" ", text) + + +# [text](url) resolves to text. Applied before the character-class strip +# below, because the URL inside the parentheses can itself contain '*', '_', +# or '#', and stripping those first would leave the parentheses and the URL +# behind as stray text. +_MARKDOWN_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") + +# Markdown markup characters with no reading-order meaning of their own: +# emphasis (* _), blockquote (>), inline code (`), and heading (#). Removing +# them, rather than parsing Markdown into a tree, is enough because prep() +# only needs the reading text, not the structure. +_MARKDOWN_MARKUP = re.compile(r"[*_>`#]") + +_WHITESPACE_RUN = re.compile(r"\s+") + + +def strip_md(text): + """Return the Markdown-free reading text of text, for comparison. + + The extractor's output is Markdown; a source string pasted out of the + Meetup export never was. The two cannot be compared for equality until + both go through this: it resolves a link to its visible text, drops the + backslash of an escaped character, drops the emphasis/quote/code/heading + markup characters, drops every emoji and formatting character (Unicode + categories So "other symbol", Sk "modifier symbol", and Cf "format", which + covers variation selectors and zero-width joiners), and collapses all + whitespace, including a newline, to one space. + """ + text = (text or "").replace("\\", "") + text = _MARKDOWN_LINK.sub(r"\1", text) + text = _MARKDOWN_MARKUP.sub("", text) + text = "".join( + character for character in text + if unicodedata.category(character) not in ("So", "Sk", "Cf") + ) + return _WHITESPACE_RUN.sub(" ", text).strip() + + +def prep(text): + """Return text ready to compare: strip_md(canon(text)). + + This is what the extractor and the check run both the source and the + extracted text through before comparing them, so neither side's leftover + typography or Markdown can make a true match look like a mismatch, or + the reverse. + """ + return strip_md(canon(text)) + + +# A grid label carries its own leading time ("18:30: Welcome and Talks") +# because that is how it read on the Meetup page, but the imported event +# already stores the time in its start and end fields. Leaving it in the +# label would duplicate it and risk the two disagreeing after an edit to +# either one. The pattern accepts the two separators the export uses (":" or +# ".") between hour and minute, an optional "h" or "hs" unit after the +# minutes, and a colon, a hyphen, or nothing between the time and the label +# text. It runs after prep(), so by this point any dash has already become +# the plain hyphen "-" that canon() produces; there is no other dash form to +# match here. +# +# "hs" has to come before "h" in the alternation. Regex alternation tries +# each branch in order and stops at the first one that lets the rest of the +# pattern match, not the longest one: with "h" first, "9:00hs: Talk" matched +# only the "h", because the remaining "s: Talk" still satisfied the optional +# groups that follow, and the label came out as "s: Talk" instead of "Talk". +_TIME = r"\d{1,2}[:.]\d{2}\s*(?:hs|h)?\s*" +# The optional second time covers a range ("09:30 - 10:45 Keynote"), which is +# how the PyDay and PyData grids print a slot. Without it only the start time +# came off and the end time stayed in the label text. +_LEADING_TIME = re.compile(r"^" + _TIME + r"(?:[:-]\s*" + _TIME + r")?[:-]?\s*") + + +def strip_time(label): + """Return label with a leading grid time removed, for example: + + "18:30: Welcome and Talks" -> "Welcome and Talks" + "Welcome and Talks" -> "Welcome and Talks" (unchanged, no leading time) + """ + return _LEADING_TIME.sub("", prep(label)) + + +# --------------------------------------------------------------------------- +# Self-test +# --------------------------------------------------------------------------- + +# Each row is one behavior this module has to keep. "kind" says which +# function the row checks, "input" is what goes in, "expect" is what has to +# come out. This is the only test this repository has for canon.py, so the +# table is also where a regression gets added: a bug found later becomes a +# new row here, not a note in a pull request description. +_SELFTEST_CASES = [ + ("canon", "d’una", "d'una"), # Catalan curly apostrophe + ("canon", "Història", "Història"), # accent must not change + ("canon", "", ""), + ("canon", None, ""), + ("strip_md", None, ""), + ("strip_md", + "how [Yaetos](https://github.com/arthurprevot/yaetos) can help", + "how Yaetos can help"), + ("prep", None, ""), + ("strip_time", "18:30: Welcome and Talks", "Welcome and Talks"), + ("strip_time", "18.30 Welcome and Talks", "Welcome and Talks"), + ("strip_time", "9:00h - Talk", "Talk"), + ("strip_time", "9:00hs: Talk", "Talk"), + ("strip_time", "09:30 - 10:45 Keynote", "Keynote"), # a range, as PyDay prints it + ("strip_time", "09:30-10:45 Keynote", "Keynote"), + ("strip_time", "10:00 - 10:30: Coffee break", "Coffee break"), + ("strip_time", "Welcome and Talks", "Welcome and Talks"), # no leading time + ("strip_time", None, ""), +] + +_FUNCTIONS = {"canon": canon, "strip_md": strip_md, "prep": prep, "strip_time": strip_time} + + +def _run_selftest(): + """Run _SELFTEST_CASES plus the two checks that do not fit a (kind, input, + expect) row, print one line per case, and return True if all of them pass. + """ + passed = True + + for kind, value, expect in _SELFTEST_CASES: + got = _FUNCTIONS[kind](value) + ok = got == expect + passed = passed and ok + print("%-4s %-10s %-45r -> %-25r (expect %r)" + % ("ok" if ok else "FAIL", kind, value, got, expect)) + + # Història and História must stay distinct after prep(), because they + # are different words in different languages. A transform that folded + # accents, rather than merely normalizing typography, would make this + # pair equal and silently merge two different source texts. + distinct = prep("Història") != prep("História") + passed = passed and distinct + print("%-4s %-10s %r != %r" + % ("ok" if distinct else "FAIL", "prep", "Història", "História")) + + # canon() has to be idempotent: running it twice must give the same + # result as running it once, because prep() calls it once but the + # extractor and the check may each run it again on the other's output. + for kind, value, _ in _SELFTEST_CASES: + if kind != "canon" and value is None: + continue + once = canon(value) + twice = canon(once) + ok = once == twice + passed = passed and ok + print("%-4s %-10s canon(canon(%r)) == canon(%r)" + % ("ok" if ok else "FAIL", "idempotent", value, value)) + + return passed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--selftest", action="store_true", + help="run the built-in test table and exit 0 or 1") + args = parser.parse_args() + + if not args.selftest: + parser.print_help() + return 1 + + ok = _run_selftest() + print("\nPASS" if ok else "\nFAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/slug.py b/bin/slug.py new file mode 100644 index 000000000..e2da6fccc --- /dev/null +++ b/bin/slug.py @@ -0,0 +1,46 @@ +"""The URL form of a title, for this archive. + +A slug is an address: the file name of an imported meetup, and the last +segment of the URL of a talk page. It drops the accents, which canon() +must never do to a stored string and which a URL cannot carry. That is +safe here and nowhere else: the slug is the address, the title beside it +keeps every letter, and the two are written from the same record. + +One definition, because two imports need it. bin/import-meetups makes the +file name of a meetup out of its title, and bin/import-talks makes the id +of a talk out of its title, and the id is what content/talks/_content.gotmpl +puts in the URL of the talk page. A second copy of the rule would go on +making one of the two kinds of address after the rule changed. +""" + +import re +import unicodedata + +from canon import canon + +SLUG_LIMIT = 60 + + +def slugify(text, limit=SLUG_LIMIT): + """Return the URL form of a title: ASCII, lower case, hyphen separated. + + Accents are decomposed and the combining marks dropped, every character + that is not a letter or a digit becomes a hyphen, and runs collapse. + That folds an emoji, a quote, and a colon to the same separator, which + is what a reader of a URL expects. + + The result is cut at the last whole word inside limit, so a 150 + character title does not become a 150 character file name. The cut can + make two slugs equal, which is why the caller reports a collision + instead of overwriting a file. + """ + text = unicodedata.normalize("NFKD", canon(text)) + text = "".join(c for c in text if not unicodedata.combining(c)) + text = "".join(c if (c.isascii() and c.isalnum()) else "-" for c in text.lower()) + text = re.sub(r"-+", "-", text).strip("-") + if len(text) <= limit: + return text + cut = text[:limit] + if "-" in cut: + cut = cut[:cut.rindex("-")] + return cut.strip("-") diff --git a/bin/titlematch.py b/bin/titlematch.py new file mode 100644 index 000000000..17812f2da --- /dev/null +++ b/bin/titlematch.py @@ -0,0 +1,228 @@ +"""The comparison form of a title, for finding one title inside a text. + +Two imports of this archive have the same problem: a title written for a +reader has to be recognised inside another string that carries it. +bin/import-x-photos reads a tweet caption that quotes the title of the talk +it photographs, and bin/import-videos reads the title of a YouTube upload +that names the talk it recorded. The rule lives here, once, because a second +copy of it would go on matching one of the two after the rule changed. That +is the reason bin/slug.py exists as well. + +Why the two titles differ at all, which is the thing to understand before +trying to make the match stricter or looser. A Meetup announcement carries +the title the speaker wrote in the call for proposals, months earlier. The +video carries the title the speaker actually used on the day. They are two +honest records of the same talk at two moments, and neither is wrong when +they disagree: "GeoDjango in Agroptima" and "GeoDjango use case: mapping +farmer fields all over Spain" are the same session. So a miss here is the +normal case and not a defect to chase, and the right answer to one is a +human reading the two titles, not a lower guard. + +The rule, and its two guards. + +fold() is the comparison form: prep() from canon.py, then lower case, the +accents decomposed and dropped, and every mark replaced by a space. This is +the one place in this repository that folds an accent on purpose, and it is +safe only because nothing it returns is ever stored or published. A title +printed with its own punctuation has to compare equal to the programme: +"Models de regressio bayesians," in a post and "Models de regressió +bayesians" in the front matter are the same title. + +TITLE_MINIMUM is the length guard. A folded title of a few characters +appears inside unrelated prose, so matching it links the wrong talk. +Measured on 2026-10-13 over the 231 photos the X import brought in, this +cut keeps the 46% that carry a real title, 107 of them, and costs none. + +match_title() is the ambiguity guard. When two candidates match one text it +returns no key and both of them. A wrong link is worse than no link, +because the page then points the reader at another talk and nothing on it +says so. + +Containment runs one way by default: the candidate title inside the text. A +caption quotes the whole title and adds the speaker, so there the title is +the shorter side. A YouTube title is not always the longer side: the +uploader shortens a long title and writes "[...]" where the cut is, so the +text is then the shorter one. both_ways accepts containment in either +direction, with the length guard on whichever side is the needle, and the +caller says which of the two it needs. +""" + +import argparse +import re +import sys +import unicodedata + +# canon.py sits next to this module and is not an installed package. A +# caller already puts bin/ on the path to import this file, so the plain +# import works for the caller too. +from canon import prep + +# The shortest folded title that may be matched inside another string. See +# the length guard above. +TITLE_MINIMUM = 12 + +# Every character that is not a letter or a digit becomes a space, so a +# title compares equal to the same title written with other punctuation. A +# space and not a hyphen, because the result is read as words and never as +# an address: bin/slug.py is what makes an address. +_NOT_WORD = re.compile(r"[^a-z0-9]+") + + +def fold(text): + """Return the comparison form of text. + + prep() first, so the Markdown and the emoji of a source string are gone + and both sides arrive in the one normalized form the archive import + shares. Then lower case, the accents off, and every mark replaced by a + space. + """ + lowered = prep(text).lower() + folded = unicodedata.normalize("NFKD", lowered) + folded = "".join(character for character in folded + if not unicodedata.combining(character)) + return re.sub(r"\s+", " ", _NOT_WORD.sub(" ", folded)).strip() + + +def contains(haystack, needle): + """Return True when folded needle sits inside folded haystack. + + Both arguments are already folded. A needle shorter than TITLE_MINIMUM + never matches, whichever side it came from. + """ + return len(needle) >= TITLE_MINIMUM and needle in haystack + + +def match_title(text, candidates, both_ways=False): + """Return (key, hits) for one text against folded candidate titles. + + candidates is a sequence of (key, folded title) pairs, folded once by + the caller because the same candidates are matched against many texts. + The return is the one key that matched and no hit, or no key and every + hit: one hit is a match, and none or several is not. + """ + folded = fold(text) + if not folded: + return None, [] + hits = [key for key, candidate in candidates + if contains(folded, candidate) + or (both_ways and contains(candidate, folded))] + if len(hits) == 1: + return hits[0], [] + return None, hits + + +# --------------------------------------------------------------------------- +# Self-test +# --------------------------------------------------------------------------- + +# Each row is one behavior this module has to keep: the folded form of a +# string, or the (key, hits) of one match. A bug found later becomes a new +# row here, not a note in a pull request description. +_FOLD_CASES = [ + ("Models de regressió bayesians", "models de regressio bayesians"), + ("Models de regressio bayesians,", "models de regressio bayesians"), + ("PySpark on Kubernetes -- Avoiding the pain of YARN", + "pyspark on kubernetes avoiding the pain of yarn"), + ("PySpark on Kubernetes: Avoiding the pain of YARN", + "pyspark on kubernetes avoiding the pain of yarn"), + ("Architecture Decision Records: documenta tu arquitectura [...]", + "architecture decision records documenta tu arquitectura"), + ("A Day Has Only 24±1 Hours", "a day has only 24 1 hours"), + ("", ""), + (None, ""), +] + +# One programme, folded once, as a caller passes it. +_TALKS = [ + ("api-by-example", fold("API by example")), + ("hacking-the-bcn-housing-market-scrapy", + fold('"Hacking the BCN housing market" (Scrapy)')), + ("comprehensions-in-5-min", fold("Comprehensions in 5 min")), + ("frozen-python-satice-ice-dataloggers", + fold("Frozen Python: satice ice dataloggers")), +] + +_MATCH_CASES = [ + # A caption quotes the whole title and adds the speaker: one way is enough. + ("PyBCN Meetup 2015/10 - API by example by C. Barrobés", False, + "api-by-example", []), + # The uploader shortened the title, so the text is the shorter side and + # one way finds nothing. + ("PyBCN Meetup 2015/09 - Hacking the BCN housing market by T. Barbu", + False, None, []), + ("Hacking the BCN housing market", True, + "hacking-the-bcn-housing-market-scrapy", []), + ("Frozen Python", True, "frozen-python-satice-ice-dataloggers", []), + # Under the length guard, both ways: "comprehensions" alone is 14 + # characters, but the folded text here is 11 and never matches. + ("Comprehensi", True, None, []), + # Nothing of the programme is in this text. + ("PyDayBCN 2020 Track 1 - Retrying, a better way by C. Adell", True, + None, []), + ("", True, None, []), +] + +# Two talks of one day with the same title in them: the match is ambiguous +# and the return names both, which is what stops a wrong link. +_AMBIGUOUS_TALKS = [ + ("introduction-to-pyscript", fold("Introduction to PyScript")), + ("introduction-to-pyscript-2", fold("Introduction to PyScript")), +] + + +def _run_selftest(): + """Run the tables above, print one line per case, and return True when + every one of them passes. + """ + passed = True + + for value, expect in _FOLD_CASES: + got = fold(value) + ok = got == expect + passed = passed and ok + print("%-4s %-10s %-62r -> %r" + % ("ok" if ok else "FAIL", "fold", value, got)) + + for text, both_ways, expect_key, expect_hits in _MATCH_CASES: + key, hits = match_title(text, _TALKS, both_ways=both_ways) + ok = key == expect_key and hits == expect_hits + passed = passed and ok + print("%-4s %-10s %-62r -> %r %r" + % ("ok" if ok else "FAIL", + "both" if both_ways else "one way", text, key, hits)) + + key, hits = match_title("Introduction to PyScript", _AMBIGUOUS_TALKS, + both_ways=True) + ok = key is None and len(hits) == 2 + passed = passed and ok + print("%-4s %-10s two talks of one title -> %r %r" + % ("ok" if ok else "FAIL", "ambiguous", key, hits)) + + # The guard is on the needle and not on the text, so a long text never + # buys a short title a match. + ok = not contains(fold("Comprehensions in 5 min and more words"), + fold("in 5 min")) + passed = passed and ok + print("%-4s %-10s a short title is not matched in a long text" + % ("ok" if ok else "FAIL", "guard")) + + return passed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--selftest", action="store_true", + help="run the built-in test tables and exit 0 or 1") + args = parser.parse_args() + + if not args.selftest: + parser.print_help() + return 1 + + ok = _run_selftest() + print("\nPASS" if ok else "\nFAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) From cf8d50f4e3aa992387543c586197578758ade638 Mon Sep 17 00:00:00 2001 From: David Arcos Date: Thu, 15 Oct 2026 09:59:38 +0200 Subject: [PATCH 2/5] Add bin/check-extraction, the literalness check, with its prompt and fixtures The import cannot parse 19 years of volunteer-written Markdown with one parser, so a model reads each event description and returns the talks it announced. A model can rewrite a word instead of copying it, and a rewritten word on a published page looks like the real one. bin/check-extraction compares every extracted title and description against the normalized source, field by field, through bin/canon.py. A curly apostrophe turned straight passes; a changed accent fails. That is the boundary the archive needs: it holds Catalan, Spanish, and French. bin/prompts/extract-talks.md is the prompt the model gets. It asks for a copy, never a correction, and says nothing about typography, because the normalizer owns that. tests/extraction/ records the measurement of 2026-10-11: 9 events from the export, 3 per format era, the talks each one announces read by hand, 7 real model runs, and the verdict the check has to give on each. One run has to fail, so the selftest proves the check still rejects a changed accent. The README of the directory says what each file is and why the recorded runs keep their old key names. pr.yml runs both selftests, and CONTRIBUTING.md lists five checks instead of three. --- .github/workflows/pr.yml | 9 + CONTRIBUTING.md | 6 +- bin/check-extraction | 926 ++++++++++++++++++ bin/prompts/extract-talks.md | 100 ++ tests/extraction/README.md | 90 ++ tests/extraction/events.json | 65 ++ tests/extraction/expectations.yaml | 137 +++ tests/extraction/model-runs/haiku-1.json | 309 ++++++ tests/extraction/model-runs/haiku-2.json | 239 +++++ tests/extraction/model-runs/haiku-3.json | 243 +++++ .../model-runs/haiku-typography-rule-1.json | 222 +++++ .../model-runs/haiku-typography-rule-2.json | 239 +++++ tests/extraction/model-runs/opus-1.json | 254 +++++ tests/extraction/model-runs/sonnet-1.json | 309 ++++++ tests/extraction/truth.yaml | 121 +++ 15 files changed, 3268 insertions(+), 1 deletion(-) create mode 100755 bin/check-extraction create mode 100644 bin/prompts/extract-talks.md create mode 100644 tests/extraction/README.md create mode 100644 tests/extraction/events.json create mode 100644 tests/extraction/expectations.yaml create mode 100644 tests/extraction/model-runs/haiku-1.json create mode 100644 tests/extraction/model-runs/haiku-2.json create mode 100644 tests/extraction/model-runs/haiku-3.json create mode 100644 tests/extraction/model-runs/haiku-typography-rule-1.json create mode 100644 tests/extraction/model-runs/haiku-typography-rule-2.json create mode 100644 tests/extraction/model-runs/opus-1.json create mode 100644 tests/extraction/model-runs/sonnet-1.json create mode 100644 tests/extraction/truth.yaml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 019cdbdd8..6d175291b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -64,6 +64,15 @@ jobs: - name: Check content for dangerous raw HTML run: bin/check-html-safety + - name: Check the text normalizer + run: python3 bin/canon.py --selftest + + - name: Check the archive extraction fixtures + # Reads only tests/extraction/, so it needs no Meetup export. It proves + # the check still rejects a run that changed an accent, which is the + # one defect no normalizer can forgive. + run: bin/check-extraction --selftest + links: name: Check the links runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6b672f48..7eb00b302 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,12 +122,14 @@ Nothing is pushed to `edition` directly. verifies its checksum. You do not need Python, Go, or npm for this step. 3. Run `bin/serve` to see the site at `http://localhost:1313` while you work. 4. Make the change. -5. Run the three checks locally, so you find what the pull request would find: +5. Run the five checks locally, so you find what the pull request would find: ``` pip install pyyaml pillow bin/check-content bin/check-html-safety + python3 bin/canon.py --selftest + bin/check-extraction --selftest bin/hugo --minify -D -d public && bin/check-rendered ``` @@ -151,6 +153,8 @@ What each check is for: | `bin/check-content` | Front matter does not parse, a person `id` does not match its filename, an id is duplicated, a declared photo is missing, an event references a person or sponsor that does not exist, a social URL has the wrong shape, or a field is empty. It warns, without failing, about a photo that is too small or has no colour | | `bin/check-html-safety` | Content carries raw HTML that turns a content change into script execution, a redirect, a credential prompt, or a page overlay. See [Content safety check](README.md#content-safety-check) | | `bin/check-rendered` | The built pages carry that same markup, which catches a template that produces it even when no content file does, or they link to anything over `http`. See [Rendered page check](README.md#rendered-page-check) | +| `bin/canon.py --selftest` | The one normalization of imported text stops being idempotent, or changes what it strips | +| `bin/check-extraction --selftest` | The recorded model runs of `tests/extraction/` stop giving the verdict they are recorded with, one of which has to be a failure | | Check the links | Reported, never blocking, because external sites rate-limit | A merge to `edition` deploys the site. The `github-pages` workflow builds it and diff --git a/bin/check-extraction b/bin/check-extraction new file mode 100755 index 000000000..33892cceb --- /dev/null +++ b/bin/check-extraction @@ -0,0 +1,926 @@ +#!/usr/bin/env python3 +"""Validate a talk extraction against the Meetup source it was read from. + +The archive import cannot parse 19 years of volunteer-written Markdown with +one parser, so a model reads each event description and returns the talks it +announced. A model can rewrite a word instead of copying it, and a rewritten +word on a published page looks like the real one. This check is what makes +that impossible to miss. + +It compares every extracted title and description against the normalized +source, not against the raw source. The extractor's output went through +Markdown and the source string never did, so the raw forms cannot be +compared at all: a link in the source ("[Yaetos](https://...)") makes a +correct abstract look invented. bin/canon.py defines the one normalization +both sides go through, and it never touches a letter or an accent. So a +curly apostrophe turned straight passes, and a changed accent fails. That +boundary is the whole point: the archive holds Catalan, Spanish, and French +text, where "Historia" with an acute accent and with a grave accent are two +different words. + +Three modes: + + Literalness (the default) reads an extraction and its source events, and + reports every field that is not a substring of the normalized source, + with the divergence located and classified: spacing (a warning), + accent_or_case (an error), or content (an error). It also reports a field + the source does not ground, a description the model cut short (an error), + and a shape defect such as a missing event. + + Two divergences from the source are not defects, and the literalness + mode resolves them instead of reporting them. A bare URL in parentheses + may be absent from a title or an abstract, because it is a reference and + not prose: see drop_urls(). A duration may be the span between two times + of the source, because reading "09:30" and "10:30" as 60 minutes is a + derivation: see spans_of(). + + The substring test has one hole by construction: a shortened text is + still a substring of the one it was cut from, so a model that drops half + an abstract passes it. Losing half a description in an archive nobody + reads again is exactly the silent failure this check exists to prevent, + so a description that stops in the middle of its own source line, at no + sentence end, is reported as truncated. See truncation(). + + Benchmark scores an extraction against tests/extraction/truth.yaml, the + reference read by hand before any model ran. + + Selftest runs both modes over the 7 recorded runs in + tests/extraction/model-runs/ and compares the result with + tests/extraction/expectations.yaml, field by field. One recorded run is + corrupted on purpose and has to fail, with exactly one accent_or_case + error. A check that only ever sees good input proves nothing. + +Exit codes: + 0 no error. Warnings do not fail the build + 1 at least one error, or a selftest result that differs from the + expectations + 2 the check could not run: bad arguments, a missing file, or a file it + cannot parse + +Usage: + bin/check-extraction EXTRACTION.json EVENTS.json + bin/check-extraction --benchmark EXTRACTION.json [--truth TRUTH.yaml] + bin/check-extraction --selftest +""" + +import argparse +import difflib +import json +import pathlib +import re +import sys +import unicodedata + +import yaml + +# canon.py sits next to this script and is not an installed package. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from canon import canon, prep, strip_time # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parent.parent +FIXTURES = ROOT / "tests" / "extraction" +MODEL_RUNS = FIXTURES / "model-runs" +EVENTS = FIXTURES / "events.json" +TRUTH = FIXTURES / "truth.yaml" +EXPECTATIONS = FIXTURES / "expectations.yaml" + +# The two text fields the model has to copy and not write. Everything else it +# returns is either a name, a number, or a structure, and each of those has +# its own check below. +TEXT_FIELDS = ("title", "description") + +# A schedule is "agenda" in bin/prompts/extract-talks.md. The recorded runs +# predate that name and write "non_talk_items", so both are read: the check +# has to work on the fixtures it ships with. +AGENDA_KEYS = ("agenda", "non_talk_items") + +# The shortest run of characters that anchors the extracted text to its place +# in the source. Three characters or fewer match by accident ("the", " a "), +# which drags the window to the wrong paragraph and turns one changed letter +# into a whole false divergence. A long field needs a long anchor for the +# same reason, so the floor grows with the field: an anchor has to be at +# least one ANCHOR_SHARE of it. Without that, an invented sentence anchored +# on " a " and "ing" produced a window the length of the whole event and a +# report nobody could read. +MIN_ANCHOR = 4 +ANCHOR_SHARE = 8 + +# How much of the source to print on each side of a divergence, how long a +# single fragment may get before its middle is elided, and how many +# divergences to print for one field. A field that diverges everywhere is +# invented text, and the first few spans already say so. +CONTEXT = 30 +MAX_FRAGMENT = 60 +MAX_SPANS = 3 + +# How much of the field, and of the source after it, a truncation report +# prints. Enough to show the word the cut fell inside and the clause that +# follows it, short enough to keep each line readable. +CONTINUATION = 50 + +# The characters a complete description is allowed to stop at: the sentence +# marks, a closing parenthesis, and the straight quotes canon() produces +# from the curly ones. A description that ends on one of these is finished, +# even when its source line carries more text after it, because the line +# usually goes on with the next field of the template. +SENTENCE_END = ".!?:)'\"" + + +# --------------------------------------------------------------------------- +# Comparison forms +# +# These three reductions classify a divergence. None of them ever runs over +# text this repository stores: fold() strips accents, which canon.py refuses +# to do precisely because it corrupts Catalan. Here that is the point. A pair +# that differs before folding and matches after it is a changed accent, which +# is the defect this check exists to catch. +# --------------------------------------------------------------------------- + +def as_text(value): + """Return value as a string, so a malformed field cannot crash the check. + + An extraction is model output. A field that should hold a string can hold + a number or null, and the check has to report that, not raise on it. + """ + if value is None: + return "" + return value if isinstance(value, str) else str(value) + + +def alnum(text): + """Return only the letters and digits of text, in order.""" + return "".join(character for character in text if character.isalnum()) + + +def fold(text): + """Return text lowercased and without accents, for classification only.""" + decomposed = unicodedata.normalize("NFD", text.lower()) + return "".join(character for character in decomposed + if not unicodedata.combining(character)) + + +def loose(text): + """Return a title reduced for matching against the reference. + + prep() first, then lowercase, then drop everything that is not a letter, + a digit, an underscore, or a space. The underscore survives, because the + character class keeps a word character and an underscore is one. No title + in the archive turns on an underscore, so that costs nothing. + + The reference writes a title as the source does, and an extraction may + keep or drop a trailing period, a colon, or a pair of quotes without + being wrong about which talk it found. + """ + reduced = re.sub(r"[^\w\s]", " ", prep(as_text(text)).lower(), flags=re.UNICODE) + return re.sub(r"\s+", " ", reduced).strip() + + +# --------------------------------------------------------------------------- +# Locating a divergence +# --------------------------------------------------------------------------- + +def source_text(event): + """Return the normalized text an extraction of this event may copy from. + + The event title counts as source. A workshop with no talk title of its + own takes the event's, which bin/prompts/extract-talks.md asks for, so a + check that read only the description would report that title as invented. + """ + title = as_text(event.get("title")) + description = as_text(event.get("description")) + return prep(f"{title}\n{description}") + + +def source_lines(event): + """Return the normalized source of this event, one entry per source line. + + source_text() collapses the newlines, because a field the model copied + may span them and a comparison must not fail over that. Truncation needs + the opposite: the line is what says where a text ends. canon() keeps the + newlines, so the split happens on the normalized text, and prep() then + runs over each line on its own. + """ + title = as_text(event.get("title")) + description = as_text(event.get("description")) + return [prep(line) for line in canon(f"{title}\n{description}").split("\n")] + + +def truncation(lines, extracted): + """Return the source text that follows a cut-short field, or None. + + extracted is already prep()ed, and the caller has established that it + copies the source. The field is cut short when its source line goes on + after it and it stops at no sentence end. A field that ends a line is + complete, whatever its last character is. + + Nothing here looks at the title: a title legitimately ends on a letter, + and the event title sits on the same line as nothing else, so the rule + would report every one of them. The caller applies this to descriptions + only. + """ + if not extracted or extracted[-1] in SENTENCE_END: + return None + rest = None + for line in lines: + start = line.find(extracted) + while start >= 0: + after = line[start + len(extracted):] + # One occurrence that ends its line is enough to say the field is + # whole, even when another copy of the same text sits mid-line. + if not after: + return None + if rest is None: + rest = after + start = line.find(extracted, start + 1) + return rest + + +# The Meetup export prints a link as its visible text followed by the bare +# address in parentheses: "Ansible (http://www.ansible.com)". A model that +# copies the sentence it reads leaves that parenthesis out, and it is right +# to: an address is a reference, not prose, and the model returns it in +# `links`, which is where the import takes it from. This is the decision +# prep() already makes for the other form the same link takes in this +# archive, where "[text](url)" resolves to "text". +# +# Only a parenthesis that holds nothing but an address goes. A parenthesis +# with prose in it is prose, and a changed word inside one still fails. +_BARE_URL = re.compile(r"\s*\((?:https?://|www\.)[^\s()]*\)") + + +def drop_urls(text): + """Return text without the parentheses that hold only a bare URL. + + The substitution repeats until nothing changes. The export nests the + form inside itself when the visible text is an address as well, as in + "Kinto (https://kinto.readthedocs.io (https://kinto.readthedocs.io/))", + and one pass leaves the outer parenthesis behind. + """ + while True: + shorter = _BARE_URL.sub("", text) + if shorter == text: + return text + text = shorter + + +def align(source, extracted): + """Return the slice of source that lines up with extracted, or None. + + The window is anchored on the first and the last long matching run, and + then extended by however much of extracted hangs off each end, so the + window and extracted have the same length when the only difference is a + substitution. None means extracted has too little in common with the + source to align at all, which is a content defect and needs no window. + """ + matcher = difflib.SequenceMatcher(None, source, extracted, autojunk=False) + floor = max(MIN_ANCHOR, len(extracted) // ANCHOR_SHARE) + anchors = [block for block in matcher.get_matching_blocks() if block.size >= floor] + # Half of the field has to sit on an anchor. Below that the field is not + # a copy of anything in the source, and aligning it against the nearest + # paragraph only produces a long report about a text the source never + # had. "Not in the source" is both true and shorter. + if not anchors or sum(anchor.size for anchor in anchors) * 2 < len(extracted): + return None + first, last = anchors[0], anchors[-1] + start = max(0, first.a - first.b) + tail = len(extracted) - (last.b + last.size) + end = min(len(source), last.a + last.size + max(tail, 0)) + return start, end + + +def divergences(window, extracted): + """Return (core, spans) for a window and the text aligned against it. + + core is the window with its border noise removed. A window cut out of a + longer source carries a head and a tail that belong to the source and not + to the extracted field: the "Abstract: " that sits right before the text + is the one that bit us. Those show up as a deletion that touches an edge + of the window and consumes none of the extracted text, and they are not + divergences. Everything else is, and comes back as a span of + (source fragment, extracted fragment) with its offset in core. + """ + matcher = difflib.SequenceMatcher(None, window, extracted, autojunk=False) + opcodes = matcher.get_opcodes() + + head, tail = 0, len(window) + kept = [] + for index, (tag, i1, i2, j1, j2) in enumerate(opcodes): + if tag == "equal": + continue + border = (index == 0 and i1 == 0 and j1 == 0) or ( + index == len(opcodes) - 1 and i2 == len(window) and j2 == len(extracted)) + if tag == "delete" and border: + if i1 == 0: + head = i2 + else: + tail = i1 + continue + kept.append((i1, i2, j1, j2)) + + core = window[head:tail] + spans = [(i1 - head, window[i1:i2], extracted[j1:j2]) for i1, i2, j1, j2 in kept] + return core, spans + + +def classify(core, extracted): + """Return the class of the difference between core and extracted. + + spacing the letters and digits are the same, so only space or + punctuation moved. A warning: canon.py already decides + the typography of what gets stored. + accent_or_case the same text once accents and case are folded away, and + not before. An error, and the one this check is for. + content anything else: another word, another number, or text the + source does not have. An error. + """ + if alnum(core) == alnum(extracted): + return "spacing" + if fold(core) == fold(extracted) or fold(alnum(core)) == fold(alnum(extracted)): + return "accent_or_case" + return "content" + + +def show(core, spans): + """Return one (source line, extracted line) pair per divergence. + + Each line is the fragment in its context, with the divergence itself in + square brackets. A failure a human cannot read is worth nothing, and the + change is usually one letter in a 500-character abstract. + """ + lines = [] + for offset, from_source, from_extracted in spans[:MAX_SPANS]: + before = core[max(0, offset - CONTEXT):offset] + after = core[offset + len(from_source):offset + len(from_source) + CONTEXT] + lead = "..." if offset > CONTEXT else "" + trail = "..." if offset + len(from_source) + CONTEXT < len(core) else "" + lines.append(( + f"{lead}{before}[{elide(from_source)}]{after}{trail}", + f"{lead}{before}[{elide(from_extracted)}]{after}{trail}", + )) + left = len(spans) - len(lines) + if left: + lines.append((f"({left} more divergences in this field)", "")) + return lines + + +def elide(fragment): + """Return fragment with its middle cut out once it gets too long.""" + if len(fragment) <= MAX_FRAGMENT: + return fragment + half = (MAX_FRAGMENT - 5) // 2 + return f"{fragment[:half]} ... {fragment[-half:]}" + + +# --------------------------------------------------------------------------- +# Mode 1: literalness +# --------------------------------------------------------------------------- + +def check_field(event_id, where, field, value, source, lines): + """Return the findings for one text field. An empty list means it copies. + + A finding is a dict with the event, the field, the class, and the aligned + fragments to print. + + A field that is a substring of the source copies it, which leaves only + one question: whether it copies all of it. lines carries the source + split by line, and truncation() answers that for a description. + """ + extracted = prep(as_text(value)) + if not extracted: + return [] + + # The field is read twice: as it stands, and with every bare + # parenthesized URL dropped from both sides. Both sides, because one + # abstract keeps one of those references and drops the next. The second + # form is only built when there is a reference to drop, so a field that + # carries none is compared exactly once, as before. + forms = [(extracted, source, lines)] + without_urls = drop_urls(extracted) + if without_urls != extracted or drop_urls(source) != source: + forms.append((without_urls, drop_urls(source), + [drop_urls(line) for line in lines])) + + for copied, within, inside in forms: + if copied not in within: + continue + rest = truncation(inside, copied) if field == "description" else None + if rest is None: + return [] + return [{ + "event_id": event_id, "where": where, "field": field, + "klass": "truncated", + "lines": [ + (f"the field stops at: {copied[-CONTINUATION:]!r}", ""), + (f"the source line goes on: {rest[:CONTINUATION]!r}", ""), + ], + }] + + located = align(source, extracted) + if located is None: + return [{ + "event_id": event_id, "where": where, "field": field, + "klass": "content", "lines": [("(nothing in the source)", extracted)], + }] + + start, end = located + core, spans = divergences(source[start:end], extracted) + if not spans: + # The window carried only border noise, so the field does copy the + # source and the substring test failed on the edges alone. + return [] + return [{ + "event_id": event_id, "where": where, "field": field, + "klass": classify(core, extracted), "lines": show(core, spans), + }] + + +# A clock reading in the source, written with either separator the export +# uses: "18:30" and "18.30" are both times, which bin/canon.py already +# documents for the schedule labels it strips a time from. +# +# The hour stops at 23 and the minute at 59, so a figure that cannot be a +# time is not read as one. "Python 3.5" still reads as 3:05, and no pattern +# tells those two apart. That costs nothing here: a duration is accepted +# only when it equals the span between two readings, so a stray one has to +# land on the exact number the extraction claims. +_CLOCK = re.compile(r"(?6}{'required':>10}{'missing':>9}" \ + f"{'spurious':>10}{'speakers':>10}{'ambiguous':>11}" + print(header) + print("-" * len(header)) + for row in score["per_event"]: + print(f"{row['event_id']:<12}{row['talks']:>6}{row['required']:>10}" + f"{row['required_missing']:>9}{row['spurious']:>10}" + f"{row['wrong_speakers']:>10}{row['ambiguous_included']:>11}") + print("-" * len(header)) + print(f"{'total':<12}{score['talks']:>6}{'':>10}{score['required_missing']:>9}" + f"{score['spurious']:>10}{score['wrong_speakers']:>10}" + f"{score['ambiguous_included']:>11}") + + +# --------------------------------------------------------------------------- +# Mode 3: the selftest +# --------------------------------------------------------------------------- + +# The keys expectations.yaml records, and where each one comes from. Both +# sections read the same measurement: ungrounded_fields is measured by the +# literalness check and recorded under benchmark. +LITERAL_KEYS = ("verdict", "fields_checked", "identical_to_normalized_source", + "spacing_warnings", "accent_or_case_errors", "content_errors", + "truncated_errors") +BENCHMARK_KEYS = ("talks", "required_missing", "spurious", "wrong_speakers", + "ungrounded_fields", "ambiguous_included") + + +def run_selftest(): + """Measure every recorded run and compare it with the expectations. + + Return the number of runs that do not match. The fixtures are the only + regression test this check has, so a mismatch is a defect in the check: + expectations.yaml and truth.yaml are the reference, never the thing to + edit. + """ + events = load_json(EVENTS) + truth = load_yaml(TRUTH) + expectations = load_yaml(EXPECTATIONS) + + failed = 0 + for name in sorted(expectations): + expected = expectations[name] + path = MODEL_RUNS / f"{name}.json" + if not path.exists(): + print(f"FAIL {name}: {path} is missing") + failed += 1 + continue + + extraction = load_json(path) + literal = run_literal(extraction, events) + benchmark = run_benchmark(extraction, truth) + benchmark["ungrounded_fields"] = literal["ungrounded_fields"] + + got = { + "literal": {key: literal[key] for key in LITERAL_KEYS}, + "benchmark": {key: benchmark[key] for key in BENCHMARK_KEYS}, + } + + differences = [] + for section in ("literal", "benchmark"): + for key, want in (expected.get(section) or {}).items(): + mine = got[section].get(key) + if mine != want: + differences.append(f"{section}.{key}: expected {want!r}, got {mine!r}") + + if differences: + failed += 1 + print(f"FAIL {name}") + for line in differences: + print(f" {line}") + for finding in literal["findings"]: + print(f" measured: event {finding['event_id']} " + f"{finding['field']} {finding['klass']}") + else: + print(f"ok {name}: {expected['literal']['verdict']}, " + f"{got['literal']['fields_checked']} fields, " + f"{got['literal']['accent_or_case_errors']} accent_or_case, " + f"{got['literal']['content_errors']} content, " + f"{got['benchmark']['talks']} talks, " + f"{got['benchmark']['required_missing']} missing, " + f"{got['benchmark']['spurious']} spurious") + + total = len(expectations) + print(f"\n{total - failed} of {total} recorded runs match the expectations.") + return failed + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def unreadable(path, problem): + """Leave with code 2: the check could not run, and found no defect.""" + print(f"cannot read {path}: {problem}", file=sys.stderr) + sys.exit(2) + + +def load_json(path): + try: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError) as problem: + unreadable(path, problem) + + +def load_yaml(path): + try: + with open(path, encoding="utf-8") as handle: + return yaml.safe_load(handle) or {} + except (OSError, yaml.YAMLError) as problem: + unreadable(path, problem) + + +def main(): + parser = argparse.ArgumentParser( + description="Validate a talk extraction against its normalized source.") + parser.add_argument("extraction", nargs="?", help="the extraction JSON") + parser.add_argument("events", nargs="?", help="the source events JSON") + parser.add_argument("--benchmark", action="store_true", + help="score the extraction against truth.yaml") + parser.add_argument("--truth", default=str(TRUTH), + help=f"the reference to score against (default {TRUTH})") + parser.add_argument("--selftest", action="store_true", + help="measure the recorded runs against expectations.yaml") + args = parser.parse_args() + + if args.selftest: + if args.extraction or args.benchmark: + parser.error("--selftest takes no other argument") + return 1 if run_selftest() else 0 + + if args.benchmark: + if not args.extraction or args.events: + parser.error("--benchmark takes one extraction file") + score = run_benchmark(load_json(args.extraction), load_yaml(args.truth)) + report_benchmark(score) + # A missing required talk, a spurious one, or a wrong speaker is an + # error, so this mode exits 1 like the other one. An ambiguous talk + # never counts, in either direction: the source does not settle it. + return 1 if (score["required_missing"] or score["spurious"] + or score["wrong_speakers"]) else 0 + + if not args.extraction or not args.events: + parser.error("give an extraction file and an events file") + result = run_literal(load_json(args.extraction), load_json(args.events)) + return 1 if report_literal(result) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/prompts/extract-talks.md b/bin/prompts/extract-talks.md new file mode 100644 index 000000000..46e668f8e --- /dev/null +++ b/bin/prompts/extract-talks.md @@ -0,0 +1,100 @@ +# Extract the talks from a Meetup event description + +You get a JSON list of Meetup events. Each one has `id`, `title`, `dateTime`, `eventUrl`, and +`description`. The description is Markdown written by volunteers over 19 years, in three formats +that changed with the years. Your job is to find the talks each event announced. + +## Copy the text, do not write it + +- Copy the `title` and the `description` of each talk **from the source text**. Keep the wording, + the spelling, the punctuation, the capitalisation, and the accents. +- **Do not translate.** A Catalan or Spanish title stays in Catalan or Spanish. +- **Do not correct** a typo, a misspelling, or a name. If the source says `simluation`, you write + `simluation`. +- **Do not summarise, rewrite, complete, or improve** a title or a description. +- You may drop the Markdown markers (`**`, `*`, `_`, `>`), the escape backslashes, the decorative + emoji of the template itself, and the leading and trailing spaces. +- If a title is cut short or badly built in the source, you copy it cut short. + +## What counts as a talk + +A talk is a presentation announced with a title. Record it with `type: talk`. + +A **workshop, a practice session, or a tutorial** is also a talk, with `type: workshop`. + +When such a session announces no talk title of its own, and the event is one single session that +teaches or presents something, record **one** talk. Its title is the event title with the series +prefix removed, which is the text after the final colon. For an event titled +`May PyBCN Practice Sessions: Intro to Image Processing and Computer Vision`, the title is +`Intro to Image Processing and Computer Vision`. Its description is the part of the event +description that says what the session does. + +When the event is a work session with no presentation at all, a sprint, a hack night, a code +dojo with no programme, or a purely social meeting, record **no talk**. A documentation sprint +announces no talk, so the right answer is an empty list. + +Never record as a talk: a coffee break, networking, the welcome, the goodbye, the venue, the +access control, the call for proposals, or a job offer. + +## Output + +Write **only** a JSON file, in this exact shape: + +```json +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "...", + "speakers": ["Firstname Lastname"], + "description": "... or null when the event gives no abstract ...", + "language": "English | Spanish | Catalan | null", + "duration_minutes": 30, + "type": "talk | workshop | lightning | panel | keynote", + "links": [{"type": "slides|video|github|site|speaker_profile", "url": "https://..."}] + } + ], + "agenda": [ + {"start": "18:30", "end": null, "label": "Welcome and Talks", "talk_title": null} + ] + } +] +``` + +- One object per event, in the same order as the input, with every event present. +- `speakers`: the names as the source writes them. An empty list when nobody is named. +- `description`, `language`, `duration_minutes`: `null` when the source does not say. Never guess + and never infer. +- `links`: only URLs that are in the text and belong to that talk, so the speaker profile, the + project repository, the slides, or the video. An empty list when there are none. +- `agenda`: one entry per slot, but **only when the event prints a schedule**. An empty list + otherwise. Put the time in `start` and `end`, never inside `label`. Point `talk_title` at the + talk when the slot is a talk, and leave it `null` for a break or for networking. +- Do not invent an `id` or a slug. The import derives those in code. + +## Constraints + +- Read no other file. Do not search the internet. Everything you need is in the input. +- Write no file other than the output JSON. +- Your final message is one line: the total number of talks you extracted. + +--- + +## Why this prompt says nothing about the typography + +An earlier version of this prompt had a rule that forbade normalising the typography, so straight +quotes for curly quotes, a short hyphen for a long dash, and three dots for an ellipsis. + +**That rule made the result worse, and we measured it.** It did not stop the model from changing +the apostrophe. It did make the model write `História` where the source has `Història`, which is a +spelling change in Catalan and not a typography change. The plain prompt made no such error in +three runs; the prompt with the rule made it in one run of two. + +Typography is normalised in code instead, by `bin/canon.py`, which runs over every imported +string. That makes the result consistent by construction, and it keeps letters and accents out of +reach. `bin/check-extraction` compares what the model wrote against the normalized source, so a +curly quote turned straight passes and a changed accent fails. + +Measurement and reasoning: `Personal/PyBCN/2026-10-11 Fase 3 - El Archive, propuesta de modelos` +in David's vault, section "Qué modelo hace falta, medido". diff --git a/tests/extraction/README.md b/tests/extraction/README.md new file mode 100644 index 000000000..80ff5eb74 --- /dev/null +++ b/tests/extraction/README.md @@ -0,0 +1,90 @@ +# Extraction fixtures + +What `bin/check-extraction --selftest` reads. Nothing here is built by the site, and +Hugo never looks in this directory. + +## Where this came from + +The Meetup archive import reads event descriptions written by volunteers over 19 years. +The format changed three times, so the import cannot be a single parser: a model reads +each description and returns the talks it announced. That raises two questions this +fixture set answers. Does the extraction keep the source wording? And which model is +enough? + +The files record a measurement made on 2026-10-11. + +| File | What it is | +|---|---| +| `events.json` | 9 events from the Meetup export, 3 per format era. For each year chosen, the event with the median description length, so the sample is not a pick of the easy ones | +| `truth.yaml` | The talks each event announces, read by hand before any model ran | +| `model-runs/` | 7 real extraction outputs: Haiku 4.5 three times, Haiku with an extra prompt rule twice, Sonnet 5, and Opus 5 | +| `expectations.yaml` | What the check has to say about each recorded run | + +## The recorded runs use the old `non_talk_items` key + +`bin/prompts/extract-talks.md` asks for an `agenda`, one object per slot, +`{start, end, label, talk_title}`. The 7 recorded runs predate that name: each one writes +`non_talk_items` instead, and each slot is a bare string with the time still inside it. +`bin/check-extraction` reads both key names and both slot shapes, so it runs on these fixtures +and on a new extraction alike. Do not rewrite the recorded runs into the new shape. They are +evidence of what each model returned, and the whole point of keeping them is that nobody edited +them afterwards. + +## Why the sample is built this way + +Each event tests something that can go wrong: + +- `247592204` is a documentation sprint. It announces no talk, so an empty list is the + right answer and any talk is a wrong one. +- `315383583` has Catalan and Spanish titles, and a real typo in the source, + `simluation`, which the extraction has to keep. +- `16488025` is free prose with no separators and three talks run together. +- `103676402` has a title that is cut short in the source. +- `203258202` gives neither an abstract nor a language, so filling either one in is an + invention. + +## Why a substring is not enough + +The check asks whether an extracted field is a substring of the normalized source. A field that +was cut short is still a substring, so that test on its own passes a model that silently drops +half an abstract, and half a description lost in an archive nobody reads again is the failure the +check exists to prevent. So it also asks whether a description stops in the middle of its own +source line at no sentence end, and reports that as `truncated`, an error. The rule runs on +descriptions only: a title legitimately ends on a letter. All 7 recorded runs have +`truncated_errors: 0`, so none of them cut a description short. + +## Why one recorded run must fail + +`model-runs/haiku-typography-rule-2.json` is a corrupted run, and the check has to +reject it. A check that only ever sees good input proves nothing. + +That run came from a prompt that told the model not to normalize the typography. The +rule did not work: the model still replaced the Catalan curly apostrophe, and it also +wrote one word with an acute accent where the source has a grave one. The first is +acceptable, because `bin/canon.py` normalizes typography anyway. The second is a +spelling change, so no normalizer can forgive it. + +The plain prompt produced no such error in three runs. That is why +`bin/prompts/extract-talks.md` says nothing about typography. + +## The em dash in the fixtures + +`events.json` holds one em dash, in the abstract a speaker wrote for event `303407709` in +2024. It is source data from Meetup and it stays as it is: editing a fixture to suit a +writing convention would corrupt the evidence the fixture exists to hold. `bin/canon.py` +maps it to a plain hyphen, so no published page carries it, and this event doubles as the +live test of that mapping. + +All 7 files in `model-runs/` carry that same em dash once, in the same abstract. Each model +was told to copy the text and it did, so the character is the check passing and not a defect. +A scan of this directory finds the em dash 8 times, in 8 files, and every one of them is +recorded data rather than text anybody here wrote. That is the whole exception: a file we +write, in this directory or any other, still never carries one. + +## Regenerating + +Do not regenerate `truth.yaml` with a model. It is the human reference the benchmark +measures against, and a model-written reference measures nothing. + +`events.json` comes from `data/raw/meetup/python-barcelona/events.json` in the +pybcn-archive export, which is not in this repository. diff --git a/tests/extraction/events.json b/tests/extraction/events.json new file mode 100644 index 000000000..fc7244068 --- /dev/null +++ b/tests/extraction/events.json @@ -0,0 +1,65 @@ +[ + { + "id": "286238841", + "title": "Python June Meetup: Running Serverless & Data Pipelines with Yaetos", + "dateTime": "2022-06-15T18:30:00+02:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/286238841/", + "description": "This month we have two awesome sessions on **Serverless** & **Yaetos**:\n\n***\n\n* ➡️ **What:** Run the extra mile with serverless\n* 📢 **Who**: [Mohammed Bouabid](https://www.linkedin.com/in/mohammed-bouabid-a326b74a/)\n* ⏱ **Duration**: \\~30 minutes\n* 👅 **Language**: English\n* 🧐 **Abstract**: sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.\n\n***\n\n* ➡️ **What:** Creating Python Data Pipelines with Yaetos\n* 📢 **Who**: [Arthur Prevot](https://www.linkedin.com/in/arthurprevot/) (🐦 [@arprevot](https://twitter.com/arprevot))\n* ⏱ **Duration**: \\~30 minutes\n* 👅 **Language**: English\n* 🧐 **Abstract**: The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how [Yaetos](https://github.com/arthurprevot/yaetos) can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.\n\n***\n\n➡️ Only register if you can attend!\n\n* This is a reduced capacity event. Only register if you plan on attending, and if something comes up please free your spot so somebody else can pick it up!\n\n➡️ Location\n\n* sennder at Aticco Verdaguer: [Carrer de Provença, 339, 08037 Barcelona](https://goo.gl/maps/hEeZgF2XvuVmHobt5)\n\n➡️ COVID safe event\n\n* We will ask all attendees to show a vaccination certificate, or have a negative covid test taken in the 24 hours previous to the event.\n* We will have a reduced attendance ratio.\n\n➡️ Access Control\n\n* 🛂 The security access control requires an attendee list. Provide your full name in your profile, otherwise you will be REMOVED from the list.\n* 🔞 People under 18 years old only allowed accompanied by one parent or legal guardian.\n* 🚷 There are limited seats, please BE RESPONSIBLE when signing up. If you can't make it, please free up your seat, so someone else can attend. We keep a list with repeated offenders!\n\n➡️ More\n\n* 📢 We need talk proposals! [Send yours](https://pybcn.org/call-for-proposals)\n* 🦄 Wanna [publish a job offer](https://pybcn.org/job-offers)?\n* 🐦 Follow [@PyBCN](https://twitter.com/pybcn) for pictures, slides, videos and more" + }, + { + "id": "303407709", + "title": "Python Meetup September: AI Goes Back To School", + "dateTime": "2024-09-26T18:30:00+02:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/303407709/", + "description": "For the month of September, we have two amazing talks about **LLMs** and **RAGs**.\n\\-\\-\\-\nSchedule:\n\n* 👋 & 🗣️ 18:30: Welcome and Talks\n* 🍕 **&** 🍻 20:00: Networking, food and drinks!\n* 👋 21:00: Bye!\n\n\\-\\-\\-\n\n* ➡️ **What:** *From text to actions: LLMs as the new software consumers*\n* 📢 **Who**: Pol Alvarez Vecino\n* ⏱ **Duration**: 30 minutes\n* 👅 **Language**: English\n* 🧐 **Abstract**: QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.\n\n\\-\\-\\-\\-\n\n* ➡️ **What:** *Ragtime: An Approach to Retrieval-Augmented Generation*\n* 📢 **Who**: Javier Martinez Alcantara\n* ⏱ **Duration**: 30 minutes\n* 👅 **Language**: English\n* 🧐 **Abstract**: In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.\n\n\\-\\-\\-\\-\n➡️ Location:\nCodurance Offices Barcelona:\n[Carrer de Pallars 99, 4th floor, room 41, Barcelona, 08018](https://maps.app.goo.gl/Z7eb1XyPJZYfLM3h8)\n\\-\\-\\-\\-\n➡️ Access Control\n\n* 🛂 The security access control requires an **attendee list** (full name) and a **means of identification** (DNI, NIE, passport)\n* ⚠️ Make sure you register for the event with your **full name** and **ID**. Otherwise, you will be REMOVED from the attendance list.\n* 🪪 You’ll be asked to provide that ID to access the building.\n* 🚸 People under 18 years old shall be accompanied by a legal guardian\n\n\\-\\-\\-\\-\n➡️ Only register if you can attend!\n\n* 🪑 There are limited seats, so please be responsible when signing up\n* 🚷 If you can't make it, please free up your seat, so someone else can attend. We keep a list with repeated offenders!\n\n\\-\\-\\-\\-\n➡️ More at [PyBCN.org](https://pybcn.org/)\n\n* 📢 We need talk proposals! [Send yours](https://pybcn.org/pybcn_association/propose-a-talk/)\n* 🦄 Want to [publish a job offer](https://pybcn.org/job-offers)?\n* 🐦 Follow [@PyBCN](https://twitter.com/pybcn) for pictures, slides, videos and more" + }, + { + "id": "315383583", + "title": "Python Meetup June: Summer is already here!", + "dateTime": "2026-06-30T18:00:00+02:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/315383583/", + "description": "For this month's Meetup, we are going back to the Cibernàrium at Barcelona Activa, hand in hand with **Mobile World Capital**. We will have two talks: one about **Feedback for programming learners** and another one about a **Python-powered security lab**.\n\n\\-\\-\\-\nSchedule:\n\n* 18:00: Welcome\n* ️ 18:15: Talks\n* **&** 20:00: Networking!\n* 20:45: Bye!\n\n\\-\\-\\-\n\n* ➡️ **What: Feedback immediat per estudiants de programació: Powered by Python**\n* **Who**: Moisès Gómez Girón\n* ⏱ **Duration**: 45 minutes\n* **Language**: Catalan\n* **Abstract**: Presentació d’una eina per a l’ensenyament de la programació, creada per minimitzar el temps entre la resolució de l’exercici i la recepció de la retroacció. L’eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l’aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d’enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l’ús exclusiu de tecnologies gratuïtes.\n\\-\\-\\-\\-\n* ➡️ **What: Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation**\n* **Who**: Mauricio Volcanes\n* ⏱ **Duration**: 45 minutes\n* **Language**: Spanish\n* **Abstract**: En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.\n\n\\-\\-\\-\\-\n➡️ Location: Cibernàrium 22@ - Barcelona Activa. Carrer de Roc Boronat 117, 08018 Barcelona\n\\-\\-\\-\\-\n➡️ Only register if you can attend!\n\n* There are limited seats, so please be responsible when signing up; we will be checking the attendee list in case full capacity is reached.\n* If you can't make it, please free up your seat, so someone else can attend. We keep a list with repeated offenders!\n\n\\-\\-\\-\\-\n➡️ More at https://pybcn.org:\n\n* We want talk proposals! [Send yours](https://pybcn.org/pybcn_association/propose-a-talk/)\n* Want to [publish a job offer](https://pybcn.org/job-offers)?\n* Follow [@PyBCN](https://twitter.com/pybcn), [@PyBCN in BlueSky](https://bsky.app/profile/pybcn.org) for pictures, slides, videos and more" + }, + { + "id": "203258202", + "title": "Python January Meetup", + "dateTime": "2015-01-22T19:00:00+01:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/203258202/", + "description": "If you want to give a talk at the next python Meetup, please submit your ideas through this form: http://pybcn.org/call-for-speakers.html (http://tiny.cc/pybcn-propose-talk)\n\nThe next Python Barcelona Meetup will be held on Thursday, 22th of January 2015 from 7:00pm to 9:00pm.\n\nThe list of confirmed speakers:\n\n- An introduction to SQLAlchemy and ORM in Python by Oriol Andrés (@Oriol_And) - 20 min\n\n- How to open a bar in Python by Edward van Kuik (@AlexanderBarCT) - 20 min\n\n- Continuous Integration on a large open source software: Plone & Plone 5 is here! by Ramon Navarro Bosch (@bloodbare) - Plone Framework Team member - 40 min\n\nThe Meetup will be held at ITNIG - http://itnig.net/\n\nVenue partners:\n\nhttp://photos1.meetupstatic.com/photos/event/7/8/b/d/600_433110909.jpeg" + }, + { + "id": "247592204", + "title": "Pandas Documentation Sprint", + "dateTime": "2018-03-10T10:00:00+01:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/247592204/", + "description": "> What?\n\nWe'll be joining a worldwide event to improve Pandas documentation: https://python-sprints.github.io/pandas/\n\nPandas is an open source Python library for data manipulation and analysis. Its popularity is skyrocketing, and it is becoming the de-facto standard for data science and data engineering. But the number of core developers and contributors did not grow as fast as its popularity, and things like the API documentation would benefit from some help.\n\nWe plan to help leave the API documentation for the Series and DataFrame classes in a perfect state.\n\n> Who?\n\nTo join the sprint, you should have some previous knowledge of Pandas, and bring your own laptop. That's it!\n\nThe organizers will provide help and coaching with Pandas, doing Pull Requests, setting up the environment, or whatever you need.\n\nWe will provide snacks and drinks during the event, including vegan stuff. If you have some food intolerance please let us know in advance so we can prepare.\n\n> How?\n\nJust join the waitlist for this event!\n\nPlease, make sure you answer the question when joining. We need to keep a good balance in terms of experience and diversity, and we'll select attendees manually. We have room for about 25 people." + }, + { + "id": "261650248", + "title": "May PyBCN Practice Sessions: Intro to Image Processing and Computer Vision", + "dateTime": "2019-05-25T11:00:00+02:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/261650248/", + "description": "• What we will do?\nCome to start the journey into image processing & computer vision using PILlow & OpenCV in Python.\nThis workshop will take us into concepts like images, filters, object recognition ...\n• What you need to bring?\nWilling to enjoy and learn, a laptop with Python 3.6 and your preferred virtual environment system installed.\n\n• Requirements:\n* You should know basic Python and basic numpy\n* You should bring your laptop with Python 3.6.\n* Your preferred virtual environment system already installed.\n* OpenCV python package installed.\nFollow this instructions if you don't have opencv already installed:\nhttps://docs.opencv.org/4.1.0/df/d65/tutorial_table_of_content_introduction.html\n\n• Venue\nPunt Multimèdia Espanya Industrial\nCarrer de Muntadas 5 (Casa del Mig) · Barcelona\n\nThis session will be in English or Catalan or Spanish, depending on the audience.\n\n--------------------------------------------------------------------------------\n• Twitter\nFollow @PyBCN (https://twitter.com/pybcn) for pictures, slides and more.\n\n• Call for Proposals\nWe need talk proposals to organize future Meetup events: send your proposal (http://pybcn.org/call-for-proposals/)!\n\n• Relevant information\nPyBCN Practice Sessions are spaces for our community members to learn, have fun, share knowledge, and meet. These sessions are not courses, nor competitions." + }, + { + "id": "9801883", + "title": "Barcelona Python Meetup February 2009", + "dateTime": "2009-02-26T19:00:00+01:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/9801883/", + "description": "The two 30 minute talks for this edition of the Barcelona Python Meetup are:\n\n1. Jordi Molina: Python tips for the perl-minded system administrator\n2. Timo Stollenwerk et al. : The Zope Component Architecture - a Component Architecture for Python\n\nThe meeting is back to the PRBB!\n\nIt would be nice to make the next meeting in a more central place, so if somebody has\nsome contacts to organize the march meetup somewhere else, let us know!" + }, + { + "id": "16488025", + "title": "Barcelona Python Meetup February 2011", + "dateTime": "2011-02-24T20:00:00+01:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/16488025/", + "description": "Introduction to Python - 45 min- Maik Röder Leveraging multiple processors on a given machine with the Python multiprocessing module - 15min - Jørgen Skancke Statistics and Plotting with Python - 30min - Juan González-Vallinas" + }, + { + "id": "103676402", + "title": "Python Web Frameworks", + "dateTime": "2013-03-21T19:00:00+01:00", + "eventUrl": "https://www.meetup.com/python-barcelona/events/103676402/", + "description": "The meetup will take place in the \"Marie Curie\" room on the ground floor of the PRBB.\n\nProgram:\n\n1. Introduction to Django or REST with Tastypie in / Colin Kingswood\n2. Django - Best practices / David Arcos\n3. Plone / Ramon Navarro\n4. Pyramid / Víctor Fernández de Alba\n5. Flask / Areski Belaid\n6. Offline websites (appcache and filesystem api) / Marc Garcia\n7. NodeJs / Guillem serra\n8. Interactive Session / brainstorming / Grzegorz Świerad\n\nEvery presentation is going to take 15 minutes. We will keep it short and interesting :)\n\nAfterwards, let's go out for dinner together.\n\nHope to see you all there!\n\nTwitter: @pybcn and #pybcn\n\nPS: There were more talk proposals than we could fit in the schedule. We are going to use them for another meetup. We could even dedicate one Meetup to Django, and some other frameworks like Plone if there is enough interest." + } +] \ No newline at end of file diff --git a/tests/extraction/expectations.yaml b/tests/extraction/expectations.yaml new file mode 100644 index 000000000..59f84fb38 --- /dev/null +++ b/tests/extraction/expectations.yaml @@ -0,0 +1,137 @@ +# What bin/check-extraction must say about each recorded model run. +# +# These are real outputs from the 2026-10-11 measurement, kept so the check +# itself has a regression test. Six runs must pass and one must fail: a check +# that only ever sees good input proves nothing. +# +# literal: the verdict of the literalness check against the normalized source. +# benchmark: the score against truth.yaml. + +haiku-1: + note: "Haiku 4.5, plain prompt, run 1 of 3. Clean." + literal: + verdict: pass + fields_checked: 29 + identical_to_normalized_source: 29 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 23 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 2 + +haiku-2: + note: "Haiku 4.5, plain prompt, run 2 of 3. Clean." + literal: + verdict: pass + fields_checked: 30 + identical_to_normalized_source: 30 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 23 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 2 + +haiku-3: + note: "Haiku 4.5, plain prompt, run 3 of 3. Clean. It leaves one ambiguous talk out, which is allowed." + literal: + verdict: pass + fields_checked: 28 + identical_to_normalized_source: 28 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 22 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 1 + +haiku-typography-rule-1: + note: "Haiku 4.5 with a prompt rule that forbids changing the typography, run 1 of 2. Clean, but it finds neither ambiguous talk." + literal: + verdict: pass + fields_checked: 27 + identical_to_normalized_source: 27 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 21 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 0 + +haiku-typography-rule-2: + note: "Haiku 4.5 with the same rule, run 2 of 2. THIS RUN MUST FAIL. The rule did not stop it from changing the apostrophe, and it wrote Historia with an acute accent where the source has Historia with a grave accent. That is a spelling change, not typography, so no normalizer can forgive it. This fixture is what proves the check rejects a corrupted run." + literal: + verdict: fail + fields_checked: 29 + identical_to_normalized_source: 28 + spacing_warnings: 0 + accent_or_case_errors: 1 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 22 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 1 + +sonnet-1: + note: "Sonnet 5. Clean. The model the import uses." + literal: + verdict: pass + fields_checked: 30 + identical_to_normalized_source: 30 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 23 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 2 + +opus-1: + note: "Opus 5. Clean. Identical to sonnet-1 in all six fields of every talk + once the Markdown is resolved. Raw, one of the 138 fields differs: an + abstract where Opus wrote a link as its visible text and Sonnet kept the + Markdown. Opus buys nothing over Sonnet here." + literal: + verdict: pass + fields_checked: 30 + identical_to_normalized_source: 30 + spacing_warnings: 0 + accent_or_case_errors: 0 + content_errors: 0 + truncated_errors: 0 + benchmark: + talks: 23 + required_missing: 0 + spurious: 0 + wrong_speakers: 0 + ungrounded_fields: 0 + ambiguous_included: 2 diff --git a/tests/extraction/model-runs/haiku-1.json b/tests/extraction/model-runs/haiku-1.json new file mode 100644 index 000000000..068f9ae63 --- /dev/null +++ b/tests/extraction/model-runs/haiku-1.json @@ -0,0 +1,309 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": [ + "Mohammed Bouabid" + ], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [ + { + "type": "speaker_profile", + "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/" + } + ] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": [ + "Arthur Prevot" + ], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [ + { + "type": "speaker_profile", + "url": "https://www.linkedin.com/in/arthurprevot/" + }, + { + "type": "github", + "url": "https://github.com/arthurprevot/yaetos" + }, + { + "type": "speaker_profile", + "url": "https://twitter.com/arprevot" + } + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": [ + "Pol Alvarez Vecino" + ], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": [ + "Javier Martinez Alcantara" + ], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [ + "Welcome", + "Networking", + "Bye" + ] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": [ + "Moisès Gómez Girón" + ], + "description": "Presentació d'una eina per a l'ensenyament de la programació, creada per minimitzar el temps entre la resolució de l'exercici i la recepció de la retroacció. L'eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l'aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d'enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l'ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": [ + "Mauricio Volcanes" + ], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": [ + "Welcome", + "Networking", + "Bye" + ] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": [ + "Oriol Andrés" + ], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": [ + "Edward van Kuik" + ], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": [ + "Ramon Navarro Bosch" + ], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": null, + "language": null, + "duration_minutes": null, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": [ + "Jordi Molina" + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": [ + "Timo Stollenwerk et al." + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": [ + "Maik Röder" + ], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": [ + "Jørgen Skancke" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": [ + "Juan González-Vallinas" + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": [ + "Colin Kingswood" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": [ + "David Arcos" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": [ + "Ramon Navarro" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": [ + "Víctor Fernández de Alba" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": [ + "Areski Belaid" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": [ + "Marc Garcia" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": [ + "Guillem serra" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Interactive Session / brainstorming", + "speakers": [ + "Grzegorz Świerad" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] \ No newline at end of file diff --git a/tests/extraction/model-runs/haiku-2.json b/tests/extraction/model-runs/haiku-2.json new file mode 100644 index 000000000..f69f93286 --- /dev/null +++ b/tests/extraction/model-runs/haiku-2.json @@ -0,0 +1,239 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": ["Mohammed Bouabid"], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [{"type": "speaker_profile", "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/"}] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": ["Arthur Prevot"], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [{"type": "speaker_profile", "url": "https://www.linkedin.com/in/arthurprevot/"}, {"type": "speaker_profile", "url": "https://twitter.com/arprevot"}, {"type": "github", "url": "https://github.com/arthurprevot/yaetos"}] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": ["Pol Alvarez Vecino"], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": ["Javier Martinez Alcantara"], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": ["Networking, food and drinks", "Bye"] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": ["Moisès Gómez Girón"], + "description": "Presentació d'una eina per a l'ensenyament de la programació, creada per minimitzar el temps entre la resolució de l'exercici i la recepció de la retroacció. L'eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l'aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d'enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l'ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": ["Mauricio Volcanes"], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": ["Welcome", "Networking", "Bye"] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": ["Oriol Andrés"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": ["Edward van Kuik"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": ["Ramon Navarro Bosch"], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": "Come to start the journey into image processing & computer vision using PILlow & OpenCV in Python. This workshop will take us into concepts like images, filters, object recognition ...", + "language": null, + "duration_minutes": null, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": ["Jordi Molina"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": ["Timo Stollenwerk"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": ["Maik Röder"], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": ["Jørgen Skancke"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": ["Juan González-Vallinas"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": ["Colin Kingswood"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": ["David Arcos"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": ["Ramon Navarro"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": ["Víctor Fernández de Alba"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": ["Areski Belaid"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": ["Marc Garcia"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": ["Guillem serra"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Interactive Session / brainstorming", + "speakers": ["Grzegorz Świerad"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] diff --git a/tests/extraction/model-runs/haiku-3.json b/tests/extraction/model-runs/haiku-3.json new file mode 100644 index 000000000..25d3aff35 --- /dev/null +++ b/tests/extraction/model-runs/haiku-3.json @@ -0,0 +1,243 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": ["Mohammed Bouabid"], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/"} + ] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": ["Arthur Prevot"], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/arthurprevot/"}, + {"type": "speaker_profile", "url": "https://twitter.com/arprevot"}, + {"type": "github", "url": "https://github.com/arthurprevot/yaetos"} + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": ["Pol Alvarez Vecino"], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": ["Javier Martinez Alcantara"], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": ["Networking, food and drinks!", "Bye!"] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": ["Moisès Gómez Girón"], + "description": "Presentació d'una eina per a l'ensenyament de la programació, creada per minimitzar el temps entre la resolució de l'exercici i la recepció de la retroacció. L'eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l'aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d'enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l'ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": ["Mauricio Volcanes"], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": ["Welcome", "Networking!", "Bye!"] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": ["Oriol Andrés"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [ + {"type": "speaker_profile", "url": "https://twitter.com/Oriol_And"} + ] + }, + { + "title": "How to open a bar in Python", + "speakers": ["Edward van Kuik"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [ + {"type": "speaker_profile", "url": "https://twitter.com/AlexanderBarCT"} + ] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": ["Ramon Navarro Bosch"], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [ + {"type": "speaker_profile", "url": "https://twitter.com/bloodbare"} + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": null, + "language": null, + "duration_minutes": null, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": ["Jordi Molina"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": ["Timo Stollenwerk et al."], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": ["Maik Röder"], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": ["Jørgen Skancke"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": ["Juan González-Vallinas"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": ["Colin Kingswood"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": ["David Arcos"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": ["Ramon Navarro"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": ["Víctor Fernández de Alba"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": ["Areski Belaid"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": ["Marc Garcia"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": ["Guillem serra"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] diff --git a/tests/extraction/model-runs/haiku-typography-rule-1.json b/tests/extraction/model-runs/haiku-typography-rule-1.json new file mode 100644 index 000000000..78ebf99a2 --- /dev/null +++ b/tests/extraction/model-runs/haiku-typography-rule-1.json @@ -0,0 +1,222 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": ["Mohammed Bouabid"], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [{"type": "speaker_profile", "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/"}] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": ["Arthur Prevot"], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [{"type": "speaker_profile", "url": "https://www.linkedin.com/in/arthurprevot/"}, {"type": "speaker_profile", "url": "https://twitter.com/arprevot"}, {"type": "github", "url": "https://github.com/arthurprevot/yaetos"}] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": ["Pol Alvarez Vecino"], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": ["Javier Martinez Alcantara"], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": ["Welcome and Talks", "Networking, food and drinks!", "Bye!"] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": ["Moisès Gómez Girón"], + "description": "Presentació d'una eina per a l'ensenyament de la programació, creada per minimitzar el temps entre la resolució de l'exercici i la recepció de la retroacció. L'eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l'aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d'enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l'ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": ["Mauricio Volcanes"], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": ["Welcome", "Talks", "Networking!", "Bye!"] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": ["Oriol Andrés"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": ["Edward van Kuik"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": ["Ramon Navarro Bosch"], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": ["Jordi Molina"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": ["Timo Stollenwerk et al."], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": ["Maik Röder"], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": ["Jørgen Skancke"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": ["Juan González-Vallinas"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": ["Colin Kingswood"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": ["David Arcos"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": ["Ramon Navarro"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": ["Víctor Fernández de Alba"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": ["Areski Belaid"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": ["Marc Garcia"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": ["Guillem serra"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] diff --git a/tests/extraction/model-runs/haiku-typography-rule-2.json b/tests/extraction/model-runs/haiku-typography-rule-2.json new file mode 100644 index 000000000..0dc274a95 --- /dev/null +++ b/tests/extraction/model-runs/haiku-typography-rule-2.json @@ -0,0 +1,239 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": ["Mohammed Bouabid"], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/"} + ] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": ["Arthur Prevot"], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/arthurprevot/"}, + {"type": "site", "url": "https://twitter.com/arprevot"}, + {"type": "github", "url": "https://github.com/arthurprevot/yaetos"} + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": ["Pol Alvarez Vecino"], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": ["Javier Martinez Alcantara"], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": ["Networking, food and drinks!", "Bye!"] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": ["Moisès Gómez Girón"], + "description": "Presentació d'una eina per a l'ensenyament de la programació, creada per minimitzar el temps entre la resolució de l'exercici i la recepció de la retroacció. L'eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l'aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d'enganyar el sistema. El resultat és una história de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l'ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": ["Mauricio Volcanes"], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": ["Welcome", "Networking!", "Bye!"] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": ["Oriol Andrés"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": ["Edward van Kuik"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": ["Ramon Navarro Bosch"], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": "Come to start the journey into image processing & computer vision using PILlow & OpenCV in Python. This workshop will take us into concepts like images, filters, object recognition ...", + "language": null, + "duration_minutes": null, + "links": [ + {"type": "site", "url": "https://docs.opencv.org/4.1.0/df/d65/tutorial_table_of_content_introduction.html"} + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": ["Jordi Molina"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": ["Timo Stollenwerk et al."], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": ["Maik Röder"], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": ["Jørgen Skancke"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": ["Juan González-Vallinas"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": ["Colin Kingswood"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": ["David Arcos"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": ["Ramon Navarro"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": ["Víctor Fernández de Alba"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": ["Areski Belaid"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": ["Marc Garcia"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": ["Guillem serra"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": ["Interactive Session / brainstorming"] + } +] diff --git a/tests/extraction/model-runs/opus-1.json b/tests/extraction/model-runs/opus-1.json new file mode 100644 index 000000000..2fa578427 --- /dev/null +++ b/tests/extraction/model-runs/opus-1.json @@ -0,0 +1,254 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": ["Mohammed Bouabid"], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/"} + ] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": ["Arthur Prevot"], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how Yaetos can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [ + {"type": "speaker_profile", "url": "https://www.linkedin.com/in/arthurprevot/"}, + {"type": "speaker_profile", "url": "https://twitter.com/arprevot"}, + {"type": "github", "url": "https://github.com/arthurprevot/yaetos"} + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": ["Pol Alvarez Vecino"], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": ["Javier Martinez Alcantara"], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [ + "18:30: Welcome and Talks", + "20:00: Networking, food and drinks!", + "21:00: Bye!" + ] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": ["Moisès Gómez Girón"], + "description": "Presentació d’una eina per a l’ensenyament de la programació, creada per minimitzar el temps entre la resolució de l’exercici i la recepció de la retroacció. L’eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l’aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d’enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l’ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": ["Mauricio Volcanes"], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": [ + "18:00: Welcome", + "18:15: Talks", + "20:00: Networking!", + "20:45: Bye!" + ] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": ["Oriol Andrés"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": ["Edward van Kuik"], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": ["Ramon Navarro Bosch"], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": "Come to start the journey into image processing & computer vision using PILlow & OpenCV in Python. This workshop will take us into concepts like images, filters, object recognition ...", + "language": null, + "duration_minutes": null, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": ["Jordi Molina"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": ["Timo Stollenwerk et al."], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": ["Maik Röder"], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": ["Jørgen Skancke"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": ["Juan González-Vallinas"], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": ["Colin Kingswood"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": ["David Arcos"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": ["Ramon Navarro"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": ["Víctor Fernández de Alba"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": ["Areski Belaid"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": ["Marc Garcia"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": ["Guillem serra"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Interactive Session / brainstorming", + "speakers": ["Grzegorz Świerad"], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] diff --git a/tests/extraction/model-runs/sonnet-1.json b/tests/extraction/model-runs/sonnet-1.json new file mode 100644 index 000000000..1cbfdf7fb --- /dev/null +++ b/tests/extraction/model-runs/sonnet-1.json @@ -0,0 +1,309 @@ +[ + { + "event_id": "286238841", + "talks": [ + { + "title": "Run the extra mile with serverless", + "speakers": [ + "Mohammed Bouabid" + ], + "description": "sennder is migration from monolith to microservices. We will explain the requirements and architecture that they're working on, and see some live coding on how to consume Kafka events using AWS Lambda.", + "language": "English", + "duration_minutes": 30, + "links": [ + { + "type": "speaker_profile", + "url": "https://www.linkedin.com/in/mohammed-bouabid-a326b74a/" + } + ] + }, + { + "title": "Creating Python Data Pipelines with Yaetos", + "speakers": [ + "Arthur Prevot" + ], + "description": "The talk will describe the challenges of creating python data pipelines and putting them in production in the cloud, and how [Yaetos](https://github.com/arthurprevot/yaetos) can help. It is targeted at data engineers, data scientists, analysts and python enthusiasts. Yaetos is a data framework, simplifying the use of Apache Spark in the AWS cloud, and with implementations in production at Infojobs and The Hotels Network.", + "language": "English", + "duration_minutes": 30, + "links": [ + { + "type": "speaker_profile", + "url": "https://www.linkedin.com/in/arthurprevot/" + }, + { + "type": "speaker_profile", + "url": "https://twitter.com/arprevot" + }, + { + "type": "github", + "url": "https://github.com/arthurprevot/yaetos" + } + ] + } + ], + "non_talk_items": [] + }, + { + "event_id": "303407709", + "talks": [ + { + "title": "From text to actions: LLMs as the new software consumers", + "speakers": [ + "Pol Alvarez Vecino" + ], + "description": "QA bots, copilots, workflow automation - LLMs are evolving to control our software through natural language. The endgame is APIs. LLMs that can convert user intents directly into HTTP requests, like Talk2Apis does today, will have the flexibility to access any service. This creates a machine-first software economy with big implications for developers. Python examples included.", + "language": "English", + "duration_minutes": 30, + "links": [] + }, + { + "title": "Ragtime: An Approach to Retrieval-Augmented Generation", + "speakers": [ + "Javier Martinez Alcantara" + ], + "description": "In this session, we'll explore a pattern that has sparked significant interest in the world of natural language models (LLMs): Retrieval-Augmented Generation (RAG). We'll walk through a practical implementation, dive into its details, and share insights based on our experiences working with these methods. Whether you're a beginner or an expert, we encourage you to join—what matters most is your interest in LLMs, machine learning, and engineering, and how to leverage these technologies to bring value to your company or organization.", + "language": "English", + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [ + "Welcome and Talks", + "Networking, food and drinks!", + "Bye!" + ] + }, + { + "event_id": "315383583", + "talks": [ + { + "title": "Feedback immediat per estudiants de programació: Powered by Python", + "speakers": [ + "Moisès Gómez Girón" + ], + "description": "Presentació d’una eina per a l’ensenyament de la programació, creada per minimitzar el temps entre la resolució de l’exercici i la recepció de la retroacció. L’eina ha estat desenvolupada al llarg dels anys en paral·lel al seu ús a l’aula, sota pressió per terminis ajustats, canvis de temari i estudiants amb imaginació per trobar maneres enginyoses d’enganyar el sistema. El resultat és una història de compromisos i solucions creatives, des de prohibir funcionalitats concretes del llenguatge fins a l’ús exclusiu de tecnologies gratuïtes.", + "language": "Catalan", + "duration_minutes": 45, + "links": [] + }, + { + "title": "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation", + "speakers": [ + "Mauricio Volcanes" + ], + "description": "En esta charla mostraremos un laboratorio completo de ciberseguridad compuesto por máquinas virtuales, Elastic Stack y el framework MITRE Caldera (Python). A través de una demostración guiada, analizaremos cómo simular técnicas del framework MITRE ATT&CK y cómo correlacionarlas en tiempo real con logs de sistemas Linux para detectar comportamiento malicioso.", + "language": "Spanish", + "duration_minutes": 45, + "links": [] + } + ], + "non_talk_items": [ + "Welcome", + "Networking!", + "Bye!" + ] + }, + { + "event_id": "203258202", + "talks": [ + { + "title": "An introduction to SQLAlchemy and ORM in Python", + "speakers": [ + "Oriol Andrés" + ], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "How to open a bar in Python", + "speakers": [ + "Edward van Kuik" + ], + "description": null, + "language": null, + "duration_minutes": 20, + "links": [] + }, + { + "title": "Continuous Integration on a large open source software: Plone & Plone 5 is here!", + "speakers": [ + "Ramon Navarro Bosch" + ], + "description": null, + "language": null, + "duration_minutes": 40, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "247592204", + "talks": [], + "non_talk_items": [] + }, + { + "event_id": "261650248", + "talks": [ + { + "title": "Intro to Image Processing and Computer Vision", + "speakers": [], + "description": "Come to start the journey into image processing & computer vision using PILlow & OpenCV in Python. This workshop will take us into concepts like images, filters, object recognition ...", + "language": null, + "duration_minutes": null, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "9801883", + "talks": [ + { + "title": "Python tips for the perl-minded system administrator", + "speakers": [ + "Jordi Molina" + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + }, + { + "title": "The Zope Component Architecture - a Component Architecture for Python", + "speakers": [ + "Timo Stollenwerk et al." + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "16488025", + "talks": [ + { + "title": "Introduction to Python", + "speakers": [ + "Maik Röder" + ], + "description": null, + "language": null, + "duration_minutes": 45, + "links": [] + }, + { + "title": "Leveraging multiple processors on a given machine with the Python multiprocessing module", + "speakers": [ + "Jørgen Skancke" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Statistics and Plotting with Python", + "speakers": [ + "Juan González-Vallinas" + ], + "description": null, + "language": null, + "duration_minutes": 30, + "links": [] + } + ], + "non_talk_items": [] + }, + { + "event_id": "103676402", + "talks": [ + { + "title": "Introduction to Django or REST with Tastypie in", + "speakers": [ + "Colin Kingswood" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Django - Best practices", + "speakers": [ + "David Arcos" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Plone", + "speakers": [ + "Ramon Navarro" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Pyramid", + "speakers": [ + "Víctor Fernández de Alba" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Flask", + "speakers": [ + "Areski Belaid" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Offline websites (appcache and filesystem api)", + "speakers": [ + "Marc Garcia" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "NodeJs", + "speakers": [ + "Guillem serra" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + }, + { + "title": "Interactive Session / brainstorming", + "speakers": [ + "Grzegorz Świerad" + ], + "description": null, + "language": null, + "duration_minutes": 15, + "links": [] + } + ], + "non_talk_items": [] + } +] diff --git a/tests/extraction/truth.yaml b/tests/extraction/truth.yaml new file mode 100644 index 000000000..f775f0f37 --- /dev/null +++ b/tests/extraction/truth.yaml @@ -0,0 +1,121 @@ +# Ground truth for the extraction benchmark. +# +# Read by hand from the Meetup export before any model ran, so the benchmark +# measures a model against a human reference and not against another model. +# Never regenerate this file with a model. +# +# required: every talk that any correct extraction must find. +# ambiguous: the source does not settle it. Finding it or not is both correct, +# so these count neither as a miss nor as a spurious talk. This file records +# what the source supports, not what we chose to do about it: +# bin/prompts/extract-talks.md settles both of these by convention, so a run +# made with that prompt is expected to include them. The recorded runs under +# model-runs/ predate the convention, which is why they disagree here. +# speakers: the expected speaker names, as the source writes them. + +286238841: + date: 2022-06-15 + event_title: "Python June Meetup: Running Serverless & Data Pipelines with Yaetos" + note: "2021-2026 era. The What/Who/Duration/Language template." + required: + - title: "Run the extra mile with serverless" + speakers: ["Mohammed Bouabid"] + - title: "Creating Python Data Pipelines with Yaetos" + speakers: ["Arthur Prevot"] + ambiguous: [] + +303407709: + date: 2024-09-26 + event_title: "Python Meetup September: AI Goes Back To School" + note: "2021-2026 era. Template plus a schedule with non-talk slots." + required: + - title: "From text to actions: LLMs as the new software consumers" + speakers: ["Pol Alvarez Vecino"] + - title: "Ragtime: An Approach to Retrieval-Augmented Generation" + speakers: ["Javier Martinez Alcantara"] + ambiguous: [] + +315383583: + date: 2026-06-30 + event_title: "Python Meetup June: Summer is already here!" + note: "2021-2026 era. Catalan and Spanish titles, and a real typo in the source: simluation." + required: + - title: "Feedback immediat per estudiants de programació: Powered by Python" + speakers: ["Moisès Gómez Girón"] + - title: "Python-powered Cybersecurity Lab: Caldera, Elastic Stack & ATT&CK simluation" + speakers: ["Mauricio Volcanes"] + ambiguous: [] + +203258202: + date: 2015-01-22 + event_title: "Python January Meetup" + note: "2014-2020 era. A list of 'title by NAME - N min', no abstracts." + required: + - title: "An introduction to SQLAlchemy and ORM in Python" + speakers: ["Oriol Andrés"] + - title: "How to open a bar in Python" + speakers: ["Edward van Kuik"] + - title: "Continuous Integration on a large open source software: Plone & Plone 5 is here!" + speakers: ["Ramon Navarro Bosch"] + ambiguous: [] + +247592204: + date: 2018-03-10 + event_title: "Pandas Documentation Sprint" + note: "2014-2020 era. A documentation sprint. It announces no talk, so zero is the right answer." + required: [] + ambiguous: [] + +261650248: + date: 2019-05-25 + event_title: "May PyBCN Practice Sessions: Intro to Image Processing and Computer Vision" + note: "2014-2020 era. A workshop with no talk title of its own. Ambiguous on purpose." + required: [] + ambiguous: + - title: "Intro to Image Processing and Computer Vision" + +9801883: + date: 2009-02-26 + event_title: "Barcelona Python Meetup February 2009" + note: "2007-2013 era. A numbered list of 'NAME: title'." + required: + - title: "Python tips for the perl-minded system administrator" + speakers: ["Jordi Molina"] + - title: "The Zope Component Architecture - a Component Architecture for Python" + speakers: ["Timo Stollenwerk"] + ambiguous: [] + +16488025: + date: 2011-02-24 + event_title: "Barcelona Python Meetup February 2011" + note: "2007-2013 era. Free prose, no separators, three talks run together. The hardest case." + required: + - title: "Introduction to Python" + speakers: ["Maik Röder"] + - title: "Leveraging multiple processors on a given machine with the Python multiprocessing module" + speakers: ["Jørgen Skancke"] + - title: "Statistics and Plotting with Python" + speakers: ["Juan González-Vallinas"] + ambiguous: [] + +103676402: + date: 2013-03-21 + event_title: "Python Web Frameworks" + note: "2007-2013 era. A numbered grid of 'title / NAME'. One title is cut short in the source." + required: + - title: "Introduction to Django or REST with Tastypie in" + speakers: ["Colin Kingswood"] + - title: "Django - Best practices" + speakers: ["David Arcos"] + - title: "Plone" + speakers: ["Ramon Navarro"] + - title: "Pyramid" + speakers: ["Víctor Fernández de Alba"] + - title: "Flask" + speakers: ["Areski Belaid"] + - title: "Offline websites (appcache and filesystem api)" + speakers: ["Marc Garcia"] + - title: "NodeJs" + speakers: ["Guillem serra"] + ambiguous: + - title: "Interactive Session / brainstorming" From e272f43694d5b742c063397e65a4147297ef2518 Mon Sep 17 00:00:00 2001 From: David Arcos Date: Thu, 15 Oct 2026 09:59:38 +0200 Subject: [PATCH 3/5] Add the one-time import scripts and the channel reader Five scripts under bin/, each reading an export that is not in this repository, so that every published fact of the archive can be traced to a source: - bin/import-meetups writes one page per monthly meetup out of the Meetup export: title, day, year, venue, registrations, source URL, and group. Each field is a copy of one field of the export, with canon() as the only transform. - bin/import-talks writes the talks block of each meetup page from a checked extraction, and resolves a speaker name to a person record by the same name, ignoring accents and case. A name that resolves to nothing stays in speaker_names, which no template reads. - bin/import-videos writes the video of a talk as a link of type video on its talk record, and the recording of a whole session as an entry of session_videos, from one normalized record per video of the channel. - bin/import-x-photos stages the photos PyBCN posted live from one event on X: the files under deterministic names, a photos block to paste into the front matter, and a report for the person who curates the set. A photo the association only shared is listed and never imported. - bin/list-channel reads the YouTube channel with yt-dlp and prints what the inventory does not hold, which is how a new video gets noticed. AGENTS.md says in what order the imports run, that they ran once, and what a re-run against content/ destroys. The bin/ row of its table names the new scripts. README.md indexes the two decisions of bin/import-x-photos: stage for review, never write content/, and never import a retweeted photo. --- AGENTS.md | 28 +- README.md | 4 + bin/import-meetups | 231 +++++++++++ bin/import-talks | 561 ++++++++++++++++++++++++++ bin/import-videos | 827 ++++++++++++++++++++++++++++++++++++++ bin/import-x-photos | 945 ++++++++++++++++++++++++++++++++++++++++++++ bin/list-channel | 243 ++++++++++++ 7 files changed, 2838 insertions(+), 1 deletion(-) create mode 100755 bin/import-meetups create mode 100755 bin/import-talks create mode 100755 bin/import-videos create mode 100755 bin/import-x-photos create mode 100755 bin/list-channel diff --git a/AGENTS.md b/AGENTS.md index 943307efd..6ad0e87a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,32 @@ and checking the output, not by reading the file. **`merge.ff = only`.** Any local merge needs `--no-ff`. A failed merge here reports "Not possible to fast-forward", which is not a conflict. +**The archive imports run in one order and only once.** `bin/import-meetups`, +then `bin/import-talks`, then `bin/import-videos`. `bin/import-talks` replaces +the whole `talks` block of a page and everything after it in the front matter, +so running it after the others throws away what they added: a re-run over +`content/events/meetups/` deletes every video link of the talks and every +`session_videos` block, and puts the speaker credits a human resolved by hand +back in `speaker_names`. Its docstring has the detail, with the figures of the +day it was measured. Count what is at stake today with: + +```sh +grep -h 'type: video' content/events/meetups/*.md | wc -l +grep -l '^session_videos:' content/events/meetups/*.md | wc -l +``` + +Re-run any import with `--dry-run` first, and with `--out` pointing at a copy, +never at `content/`. + +**The `bin/import-*` scripts are one-time archaeology, not maintenance.** They +read an export, the pybcn-archive, that is not in this repository, so that a +published fact can be traced to a source. They ran once, in the order above, +and that is the whole of their job. Adding an event, a talk, or a video means +writing the front matter by hand, as `CONTRIBUTING.md` describes, never +re-running an importer against `content/`, which is the destructive thing the +paragraph above measures. The export is not part of maintaining this site, and +a correction to a published page goes in the page. + **`unsafe=true` in `[markup.goldmark.renderer]`.** Raw HTML in front matter and page bodies renders unescaped. Two content files still carry it, `content/events/pyday_bcn/pyday_bcn_2018.md` and `pyday_bcn_2019.md`, which @@ -151,7 +177,7 @@ plausible but different result. | `themes/pybcn_theme/layouts/partials/` | Most of the interesting template logic | | `static/archives/` | Verbatim mirrors of pre-Hugo sites, copied unprocessed | | `data/` | Tables the templates read: `roles.yaml` maps a heading to a role, `role_emoji.yaml` a role to a glyph | -| `bin/` | Setup and build scripts, plus the gitignored Hugo binary | +| `bin/` | Setup and build scripts, the checks, `bin/list-channel` (names a channel video the inventory lacks; needs `yt-dlp` and the network), the one-time `bin/import-*` scripts, plus the gitignored Hugo binary | | `.hugo-version` | The pinned Hugo version, single source of truth | The checks, which are what a pull request has to pass: diff --git a/README.md b/README.md index 909dc6e0c..864ab2868 100644 --- a/README.md +++ b/README.md @@ -434,6 +434,10 @@ it. The cost is that the reasons are scattered, so this is the index. | MIT for the code, CC BY-SA 4.0 for the content, and nothing for the photographs | [LICENSING.md](LICENSING.md) | | `edition` is the source branch and `master` is the built site | [AGENTS.md](AGENTS.md) | | Raw HTML in content is gated by three checks rather than banned | [CONTRIBUTING.md](CONTRIBUTING.md) and `bin/check-html-safety` | +| Imported text is normalized in code, and the model that extracts it is told nothing about typography | `bin/canon.py` and `bin/prompts/extract-talks.md` | +| An extraction is checked against the normalized source, and one recorded run has to fail | `bin/check-extraction` and [tests/extraction/README.md](tests/extraction/README.md) | +| A photo PyBCN retweeted is never imported, because its copyright is somebody else's | `bin/import-x-photos` and [LICENSING.md](LICENSING.md) | +| An import stages its result for review and never writes `content/` itself | `bin/import-x-photos` | One decision has no file of its own, because it is a decision not to act: diff --git a/bin/import-meetups b/bin/import-meetups new file mode 100755 index 000000000..8aebc0a91 --- /dev/null +++ b/bin/import-meetups @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Write one content file per monthly meetup, from the Meetup export. + +The site publishes the large editions, PyDay BCN and PyDataBCN, and nothing +else. The monthly meetups are the body of the archive: 19 years of events +that exist only in the Meetup export, because the Meetup REST API that +served them is retired (see partials/meetup-events.html). This script turns +each of those records into a page, so the association holds its own archive +and the /events/ list reaches all of it. + +It writes the skeleton and no more: the title, the day, the year, the venue, +the registrations, the source URL, and which group organized the event. +Every one of those is a copy of one field of the export, with canon() from +bin/canon.py as the only transform, so the result is deterministic and the +same input always gives the same file. Nothing here needs a judgement. + +The talks are not in it. Each description announces its sessions in prose +written by a different volunteer over 19 years, which no single parser +reads, so a model extracts them and bin/check-extraction compares every +field against the source. That work is separate and comes after this one. + +The Meetup description is never published. It is the logistics of signing +up: the door policy, "only register if you can attend", the call for +proposals, and a social media footer. An archive records what happened, not +how to register for something that is over. The talks inside it are the part +worth keeping, and they arrive as talks. + +Two rules this import keeps, from AGENTS.md: + + RSVPs are not attendance. attendedCount is zero on almost every event, so + yesCount is the only figure the export gives. It is stored as + `attendance`, which the page has to render as registrations, and it names + nobody. + + No personal data beyond what is already published. eventHosts, the host + names, and the description are all read past and never written out. Only + a count of RSVPs leaves this script. + +Why this import may write content/ while bin/import-x-photos may not: a +photo needs a person to choose it, caption it, and clear its copyright, so +that import stages a set for review. Here there is nothing to choose. Each +field is one field of the export, and the only transform never touches a +letter. --out is still required, so the directory the script writes is +always something a person typed. + +Exit codes: + 0 every file was written + 1 nothing was written, or a name collided + 2 the script could not run: bad arguments, or a missing export + +Usage: + bin/import-meetups EXPORT_DIR --out content/events/meetups + bin/import-meetups EXPORT_DIR --out content/events/meetups --dry-run +""" + +import argparse +import json +import pathlib +import re +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from canon import canon +from slug import slugify + +# The two Meetup groups of the association, and the name the site already +# calls each one by: content/_index.md and content/events/monthly_events.md +# both print "PyBCN" and "PyLadies BCN". The id is the directory of the +# export and the first path segment of every eventUrl. +GROUPS = { + "python-barcelona": "PyBCN", + "pyladies-bcn": "PyLadies BCN", +} + +# The large editions, which the site already publishes under +# content/events/pyday_bcn/, content/events/pydatabcn/ and +# content/events/other_events/. Their Meetup records are skipped here, so no +# day gets two pages. The test is the title, because the export has no field +# for it: any title that says PyDay or PyData matches, 19 of the 285 +# records. Two of the 19 are false positives: the call for papers mentoring +# sessions of PyDataBCN 2017 and PyDay BCN 2019 say the name of the +# conference without being it, and they were days of the association that +# existed nowhere on the site. Their pages under content/events/meetups/ +# were written by hand, each one saying so in its own front matter, and a +# re-run of this import does not recreate them. +BIG_EDITION = re.compile(r"pyda(y|ta)", re.I) + + +def yaml_string(value): + """Return value as a double quoted YAML scalar. + + Double quotes and not plain style: a title carries a colon, a leading + emoji, a quote, an asterisk, and an ampersand, and several of those end + a plain scalar or change its meaning. Only the backslash and the double + quote need escaping inside double quotes. + """ + return '"%s"' % value.replace("\\", "\\\\").replace('"', '\\"') + + +def read_export(export_dir): + """Return the meetup records of both groups, newest last. + + Each record keeps the group it came from. A group directory with no + events.json stops the run: a silent half import would look like a group + with no history. + """ + events = [] + for group_id in sorted(GROUPS): + path = export_dir / group_id / "events.json" + if not path.is_file(): + sys.exit("import-meetups: no %s" % path) + for event in json.loads(path.read_text(encoding="utf-8")): + event["group_id"] = group_id + events.append(event) + events.sort(key=lambda event: (event["dateTime"], event["group_id"])) + return events + + +def venue_of(event): + """Return the venue to store, and the comment that says where it is from. + + An online event has no room, and the export says so twice: eventType is + ONLINE and the venue it carries is the placeholder "Online event". The + stored value is "Online", the word the page should print. + + Seven events of 2013 to 2015 carry no venue at all. The field is left + out rather than guessed, which is the rule the /events/ list already + follows for a fact it does not have: Hacktoberfest 2018 prints no date + because no source gives its day. + + A hybrid event keeps its room. One event is HYBRID, and it names a real + venue, so the place it happened in is known. + """ + if event.get("eventType") == "ONLINE": + return "Online", "The export records this event as ONLINE, so it had no room." + name = canon((event.get("venue") or {}).get("name") or "").strip() + if not name: + return None, None + return name, None + + +def front_matter(event): + """Return the content file of one meetup record, front matter only. + + No body. The Meetup description is the only prose the export carries and + it is not published, so a body would be invented text. + """ + group_id = event["group_id"] + lines = [ + "---", + "# One monthly meetup, imported from data/raw/meetup/%s/events.json in" % group_id, + "# the pybcn-archive export, which is not in this repository, event %s." % event["id"], + "# bin/import-meetups writes this file from that record: a correction goes", + "# to the export or to the script, never here. The Meetup description is", + "# the logistics of signing up and is not published, and the talks it", + "# announces arrive separately. See content/events/meetups/_index.md.", + "title: %s" % yaml_string(canon(event["title"]).strip()), + "date: %s" % event["dateTime"][:10], + "year: %s" % event["dateTime"][:4], + ] + venue, note = venue_of(event) + if note: + lines.append("# %s" % note) + if venue: + lines.append("venue: %s" % yaml_string(venue)) + else: + lines.append("# No venue: the export carries none for this event, and no other") + lines.append("# source gives it, so the page says nothing rather than a guess.") + attendance = (event.get("rsvps") or {}).get("yesCount") or 0 + lines.append("# Who said yes on Meetup, not who came: attendedCount is zero on almost") + lines.append("# every record, so this is the only figure the export gives. It names") + lines.append("# nobody, which is what makes it publishable.") + if attendance: + lines.append("attendance: %d" % attendance) + else: + lines.append("# No attendance: yesCount is zero on this record.") + lines.append("meetup_url: %s" % yaml_string(event["eventUrl"])) + lines.append("group: %s" % yaml_string(GROUPS[group_id])) + lines.append("---") + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser( + description="Write one content file per monthly meetup.") + parser.add_argument("export", type=pathlib.Path, + help="the raw/meetup directory of the pybcn-archive export") + parser.add_argument("--out", type=pathlib.Path, required=True, + help="the directory to write the files in") + parser.add_argument("--dry-run", action="store_true", + help="report what would be written and write nothing") + args = parser.parse_args() + + if not args.export.is_dir(): + sys.exit("import-meetups: no %s" % args.export) + + events = read_export(args.export) + meetups = [e for e in events if not BIG_EDITION.search(e["title"])] + skipped = len(events) - len(meetups) + + files = {} + for event in meetups: + slug = slugify(event["title"]) + name = "%s-%s.md" % (event["dateTime"][:10], slug) if slug else "%s.md" % event["dateTime"][:10] + if name in files: + print("FAIL %s is the name of event %s and of event %s" + % (name, files[name]["id"], event["id"])) + return 1 + files[name] = event + + if not args.dry_run: + args.out.mkdir(parents=True, exist_ok=True) + for name, event in files.items(): + (args.out / name).write_text(front_matter(event), encoding="utf-8") + + online = sum(1 for e in meetups if e.get("eventType") == "ONLINE") + no_venue = sum(1 for e in meetups if venue_of(e) == (None, None)) + attendance = sum((e.get("rsvps") or {}).get("yesCount") or 0 for e in meetups) + print("%s %d files in %s" + % ("would write" if args.dry_run else "wrote", len(files), args.out)) + print("%d large editions skipped, of %d records in the export" + % (skipped, len(events))) + print("%s to %s, %d online, %d with no venue, %d registrations in total" + % (meetups[0]["dateTime"][:10], meetups[-1]["dateTime"][:10], + online, no_venue, attendance)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/import-talks b/bin/import-talks new file mode 100755 index 000000000..8851e1d16 --- /dev/null +++ b/bin/import-talks @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +"""Write the talks of each monthly meetup into the page of that meetup. + +bin/import-meetups wrote 266 of the 268 pages of content/events/meetups/ +out of the Meetup export: the day, the venue, the group, and the +registrations. The other two are the call for papers mentoring sessions, +which that import skips with the large editions and which were written by +hand. +The talks were not in it, because the export announces them in prose +written by a different volunteer over 19 years and no single parser reads +that. A model reads each description instead, and bin/check-extraction +compares every title and every description it returns against the +normalized source, field by field. This script is the last step: it puts a +checked extraction into the pages. + +Run the check first. This script does not re-run it, and it cannot: the +check needs the source events, which are in the export and not in this +repository. An extraction that fails the check must not be imported. + + bin/check-extraction out-a.json a-2021-2026.json + bin/import-talks out-a.json out-b.json out-c.json --out content/events/meetups + +The talks block it writes, and why each field is in it or not: + + id slugify() of the title, the same rule that makes the file + name of a meetup, because this id is an address too: + content/talks/_content.gotmpl builds one page per talk at + /talks///. Unique inside its own file, + which is the scope the adapter and a photo reference need. + Two talks of one meetup with the same title get -2 and -3, + in the order the extraction lists them, so the id of a talk + does not move when the file is written again. + title copied, through canon() + speakers the ids of the content/people/ records the announced names + resolve to, which is the field partials/agenda_entry.html + reads and bin/check-content validates. A name resolves + only to a record whose own name is the same text ignoring + accents, case, and the punctuation slugify() drops. That + is the comparison the 132 names of the thirteen large + editions were resolved with, and it is the whole rule: + this script never matches a surname alone, never matches a + shorter name against a longer one, and never matches a + single word, because AGENTS.md forbids merging two people + on a first name and the existing ids hold bare first + names. A record with draft: true is not a person and is + never matched. + speaker_names the names the announcement wrote that resolve to no + record, and only those. The field is where a name waits: + no template reads it, so the page credits nobody, which is + what AGENTS.md requires until a human verifies the person + and a record exists. A name in speakers is not repeated + here, so the two fields never hold the same person twice. + type the controlled vocabulary of partials/event_types.html + language en, es or ca, and a list of them in that order for a + session given in two, which is what AGENTS.md requires. + The extraction writes the name of the language, because + that is what its source says; the code is this repository's + form, so the map below is here and not in the model output. + duration_minutes an integer, left out when the extraction has none + description copied, through canon(), left out when there is none + + links the addresses of the talk itself, in the form the thirteen + large editions use. The extraction holds 115 links and 63 + of them are written. See LINKS below for what each of the + other 52 is and why it stays out. + title_lang NOT written. It is the language of the title, which is not + the language of the talk (see partials/title_lang.html), + and the extraction does not carry it. A Spanish title with + no title_lang is read by a screen reader with an English + voice, so this is a gap and not a finished job. + +Every string this script writes goes through canon(), the one +normalization of imported text in this repository. The extraction is +already normalized, so this is a second application of an idempotent +function, and it is here so that the rule holds whatever the input did. + +Writing a page again replaces the whole talks block, and with it +everything that follows the block in the front matter: rewrite() cuts the +front matter at the "talks:" key and writes the block at the end, so the +session_videos block bin/import-videos puts after it goes too. Nothing +before the block is touched, and a file whose meetup announced no talk gets +no block at all and no empty list, but everything another tool or a person +added inside or after the block is lost, because the extraction does not +carry it. This script is the first writer of the block and it is not safe +to run after the others. + +Measured on 2026-10-13, in the tree with 89 channel videos, a re-run over +content/events/meetups/ with the three current extractions changes 77 +files and would: + + delete all 65 video links of the talks, the whole output of + bin/import-videos for the monthly meetups, because a link of type video + is not in the extraction, and the 3 session_videos blocks after them. + + put 51 names back in speaker_names, 30 of them distinct, that are + credited in speakers today. They were resolved by hand, against a + person record whose name is spelled longer than the meetup announced + ("Ramon Navarro" against ramon-navarro-bosch), which is a judgement + this script will not make. + +So the order is bin/import-meetups, then this script, then +bin/import-videos, and a re-run of this script means running +bin/import-videos again after it and redoing the hand resolutions. Check +a re-run with --dry-run and a diff against a copy of the directory, never +against content/ itself. + +Exit codes: + 0 every page was written + 1 an event of the extraction has no page, or a page could not be read + 2 the script could not run: bad arguments, or a missing file + +Usage: + bin/import-talks EXTRACTION.json [EXTRACTION.json ...] --out DIR + bin/import-talks EXTRACTION.json --out DIR --people DIR --dry-run +""" + +import argparse +import json +import pathlib +import re +import sys +import unicodedata + +import yaml + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from canon import canon +from slug import slugify + +# The language name the extraction writes, and the code this site stores. +# AGENTS.md: language takes en, es or ca, and a list is written in that +# order, because partials/agenda_entry.html prints the codes in the order +# the list gives them and two orders print the same pair two ways. +LANGUAGES = { + "English": ["en"], + "Spanish": ["es"], + "Catalan": ["ca"], + "English/Spanish": ["en", "es"], + "English/Catalan": ["en", "ca"], +} + +# The id of the meetup inside meetup_url, which is the only field of the +# page that carries it. The comment at the top of the file names the export +# record too, but a comment is not a field. +MEETUP_ID = re.compile(r"/events/(\d+)/?") + +# The default location of the person records, so the usual run needs no +# extra argument. --people overrides it. +PEOPLE_DIR = pathlib.Path(__file__).resolve().parent.parent / "content" / "people" + +# LINKS. The extraction holds 115 links. 63 are written and 52 are not, and +# each of the three groups that stay out is a decision and not an accident. +# +# 44 are type speaker_profile: the social account of the person who gave +# the talk, read out of an event description written years ago. AGENTS.md +# forbids rendering those, in the clearest rule it has, so none of them is +# written. Two of the 44 are not even addresses ("twitter.com/mrswats", +# "x.com@pol_avec"), which is a second reason for the same answer. +# +# 4 are a social account the extraction labelled site or github instead of +# speaker_profile. The label is wrong and the content is the same thing: a +# GitHub profile is a path of one segment, github.com/, and not a +# repository, and a medium.com/@handle is a handle. The rule follows the +# content, so these are listed by hand below and stay out with the other +# 44. The personal web site of a speaker, which the label site also brought +# in twice, is not a handle and is written: it is an address of the talk's +# own material in the same sentence of the source. +SOCIAL_PROFILES = frozenset([ + "https://github.com/vperezb", + "https://github.com/SaxMan96/", + "https://github.com/oscarbc96", + "https://medium.com/@victor.perez.berruezo", +]) + +# 27 of the remaining 71 are http://, which bin/check-rendered rejects on a +# built page, and rightly: an http:// link on an https:// page is a +# downgrade the reader does not choose. They are 26 distinct addresses, and +# each was requested over https on 2026-10-13 with curl -L. 22 answered and +# are written with the https scheme and the same host and path, which is +# the whole change: no redirect target is followed into the stored value, +# because where a site moved to in 2026 is not what the announcement said. +# The four that answered nothing over https are dropped, and they are named +# below so that the next run does not have to find out again. +# +# One of the 22, the afitnerd.com kata post, answers 404 over https and +# over http alike: the host is there and the page is gone. It meets the +# test this table applies, so it is written, and it is the one link here +# worth a second look by a human. +HTTPS_VERIFIED = frozenset([ + "http://aws.amazon.com/", + "http://aws.amazon.com/dynamodb/", + "http://aws.amazon.com/ec2/", + "http://afitnerd.com/2012/11/10/change-kata/", + "http://agilekatas.co.uk/katas/Tamagotchi-Kata", + "http://boto.readthedocs.org/en/latest/", + "http://buildbot.net", + "http://chalice.readthedocs.io", + "http://conda.pydata.org", + "http://docs.python.org/library/re.html", + "http://guillotina.readthedocs.io/en/latest/", + "http://hadoop.apache.org/", + "http://httptoolkit.tech/", + "http://legacy.python.org/dev/peps/pep-0020/", + "http://pandas.pydata.org/", + "http://vagrantup.com/", + "http://webpy.org/", + "http://www.ansible.com", + "http://www.fabfile.org/", + "http://www.froscon.de/", + "http://www.mongodb.org/", + "http://www.nltk.org/", +]) + +# The four http:// addresses with no https at all, and what the request +# said. A host that fails the TLS handshake or never answers on 443 cannot +# be rewritten: the link would be dead, and a dead link is worse than a +# missing one. +HTTPS_NO_ANSWER = { + "http://codekata.com/kata/kata06-anagrams/": "connection reset on 443", + "http://flask.pocoo.org/": "certificate does not cover the host", + "http://iamnotmyself.com/2011/02/13/refactor-this-the-gilded-rose-kata/": + "no answer on 443", + "http://python3porting.com/": "nothing listening on 443", +} + +# The comment that says where the block comes from, as every other block of +# these files does. %s is the Meetup event id. +BLOCK_COMMENT = """\ +# The talks this meetup announced, read from the description of event %s in +# the pybcn-archive export by a model and compared field by field against +# that description by bin/check-extraction. bin/import-talks writes this +# block: a correction goes to the extraction or to the script, never here. +# speakers holds the content/people/ record the announced name resolves to, +# by the same name, ignoring accents and case. speaker_names holds a name +# that resolves to no record: no template reads it, so the page credits +# nobody until a human verifies the person (AGENTS.md).""" + + +def yaml_string(value): + """Return value as a double quoted YAML scalar, on one line. + + The same escaping bin/import-meetups writes, plus the newline: a talk + description holds paragraphs, and the published programmes write those + as \\n inside a one line scalar. Only the backslash, the double quote, + the newline, and the tab need escaping inside double quotes. + """ + body = (value.replace("\\", "\\\\").replace('"', '\\"') + .replace("\n", "\\n").replace("\t", "\\t")) + return '"%s"' % body + + +def fold(name): + """Return name with its accents, case, and punctuation set aside. + + slugify() already strips the accents, lowercases, and turns everything + that is not a letter or a digit into a hyphen, which is exactly the + comparison the thirteen large editions resolved their 132 names with: + "Carles Barrobes" and "Carles Barrobés" are one person, and so are + "Laura Perez-Mayos" and "Laura Pérez Mayos", because a hyphen and a + space are the same separator in a name written by two volunteers. + + It is not a latinizing transform of stored text: nothing this returns + is ever written to a page. canon() is the rule for what is stored, and + it never touches a letter. This function only compares. + """ + return slugify(unicodedata.normalize("NFC", name)) + + +def person_index(people_dir): + """Return the id of each person record, keyed by the folded name. + + Keyed by the folded id as well, because an id is the folded name of the + person in every record of this repository and a source that writes the + name with a middle name dropped still has to miss, not guess. + + A record with draft: true is left out: content/people/canary.md is a + fixture for bin/check-rendered and not a person, and crediting a talk + to it would publish the fixture. + + A key two records share is dropped from the index and reported, because + a name that matches two people resolves to neither. That cannot happen + today, and the day a second Jordi Mas gets a record it must fail here + and not pick one of them. + """ + if not people_dir.is_dir(): + sys.exit("import-talks: no %s" % people_dir) + index, clashes = {}, {} + for path in sorted(people_dir.glob("*.md")): + if path.name == "_index.md": + continue + text = path.read_text(encoding="utf-8") + found = re.match(r"^---\n(.*?)\n---", text, re.S) + if not found: + sys.exit("import-talks: %s has no front matter" % path) + data = yaml.safe_load(found.group(1)) or {} + if data.get("draft"): + continue + person_id, name = data.get("id"), data.get("name") + if not person_id or not name: + sys.exit("import-talks: %s has no id or no name" % path) + for key in {fold(name), fold(person_id)}: + if key in index and index[key] != person_id: + clashes.setdefault(key, {index[key]}).add(person_id) + index[key] = person_id + for key, ids in sorted(clashes.items()): + print("WARN '%s' is the name of %s, so a source that writes it " + "resolves to nobody" % (key, " and ".join(sorted(ids)))) + index.pop(key, None) + return index + + +def resolve_speakers(names, index): + """Return (ids, unresolved) for the announced names of one talk. + + A name of one word never resolves. AGENTS.md: never merge two people on + a first-name match, and the ids of this site include bare first names. + The source writes "David", "Nuria", "Loris", "Areski", and "Garito", + and not one of those five says which person it means. + + A name the index does not hold stays in unresolved, in the text the + announcement used. Order is kept on both lists, and a name that repeats + inside one talk is written once. + """ + ids, unresolved = [], [] + for name in names: + key = fold(name) + person_id = index.get(key) if key.count("-") else None + if person_id: + if person_id not in ids: + ids.append(person_id) + elif name not in unresolved: + unresolved.append(name) + return ids, unresolved + + +def talk_links(talk): + """Return the links of one talk that reach the page, in source order. + + See LINKS at the top of the file for the three groups that do not, and + for what was measured about each. An http:// address that is in + HTTPS_VERIFIED is written with the https scheme and nothing else + changed; one that is in neither table stops the run, because an + unmeasured address must not be guessed either way. + """ + kept = [] + for link in talk.get("links") or []: + url, kind = link.get("url") or "", link.get("type") or "" + if kind == "speaker_profile" or url in SOCIAL_PROFILES: + continue + if url.startswith("http://"): + if url in HTTPS_NO_ANSWER: + continue + if url not in HTTPS_VERIFIED: + sys.exit("import-talks: %s is an http:// address that neither " + "HTTPS_VERIFIED nor HTTPS_NO_ANSWER names. Request " + "it over https and put it in one of them." % url) + url = "https://" + url[len("http://"):] + elif not url.startswith("https://"): + sys.exit("import-talks: %r is not an address, and only the two " + "speaker_profile entries were. Say what it is." % url) + if not kind: + sys.exit("import-talks: %s has no type, and bin/check-content " + "rejects a link without one." % url) + if (kind, url) not in kept: + kept.append((kind, url)) + return kept + + +def pages_by_id(out_dir): + """Return the page of each meetup, keyed by the Meetup event id. + + A directory with no page stops the run before anything is written: an + id that matches nothing is either the wrong extraction or the wrong + directory, and both of those have to be seen and not half applied. + """ + pages = {} + for path in sorted(out_dir.glob("*.md")): + if path.name == "_index.md": + continue + found = MEETUP_ID.search(path.read_text(encoding="utf-8")) + if not found: + sys.exit("import-talks: %s carries no meetup_url" % path) + event_id = found.group(1) + if event_id in pages: + sys.exit("import-talks: event %s is on %s and on %s" + % (event_id, pages[event_id].name, path.name)) + pages[event_id] = path + if not pages: + sys.exit("import-talks: no page in %s" % out_dir) + return pages + + +def talk_ids(talks): + """Return one id per talk, unique inside this page. + + A title that slugifies to nothing, which an all emoji title does, falls + back to its position, so every talk gets a page. A repeated slug takes + a counter in the order the extraction lists the talks, so the id is a + function of the extraction and does not move between two runs. + """ + ids, used = [], {} + for position, talk in enumerate(talks, start=1): + base = slugify(canon(talk.get("title") or "")) or "talk-%d" % position + used[base] = used.get(base, 0) + 1 + ids.append(base if used[base] == 1 else "%s-%d" % (base, used[base])) + return ids + + +def talk_block(event_id, talks, people, tally): + """Return the talks block of one page, comment included, as lines. + + people is the index of person_index(). tally counts what the run + resolved, so main() can report it: this is the one place that sees + every name and every link of every talk. + """ + lines = BLOCK_COMMENT.split("\n") + lines = [line % event_id if "%s" in line else line for line in lines] + lines.append("talks:") + for talk_id, talk in zip(talk_ids(talks), talks): + lines.append(" - id: %s" % talk_id) + lines.append(" title: %s" % yaml_string(canon(talk["title"]).strip())) + names = [canon(name).strip() for name in talk.get("speakers") or []] + ids, unresolved = resolve_speakers(names, people) + tally["names"] += len(names) + tally["resolved"] += len(names) - len(unresolved) + tally["unresolved"].update(unresolved) + tally["credited"].update(ids) + if ids: + lines.append(" speakers: [%s]" % ", ".join(ids)) + if unresolved: + lines.append(" speaker_names: [%s]" + % ", ".join(yaml_string(name) for name in unresolved)) + lines.append(" type: %s" % talk["type"]) + language = LANGUAGES.get(talk.get("language")) + if language: + lines.append(" language: %s" + % (language[0] if len(language) == 1 else + "[%s]" % ", ".join(language))) + elif talk.get("language"): + sys.exit("import-talks: event %s talk '%s' has language %r, which " + "is not one of %s" % (event_id, talk_id, talk["language"], + ", ".join(sorted(LANGUAGES)))) + if talk.get("duration_minutes") is not None: + lines.append(" duration_minutes: %d" % int(talk["duration_minutes"])) + description = canon(talk.get("description") or "").strip() + if description: + lines.append(" description: %s" % yaml_string(description)) + links = talk_links(talk) + tally["links"] += len(links) + if links: + lines.append(" links:") + for kind, url in links: + lines.append(" - type: %s" % kind) + lines.append(" url: %s" % yaml_string(url)) + return lines + + +def rewrite(text, block): + """Return the page text with block as its talks block. + + The block goes at the end of the front matter, which is where the two + blocks these pages are waiting for belong: the front matter is the page + (see layouts/_default/meetup.html) and the body is empty on purpose. + An earlier block of this script, and the comment above it, is replaced, + so a second run writes the same file and not a second block. + """ + lines = text.split("\n") + if lines[0].strip() != "---": + return None + try: + close = lines.index("---", 1) + except ValueError: + return None + + head = lines[:close] + for position, line in enumerate(head): + if line.rstrip() == "talks:": + # Back up over the comment that introduces the block, so the + # comment is replaced with it and does not pile up. + start = position + while start and head[start - 1].startswith("#"): + start -= 1 + head = head[:start] + break + + return "\n".join(head + list(block) + lines[close:]) + + +def main(): + parser = argparse.ArgumentParser( + description="Write the talks of each meetup into the page of that meetup.") + parser.add_argument("extraction", nargs="+", type=pathlib.Path, + help="the checked extraction JSON files") + parser.add_argument("--out", type=pathlib.Path, required=True, + help="the directory of the meetup pages") + parser.add_argument("--people", type=pathlib.Path, default=PEOPLE_DIR, + help="the directory of the person records " + "(default: content/people)") + parser.add_argument("--dry-run", action="store_true", + help="report what would be written and write nothing") + args = parser.parse_args() + + if not args.out.is_dir(): + sys.exit("import-talks: no %s" % args.out) + for path in args.extraction: + if not path.is_file(): + sys.exit("import-talks: no %s" % path) + + pages = pages_by_id(args.out) + entries = [] + for path in args.extraction: + entries += json.loads(path.read_text(encoding="utf-8")) + + missing = [str(entry.get("event_id")) for entry in entries + if str(entry.get("event_id")) not in pages] + if missing: + print("FAIL %d event(s) of the extraction have no page in %s: %s" + % (len(missing), args.out, ", ".join(missing))) + return 1 + + people = person_index(args.people) + tally = {"names": 0, "resolved": 0, "links": 0, + "unresolved": set(), "credited": set()} + written, talks, empty, disambiguated = 0, 0, 0, 0 + for entry in entries: + event_id = str(entry["event_id"]) + path = pages[event_id] + found = entry.get("talks") or [] + if not found: + empty += 1 + continue + bases = [slugify(canon(talk.get("title") or "")) for talk in found] + disambiguated += len(bases) - len(set(bases)) + text = path.read_text(encoding="utf-8") + new = rewrite(text, talk_block(event_id, found, people, tally)) + if new is None: + print("FAIL %s has no front matter to write into" % path) + return 1 + if not args.dry_run and new != text: + path.write_text(new, encoding="utf-8") + written += 1 + talks += len(found) + + print("%s %d talks into %d of the %d pages in %s" + % ("would write" if args.dry_run else "wrote", talks, written, + len(pages), args.out)) + print("%d events announced no talk and get no talks block" % empty) + print("%d talk id(s) needed a counter to stay unique inside their page" + % disambiguated) + print("%d of %d announced names resolve to %d of the %d person records; " + "%d names, %d of them distinct, wait in speaker_names" + % (tally["resolved"], tally["names"], len(tally["credited"]), + len(set(people.values())), + tally["names"] - tally["resolved"], len(tally["unresolved"]))) + print("%d links written of the 115 in the extraction" % tally["links"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/import-videos b/bin/import-videos new file mode 100755 index 000000000..959bbc530 --- /dev/null +++ b/bin/import-videos @@ -0,0 +1,827 @@ +#!/usr/bin/env python3 +"""Write the video of a talk into the page of the event that held it. + +The PyBCN YouTube channel holds 89 videos. The export enumerates the +channel itself, its videos and streams tabs, and then its 14 playlists: a +video of a monthly meetup sits in the by-year playlist and in "PyBCN Meetup +Videos" as well, six videos sit in no playlist at all, and two unlisted +ones show in no tab and only a playlist reveals them. So this script reads +one normalized record per video and never a playlist or a tab count. + +A video belongs to a talk and not to a day. Each one is written as a link of +type video on the talk record of the event front matter, the form the +thirteen large editions already use, which gives the talk page its Video +section (layouts/talks/single.html) and the row of the talk on the event +page its Watch link (layouts/_default/meetup.html, partials/agenda.html). +The player is on the talk page and on no other: a youtube-nocookie.com +frame, which sets no cookie until the reader presses play, but which still +asks Google for the player when the page loads, so Google sees the IP +address of the reader of that one page. An event page or a listing prints +the link and no frame, so that a page of 19 videos makes no request to +Google before a reader asks for one. The privacy policy of the site says +what the frame does. + +Three recordings of the channel are of a whole session and not of one +talk: a block of lightning talks, the introduction of an evening, and the +live stream of a whole evening. No talk title can ever match them, and the +export says so, with attaches_to "meetup", "session_level" in flags, and +meetup_id naming the event, which the export picked out of the candidates +of that month by series (the monthly meetup, not the coding dojo or the +practice session held days earlier), or which a person declared (see +below). Such a video is written as an entry of an event-level +session_videos block, title and url, which partials/session_videos.html +prints under a Recordings heading after the talks. The script writes it +only when the +page meetup_id names exists and meetup_id is one of the candidates the +export considered; a record that fails either test is reported and left +out. The block is not called videos: Hugo's internal opengraph template +reads .Params.videos as a list of URL strings for og:video, and a list of +records under that key stops the build of the page. + +Three records of the export are declared, not parsed. The title grammar of +the export reads the day, the talk, and the speaker out of a video title, +and three titles defeat it: a short name an uploader wrote for a long +title ("PySpark and Arrow"), a live stream with no date in its title, and +a live stream of a whole evening whose title names no talk. For each one a +person read the programme of its day and wrote the facts by id in +tools/parse_youtube.py of the pybcn-archive: the title the programme uses, +the event, whether the video is of one talk or of the whole session, and +the reason. The record then carries "declared" in flags and in match, and +declared_reason, with no candidates, because the person named the event +and the grammar never got to choose. The export is the right place for +that decision: it is where the raw title and the evidence sit side by side, +where a reader of this site can trace a published fact back to its source, +and a decision written there survives a rebuild of the export, which a +hand edit of a content file does not. This script treats a declared record +as what it is, evidence a human recorded: it looks the declared title up +on the one page the record names, by name and not by fuzzy match, and +writes the link on that talk, or the entry in session_videos when the +record attaches to the meetup. Every declared placement is reported in its +own group with its reason, the way the PROGRAMME group reports a video +placed on another day, so a human always sees them. A declared record that +names no page of this tree, or a title that names no talk of that page, is +reported and left out. + +How a video finds its talk, and what stops a wrong match: + + the title bin/titlematch.py, the rule bin/import-x-photos matches a + talk title inside a tweet caption with, both_ways because the + uploader cuts a long title and writes "[...]" where the cut + is, so the YouTube title can be the shorter of the two. The + length guard TITLE_MINIMUM is what keeps a short title from + matching unrelated words. + the day the title of a video carries the day of the meetup, as a + month ("PyBCN Meetup 2019/03") or as a full date. The talks + of an event on that day are the first candidates. When none + of them matches and exactly one talk of the whole archive + does, the video goes on that talk, on the day of that other + event: the month in a title is a label the uploader typed + weeks after the day, and a title that matches one talk of + the whole archive is a fact of the programme, so the + programme wins. Measured on 2026-10-13, six of the 83 videos + are placed that way, and each one is named in its own group + of the report, with the day the title claims and the event + that announced the talk, so a human always sees them. The + risk this accepts: a talk given twice, where the archive + recorded only one of the two occurrences, would take the + video of the other one. No case of that exists in the current + data. The uniqueness guard below is what keeps the rule from + reaching further: a title that matches talks of two other + days is still reported and not written. + uniqueness two candidate talks that match one video are reported and + neither is written. A video on the wrong talk is worse than a + video that is missing, because the page would state it and + nothing on it would say the match was a guess. + +Everything the run could not place with those facts is printed and left +out, with what is known about it: a video whose title matches no talk of +the day, a video whose day holds no talk record at all, a video with no day +in its title, and a session recording whose event the export does not +name or this tree does not hold. A human decides each one. + +One record of the export is not a failure of the matching. The channel +holds a presentation of the Barcelona community given at PyCon ES 2020, +which is not a talk of any PyBCN event, and the export marks it as such: +"out_of_scope" in flags, with an out_of_scope_reason a person wrote. The +run reports it on its own line, with that reason, and places it nowhere. +That is correct and expected, not an error, so it does not fail the run. + +Exit codes: + 0 every video was placed, or was already in place + 1 a video could not be placed, or a page could not be written + 2 the script could not run: bad arguments, or a missing file + +Usage: + bin/import-videos VIDEOS.json [--out content/events] [--dry-run] + +VIDEOS.json is data/normalized/videos.json of the pybcn-archive export, +which is not in this repository. One record per video, with the title of the +talk, the day read from the video title, the URL, and the id of the video, +which the report names it by and which falls back to the URL. +""" + +import argparse +import datetime +import json +import pathlib +import re +import sys + +import yaml + +# canon.py and titlematch.py sit next to this script and are not installed +# packages. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from canon import canon # noqa: E402 +from titlematch import fold, match_title # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +# The default location of the event pages, so the usual run needs no extra +# argument. +EVENTS_DIR = ROOT / "content" / "events" + +# The YAML front matter of a Hugo page, the same shape bin/check-content and +# bin/import-x-photos read: the first block between two "---" lines. +FRONT_MATTER = re.compile(r"\A---\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.S) + +# The talks block is written in two indentations. The thirteen large +# editions and 189 meetup pages put the dash of a record two spaces in from +# the "talks:" key, a field four in, the dash of a link six, and a field of +# the link eight. The 35 pages that a YAML dump wrote put the dash of a +# record under the key, so a field sits two in, the dash of a link two as +# well, and a field of the link four. Both parse to the same records, so +# the offset of a dash from its key is read from the first record of the +# block and the link is written the same way. A guess that comes out wrong +# is caught before the write, because the edited text has to parse back to +# the same front matter plus one link (see unchanged_apart_from). +TALKS_KEY = re.compile(r"^talks:[ \t]*$") +RECORD_LINE = re.compile(r"^( *)- id: (.*?)[ \t]*$") + +# The event-level session_videos block, for a recording of a whole session. +# Written in the indentation of the large editions, two spaces to the dash, +# four to a field, whatever the talks block of the same page uses: the two +# are separate blocks and YAML reads each one by its own indentation. The +# key is not "videos", which Hugo reserves for og:video (see the docstring). +VIDEOS_KEY = re.compile(r"^session_videos:[ \t]*$") +VIDEOS_COMMENT = [ + "# A recording of a whole session of this meetup and not of one talk: a", + "# block of lightning talks, or the introduction of the evening, has no", + "# talk record for a video to sit on, so the video sits on the event.", + "# bin/import-videos writes this block from the YouTube export of the", + "# pybcn-archive: a correction goes to the export or to the script, never", + "# here. The page prints these under a Recordings heading, after the talks", + "# (partials/session_videos.html). Not \"videos\": Hugo reads that key", + "# as a list of og:video URLs, and a list of records there stops the build.", +] +VIDEO_TITLE = " - title: %s" +VIDEO_URL = ' url: "%s"' + + +def fail(message): + """Stop the run with exit code 2: the script cannot do its job.""" + sys.exit("import-videos: %s" % message) + + +def plural(count, noun): + """Return "1 video" or "4 videos": a count a person reads out loud.""" + return "%d %s" % (count, noun if count == 1 else noun + "s") + + +def read_events(events_dir): + """Return one record per event page, with the talks of each. + + A page with no front matter, and a talk with no id or no title, is + skipped: bin/check-content is what reports those, and this script only + needs the records it can address. + """ + events = [] + for path in sorted(events_dir.rglob("*.md")): + match = FRONT_MATTER.match(path.read_text(encoding="utf-8")) + if not match: + continue + try: + data = yaml.safe_load(match.group(1)) or {} + except yaml.YAMLError as error: + fail("cannot parse the front matter of %s: %s" % (path, error)) + day = data.get("date") + if isinstance(day, datetime.datetime): + day = day.date() + if not isinstance(day, datetime.date): + day = None + talks, held = [], {} + for record in data.get("talks") or []: + if not isinstance(record, dict): + continue + talk_id, title = record.get("id"), record.get("title") + if not talk_id or not title: + continue + videos = videos_of(record) + held.update({url: "talk '%s'" % talk_id for url in videos}) + talks.append({"id": talk_id, "title": title, + "folded": fold(title), "videos": videos}) + # A session with no talk record of its own carries its video on the + # row of the grid, which is where the welcome of PyDay BCN 2020 + # holds one (see partials/agenda_entry.html). No video is ever + # written there by this script, because a monthly meetup has no + # grid, but a video already in place has to be recognised wherever + # it sits or a second copy of it would be written. + for slot in data.get("agenda") or []: + if not isinstance(slot, dict) or slot.get("talk"): + continue + label = slot.get("label") or slot.get("type") or "a row of the grid" + held.update({url: "the grid row '%s'" % label for url in videos_of(slot)}) + # The recording of a whole session, which this script writes on the + # event (see the docstring), has to be recognised on a second run. + for entry in data.get("session_videos") or []: + if isinstance(entry, dict) and entry.get("url"): + held[entry["url"]] = "the session_videos block of the event" + events.append({"path": path, "name": path.stem, "day": day, + "talks": talks, "held": held}) + return events + + +def videos_of(record): + """Return the video URLs of one talk or one row of the grid.""" + return [link.get("url") for link in record.get("links") or [] + if isinstance(link, dict) and link.get("type") == "video"] + + +def day_agrees(video, day): + """Return True when an event day is the day the video title names. + + A full date in the title has to be that date. A month has to be that + month. A video whose title carries no day at all is matched on the year + alone, which is the only fact left, and the uniqueness guard is then + what decides whether it can be placed. + """ + if day is None: + return False + if video.get("date"): + return str(day) == video["date"] + if video.get("year") and video.get("month"): + return day.year == video["year"] and day.month == video["month"] + if video.get("year"): + return day.year == video["year"] + return False + + +def candidates_of(events, video, on_the_day): + """Return the talks to match one video against, as titlematch pairs. + + on_the_day picks the two uses: the talks of the day the title names, + which is where a video may be placed, and the talks of every event, + which is how a title that matches another day gets reported instead of + silently lost. + """ + pairs, where = [], {} + for event in events: + if on_the_day and not day_agrees(video, event["day"]): + continue + for talk in event["talks"]: + key = (event["name"], talk["id"]) + pairs.append((key, talk["folded"])) + where[key] = (event, talk) + return pairs, where + + +def classify(video, events): + """Return (state, detail) for one video. + + The states, and what each one means for the page: + + placed one talk of the day matches, and the page gets the link + already a page already carries this URL, so nothing is written + other_video the matched talk carries another video, for a human to + compare + ambiguous two talks of the day match, so neither is written + off_day no talk of the day matches, but exactly one talk of + another day does, so the programme wins over the label + in the title and the page of that other day gets the + link + other_day no talk of the day matches, and talks of two other days + do, so the programme cannot say which one and neither + is written + no_talk no talk anywhere matches this title + out_of_scope the export says this video belongs to no PyBCN event, + so it is never placed, and that is not an error + session a recording of a whole session, which the export + attaches to an event by meetup_id, and that event page + gets an entry of its session_videos block + no_session a recording of a whole session whose meetup_id is + missing, is not one of the candidates of the export, + or names no page of this tree, so nothing is written + declared a person declared the placement in the export, by id + and with a reason, and the page the record names + holds the talk it names, or gets an entry of its + session_videos block when the record attaches to the + meetup + no_declared a declared record that names no page of this tree, or + a title that names no talk of that page, so nothing is + written + """ + # A URL already on a page is in place, wherever on that page it sits. + # This is what makes a second run of the script write nothing, and what + # keeps the twenty videos of PyDay BCN 2020 where a human put them. + for event in events: + if video["url"] in event["held"]: + return "already", (event, event["held"][video["url"]]) + + flags = video.get("flags") or [] + if "out_of_scope" in flags: + return "out_of_scope", video.get("out_of_scope_reason") or "no reason given" + if "declared" in flags: + return classify_declared(video, events) + if "session_level" in flags and video.get("attaches_to") == "meetup": + return classify_session(video, events) + + title = video.get("talk_title") or "" + pairs, where = candidates_of(events, video, on_the_day=True) + key, hits = match_title(title, pairs, both_ways=True) + if key: + event, talk = where[key] + if talk["videos"]: + return "other_video", (event, talk) + return "placed", (event, talk) + if hits: + return "ambiguous", [where[hit] for hit in hits] + + pairs, where = candidates_of(events, video, on_the_day=False) + key, hits = match_title(title, pairs, both_ways=True) + if key: + event, talk = where[key] + if talk["videos"]: + return "other_video", (event, talk) + return "off_day", (event, talk) + if hits: + return "other_day", [where[hit] for hit in hits] + return "no_talk", [event for event in events if day_agrees(video, event["day"])] + + +def classify_session(video, events): + """Return (state, detail) for the recording of a whole session. + + The export names the event in meetup_id, chosen among the candidates of + the month by series. Three facts have to hold before anything is + written: meetup_id is there, it is one of those candidates (so the + export did not name an event it never considered), and this tree holds + a page of that name whose day agrees with the title. The first two are + read from the export, the third from the pages. + """ + meetup_id = video.get("meetup_id") + if not meetup_id: + return "no_session", "the export names no meetup_id" + candidates = video.get("candidates") or [] + if meetup_id not in candidates: + return "no_session", ("meetup_id %r is not one of the candidates of the " + "export: %s" % (meetup_id, ", ".join(candidates) or "none")) + event = next((event for event in events if event["name"] == meetup_id), None) + if event is None: + return "no_session", "no page of content/events is named %r" % meetup_id + if not day_agrees(video, event["day"]): + return "no_session", ("the page %s is dated %s, and the title says %s" + % (meetup_id, event["day"], label_of(video))) + return "session", event + + +def classify_declared(video, events): + """Return (state, detail) for a record a person declared in the export. + + The declaration names the event in meetup_id and the talk by the title + the programme uses (see the docstring). The record has no candidates, + because the person chose and the grammar did not, so the tests are: + meetup_id names a page of this tree, the day of that page agrees with + the record, and, for a video of one talk, exactly one talk of that page + carries the declared title. The title is compared folded, the way the + archive compares every title, but never matched partially: a declared + title is the name of the talk, not a label an uploader cut short. The + detail is (event, talk), with talk None for a recording of the whole + session, which goes in session_videos. + """ + meetup_id = video.get("meetup_id") + if not meetup_id: + return "no_declared", "the declared record names no meetup_id" + event = next((event for event in events if event["name"] == meetup_id), None) + if event is None: + return "no_declared", "no page of content/events is named %r" % meetup_id + if not day_agrees(video, event["day"]): + return "no_declared", ("the page %s is dated %s, and the record says %s" + % (meetup_id, event["day"], label_of(video))) + if video.get("attaches_to") == "meetup": + return "declared", (event, None) + folded = fold(video["talk_title"]) + found = [talk for talk in event["talks"] if talk["folded"] == folded] + if not found: + return "no_declared", ("no talk of %s is titled %r" + % (meetup_id, video["talk_title"])) + if len(found) > 1: + return "no_declared", ("%d talks of %s are titled %r: %s" + % (len(found), meetup_id, video["talk_title"], + ", ".join(talk["id"] for talk in found))) + if found[0]["videos"]: + return "other_video", (event, found[0]) + return "declared", (event, found[0]) + + +def record_bounds(lines, close, talk_id): + """Return the first line, the end line, and the dash offset of one talk. + + The record starts at its "- id:" line and runs to the line before the + next record of the block, or before the end of the front matter. Every + field of a record, and every item of a list inside it, is indented + further than the dash, which is what ends the scan. The offset is the + indentation of the dash from the "talks:" key, 0 or 2, which is also the + indentation of the dash of a link from its "links:" key in the same + file. + """ + try: + opened = next(i for i in range(close) if TALKS_KEY.match(lines[i])) + except StopIteration: + return None, None, None + offset, start = None, None + for position in range(opened + 1, close): + found = RECORD_LINE.match(lines[position]) + if not found: + continue + if offset is None: + offset = len(found.group(1)) + if len(found.group(1)) != offset: + break # a list of another block, deeper or shallower + if found.group(2).strip("'\"") == talk_id: + start = position + break + if not lines[position].startswith(" " * offset): + break + if start is None: + return None, None, None + field = " " * (offset + 2) + end = start + 1 + while end < close and lines[end].startswith(field): + end += 1 + return start, end, offset + + +def insert_video(text, talk_id, url): + """Return the page text with a video link on one talk, or None. + + The link goes first in the list, which is the order the thirteen large + editions write (video, then slides), and a list that is not there yet is + written at the end of the record. None means the record was not found, + which the caller reports rather than writing a page it did not + understand. + """ + lines = text.split("\n") + if not lines or lines[0].strip() != "---": + return None + try: + close = lines.index("---", 1) + except ValueError: + return None + + start, end, offset = record_bounds(lines, close, talk_id) + if start is None: + return None + + field = " " * (offset + 2) + links = field + "links:" + link = [field + " " * offset + "- type: video", + field + " " * offset + ' url: "%s"' % url] + for position in range(start, end): + if lines[position] == links: + return "\n".join(lines[:position + 1] + link + lines[position + 1:]) + return "\n".join(lines[:end] + [links] + link + lines[end:]) + + +def insert_session_video(text, title, url): + """Return the page text with one entry in the event-level session_videos + block. + + A block already there gets the entry at its end, after the entries it + holds. A page without one gets the block at the end of the front + matter, under the comment that says what it is. None means the front + matter was not found, which the caller reports rather than writing a + page it did not understand. + """ + lines = text.split("\n") + if not lines or lines[0].strip() != "---": + return None + try: + close = lines.index("---", 1) + except ValueError: + return None + entry = [VIDEO_TITLE % json.dumps(title, ensure_ascii=False), VIDEO_URL % url] + opened = next((i for i in range(1, close) if VIDEOS_KEY.match(lines[i])), None) + if opened is None: + return "\n".join(lines[:close] + VIDEOS_COMMENT + ["session_videos:"] + entry + lines[close:]) + end = opened + 1 + while end < close and lines[end].startswith(" "): + end += 1 + return "\n".join(lines[:end] + entry + lines[end:]) + + +def unchanged_apart_from_session(before, after, title, url): + """Return True when the only difference between two front matters is one + more entry at the end of the event-level session_videos block.""" + if {k: v for k, v in (before or {}).items() if k != "session_videos"} \ + != {k: v for k, v in (after or {}).items() if k != "session_videos"}: + return False + was = (before or {}).get("session_videos") or [] + return (after or {}).get("session_videos") == was + [{"title": title, "url": url}] + + +def write_session(video, event, dry_run): + """Put the recording of a whole session on its event. Return None, or + what went wrong.""" + path = event["path"] + title = video["talk_title"] or video.get("title_raw") or video["video_id"] + text = path.read_text(encoding="utf-8") + updated = insert_session_video(text, title, video["url"]) + if updated is None: + return "no front matter to write the session_videos block into" + if not unchanged_apart_from_session(front_matter_of(text), front_matter_of(updated), + title, video["url"]): + return "the edit would change more than the session_videos block" + if not dry_run: + path.write_text(updated, encoding="utf-8") + data = front_matter_of(path.read_text(encoding="utf-8")) or {} + urls = [entry.get("url") for entry in data.get("session_videos") or [] + if isinstance(entry, dict)] + if video["url"] not in urls: + return "the page does not carry the URL after the write" + event["held"][video["url"]] = "the session_videos block of the event" + return None + + +def written_talk(path, talk_id): + """Return the talk record of a page as the front matter now parses it. + + The page is read again from disk, so what is reported is what the file + holds and not what this script meant to write. + """ + match = FRONT_MATTER.match(path.read_text(encoding="utf-8")) + if not match: + return None + data = yaml.safe_load(match.group(1)) or {} + for record in data.get("talks") or []: + if isinstance(record, dict) and record.get("id") == talk_id: + return record + return None + + +def unchanged_apart_from(before, after, talk_id, url): + """Return True when the only difference between two front matters is the + video link on one talk. + + The page is written as text, so this is the check that the text edit did + what the record says: everything else of the front matter, every other + talk, and every other field of this talk have to come back equal, and + this talk has to carry the URL as the first link of type video. + """ + before_talks = (before or {}).get("talks") or [] + after_talks = (after or {}).get("talks") or [] + if len(before_talks) != len(after_talks): + return False + if {k: v for k, v in (before or {}).items() if k != "talks"} \ + != {k: v for k, v in (after or {}).items() if k != "talks"}: + return False + for was, now in zip(before_talks, after_talks): + if not isinstance(was, dict) or not isinstance(now, dict): + return False + if was.get("id") != talk_id: + if was != now: + return False + continue + if {k: v for k, v in was.items() if k != "links"} \ + != {k: v for k, v in now.items() if k != "links"}: + return False + links = now.get("links") or [] + if links[:1] != [{"type": "video", "url": url}]: + return False + if links[1:] != (was.get("links") or []): + return False + return True + + +def front_matter_of(text): + """Return the parsed front matter of a page text, or None.""" + match = FRONT_MATTER.match(text) + if not match: + return None + return yaml.safe_load(match.group(1)) or {} + + +def write(video, event, talk, dry_run): + """Put one video on one talk. Return None, or what went wrong.""" + path = event["path"] + text = path.read_text(encoding="utf-8") + updated = insert_video(text, talk["id"], video["url"]) + if updated is None: + return "no record '%s' in the talks block" % talk["id"] + if not unchanged_apart_from(front_matter_of(text), front_matter_of(updated), + talk["id"], video["url"]): + return ("the edit would change more than the links of '%s'" + % talk["id"]) + if not dry_run: + path.write_text(updated, encoding="utf-8") + record = written_talk(path, talk["id"]) + urls = [link.get("url") for link in (record or {}).get("links") or [] + if isinstance(link, dict) and link.get("type") == "video"] + if video["url"] not in urls: + return "the page does not carry the URL after the write" + talk["videos"] = [video["url"]] + talk["videos"] + return None + + +def label_of(video): + """Return the day a video title names, as it reads in a report.""" + if video.get("date"): + return video["date"] + if video.get("year") and video.get("month"): + return "%04d-%02d" % (video["year"], video["month"]) + if video.get("year"): + return str(video["year"]) + return "no day in the title" + + +def report(groups, dry_run): + """Print what the run did and what it left for a human.""" + placed = groups["placed"] + off_day = groups["off_day"] + already = groups["already"] + pages = sorted({event["path"].name for video, event, talk in placed}) + talks = {(event["name"], talk["id"]) for video, event, talk in placed} + + print() + print("%s on %s, over %s" + % (plural(len(placed), "video"), plural(len(talks), "talk"), + plural(len(pages), "page"))) + # Counted apart from the line above on purpose: a video placed on a day + # its title does not name is a decision the programme took over the + # label, and a human has to be able to see every one of them. + print("%s placed where the programme announced the talk, " + "not on the day the title says" % plural(len(off_day), "video")) + print("%s written on the event as the recording of a whole session" + % plural(len(groups["session"]), "video")) + # Counted apart as well: a declared placement is a decision a person + # wrote in the export, not a match the grammar found, and a human has to + # be able to see every one of them with its reason. + print("%s placed as a person declared in the export, by id and with a reason" + % plural(len(groups["declared"]), "video")) + print("%s already in place and not touched" % plural(len(already), "video")) + for video, (event, where) in sorted(already, key=lambda row: row[1][0]["name"]): + print(" in place %-12s %-62s -> %s, on %s" + % (video["video_id"], video["talk_title"][:62], event["name"], where)) + for state, heading in ( + ("ambiguous", "left out, two talks of the day match one video"), + ("other_video", "left out, the talk already carries another video"), + ("other_day", "left out, the day in the title and the programme disagree"), + ("no_talk", "left out, no talk of the archive carries this title"), + ("no_session", "left out, a session recording with no page to go to"), + ("no_declared", "left out, a declared record that names no page or no talk of this tree"), + ("failed", "left out, the page could not be written"), + ("out_of_scope", "out of scope, which the export says and is not an error")): + group = groups[state] + print("%s %s" % (plural(len(group), "video"), heading)) + + for video, event, talk in sorted(placed, key=lambda row: row[1]["name"]): + print(" placed %-12s %-62s -> %s / %s" + % (video["video_id"], video["talk_title"][:62], event["name"], talk["id"])) + for video, event, talk in sorted(off_day, key=lambda row: row[1]["name"]): + print(" PROGRAMME %-12s title says %s, %r is announced by %s / %s" + % (video["video_id"], label_of(video), video["talk_title"], + event["name"], talk["id"])) + for video, event in sorted(groups["session"], key=lambda row: row[1]["name"]): + print(" SESSION %-12s %r -> %s, in the session_videos block of the event" + % (video["video_id"], video["talk_title"], event["name"])) + for video, event, talk in sorted(groups["declared"], key=lambda row: row[1]["name"]): + where = ("%s / %s" % (event["name"], talk["id"]) if talk + else "%s, in the session_videos block of the event" % event["name"]) + print(" DECLARED %-12s %r -> %s: %s" + % (video["video_id"], video["talk_title"], where, + video.get("declared_reason") or "no reason given")) + for video, found in groups["ambiguous"]: + print(" AMBIGUOUS %-12s %r matches %s" + % (video["video_id"], video["talk_title"], + ", ".join("%s / %s" % (event["name"], talk["id"]) for event, talk in found))) + for video, (event, talk) in groups["other_video"]: + print(" OTHER VIDEO %-12s %s / %s already carries %s" + % (video["video_id"], event["name"], talk["id"], + ", ".join(talk["videos"]))) + for video, found in groups["other_day"]: + print(" OTHER DAY %-12s title says %s, %r is announced by %s" + % (video["video_id"], label_of(video), video["talk_title"], + ", ".join("%s / %s" % (event["name"], talk["id"]) for event, talk in found))) + for video, found in groups["no_talk"]: + where = ", ".join(event["name"] for event in found) or "no event of that day" + print(" NO TALK %-12s title says %s, %r, day holds: %s" + % (video["video_id"], label_of(video), video["talk_title"], where)) + for video, why in groups["no_session"]: + print(" NO SESSION PAGE %-12s title says %s, %r: %s" + % (video["video_id"], label_of(video), video["talk_title"], why)) + for video, why in groups["no_declared"]: + print(" NO DECLARED PAGE %-12s record says %s, %r: %s" + % (video["video_id"], label_of(video), video["talk_title"], why)) + for video, why in groups["failed"]: + print(" FAILED %-12s %s" % (video["video_id"], why)) + for video, why in groups["out_of_scope"]: + print(" OUT OF SCOPE %-12s %r: %s" % (video["video_id"], video["talk_title"], why)) + + if dry_run: + print("\n--dry-run: no page was written.") + + +def main(): + parser = argparse.ArgumentParser( + description="Write the video of a talk into the page of its event.") + parser.add_argument("videos", type=pathlib.Path, + help="the normalized videos.json of the archive export") + parser.add_argument("--out", type=pathlib.Path, default=EVENTS_DIR, + help="the directory of the event pages " + "(default: content/events)") + parser.add_argument("--dry-run", action="store_true", + help="report what would be written, and write nothing") + args = parser.parse_args() + + if not args.videos.is_file(): + fail("no such file: %s" % args.videos) + if not args.out.is_dir(): + fail("no such directory: %s" % args.out) + + try: + videos = json.loads(args.videos.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + fail("cannot read %s: %s" % (args.videos, error)) + if not isinstance(videos, list): + fail("%s does not hold a list of videos" % args.videos) + + seen, records = set(), [] + for video in videos: + if not isinstance(video, dict) or not video.get("url"): + fail("a record of %s has no url" % args.videos) + # An address of this archive is https. bin/check-rendered fails a + # built page that carries an http:// URL, and a video link reaches + # the page as written. + if not video["url"].startswith("https://"): + fail("video %s has the URL %r, which is not https" + % (video.get("video_id"), video["url"])) + # The playlists overlap, so the export is read by video id and a + # repeated id would write one link twice. + video_id = video.get("video_id") or video["url"] + if video_id in seen: + fail("video %s appears twice in %s" % (video_id, args.videos)) + seen.add(video_id) + # The id is written back, because the report names every video by it + # and a record without one would end the run with a KeyError after + # the pages were already written. + video["video_id"] = video_id + video["talk_title"] = canon(video.get("talk_title") or "").strip() + records.append(video) + + events = read_events(args.out) + groups = {state: [] for state in ("placed", "off_day", "already", + "session", "declared", "ambiguous", + "other_video", "other_day", "no_talk", + "no_session", "no_declared", "failed", + "out_of_scope")} + for video in records: + state, detail = classify(video, events) + if state in ("placed", "off_day"): + event, talk = detail + problem = write(video, event, talk, args.dry_run) + if problem: + groups["failed"].append((video, problem)) + else: + groups[state].append((video, event, talk)) + elif state == "declared": + event, talk = detail + if talk: + problem = write(video, event, talk, args.dry_run) + else: + problem = write_session(video, event, args.dry_run) + if problem: + groups["failed"].append((video, problem)) + else: + groups["declared"].append((video, event, talk)) + elif state == "session": + problem = write_session(video, detail, args.dry_run) + if problem: + groups["failed"].append((video, problem)) + else: + groups["session"].append((video, detail)) + elif state == "already": + groups["already"].append((video, detail)) + elif state == "other_video": + groups["other_video"].append((video, detail)) + else: + groups[state].append((video, detail)) + + report(groups, args.dry_run) + left = sum(len(groups[state]) for state in + ("ambiguous", "other_video", "other_day", "no_talk", + "no_session", "no_declared", "failed")) + return 1 if left else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/import-x-photos b/bin/import-x-photos new file mode 100755 index 000000000..f9eccfae5 --- /dev/null +++ b/bin/import-x-photos @@ -0,0 +1,945 @@ +#!/usr/bin/env python3 +"""Stage the X photos of one event, ready for a human to curate. + +PyBCN posted photos live from each PyDay on X. Those photos are the only +record of the day that the association still owns, and the X export is the +only copy that comes with the text that was published next to each one. This +script turns one event out of that export into a staging directory: the +photo files under deterministic names, a photos.yaml block to paste into the +event's front matter, and a report.md for the person who curates the set. + +Two editions were imported before this script could do all of it, and each +one cost a person the same four decisions by hand: keep one photo per post, +drop what the gallery does not want, point each photo at the talk it shows, +and write the front matter. Nine editions are left, so those four decisions +are options here: + + --until an event can last more than one day, and the Meetup date + gives only the first. PyData 2017 ran on the 19th, 20th, + and 21st of May, and PyDay 2016 on the 15th and 16th of + September. Searching one date at a time loses 11 own + photos over those two events. + --one-per-tweet a post with several photos publishes one text for all of + them, and the caption is the alt text, so the second + photo of a post arrives with the caption of the first. + One photo per post is the rule the two imported + editions followed by hand. + --drop a caption pattern that the gallery does not want. The + report names what each pattern dropped, because a silent + drop is the one thing a curation list must not do. + --event match a caption against the talks of an event file. The + caption carries the title of a talk in 46% of the photos: + 107 of the 231 the galleries hold on 2026-10-13 name one, + now that every talk has an id. + --install copy the photos that survive into the theme, which is + the last step once the set is curated. + +It never writes into content/. The event front matter stays a human edit: +photos.yaml is a block to paste, not a block this script pastes. Under +--install it copies the staged image files into the theme's own photo +directory, and nothing else of the theme is touched. + +Retweets are counted and reported, but never imported. A retweet carries +somebody else's photo, so its copyright is somebody else's too, and each one +needs its own permission before it can be published. The report names the +original account of every excluded retweet, which is the list to ask. + +Privacy: the export also holds direct messages, Grok chats, and moments. +None of that is public, and none of it is needed here, so the script reads +only data/tweets.js and data/tweets_media/. Any path that carries one of the +private markers stops the run with exit code 2 instead of being opened. + +Captions are literal. The text comes from the tweet, with the t.co links +resolved, and goes through canon() from bin/canon.py, which normalizes the +typography and never touches a letter. prep() is NOT used for a caption: +prep() also drops the Markdown markup characters and the emoji, which is +what makes it right for comparing two strings and wrong for storing one. +Nothing is rewritten, shortened, or translated. + +One thing does come off: the mentions X writes in front of a reply, to name +who it answers. The account never typed them, and a caption that opens with +"@MrSwats" says nothing about the photo under it. See REPLY_PREFIX for the +rule and for why a field of the export cannot give it. The report names +every caption this changed. + +Exit codes: + 0 the event was staged, with no problem found + 1 the event was staged, but something needs a look: no own photo for + the dates, or a media file the export does not carry + 2 the script could not run: bad arguments, a missing export, an --out + inside content/ or themes/, an unreadable event file, or a path that + touches private data + +Usage: + bin/import-x-photos --date YYYY-MM-DD [--until YYYY-MM-DD] + --export DIR --out DIR [--slug NAME] + [--event FILE] [--install] [--one-per-tweet] + [--drop PATTERN ...] + +--export is the directory that holds one subdirectory per account, each with +the unpacked export inside it (DIR/PyBCN/extracted/data/tweets.js). --slug is +the media directory the site will serve the photos from, and it prefixes +every "file" value in photos.yaml. It defaults to the first date, so a real +import usually passes the site's own name, for example --slug pyday-2019. +""" + +import argparse +import datetime +import html +import json +import pathlib +import re +import shutil +import sys +import unicodedata +from email.utils import parsedate_to_datetime +from zoneinfo import ZoneInfo + +import yaml + +# canon.py sits next to this script and is not an installed package. +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from canon import canon, prep # noqa: E402 +from titlematch import TITLE_MINIMUM, fold, match_title # noqa: E402 + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +# Where --install puts the photos. partials/photo_gallery.html resolves a +# declared file as images/photos/ with resources.Get, so this is the +# directory that makes a staged photo render. +INSTALL_ROOT = ROOT / "themes" / "pybcn_theme" / "assets" / "images" / "photos" + +# The two accounts that posted from a PyDay: the association's own account +# and the one of the member who live-tweeted several editions. +ACCOUNTS = ("PyBCN", "DZPM") + +# The photo times are reported in Barcelona local time, because that is the +# order the day happened in and the only form a curator can place against a +# schedule. The dates themselves are matched in UTC, which is what created_at +# carries, so the day boundary does not move with the summer time rule of +# whichever machine runs this. +LOCAL_ZONE = ZoneInfo("Europe/Madrid") + +# A file name has to stay readable in a URL and in a directory listing, so +# the caption slug is cut here. The number keeps the chronological order, so +# a cut tail loses nothing that identifies the photo. +SLUG_LIMIT = 50 + +# The export directories this script must never open. Direct messages, Grok +# chats, and moments are private, and no part of this import needs them. The +# check is on the name, so it also stops a symlink or a --export that points +# somewhere else from reaching them. +PRIVATE_MARKERS = ("direct_message", "grok_chat", "moments") + +# The only data files this script reads. A large export splits the tweets +# over numbered parts, so both forms are accepted, and nothing else is. +TWEETS_FILE = re.compile(r"\Atweets(-part\d+)?\.js\Z") + +# tweets.js is a JavaScript assignment ("window.YTD.tweets.part0 = [...]"), +# not JSON. Everything up to the first "=" is the assignment, and the rest +# parses as JSON. +ASSIGNMENT = "=" + +# A t.co link is X's own shortener for a URL or for the attached media. It +# resolves to nothing useful in a caption on the site, and the media link +# just points back at the photo the caption sits under. +T_CO = re.compile(r"https?://t\.co/\S+") + +# A retweet has no field of its own in the export: the only marker is the +# "RT @account:" prefix that X writes into full_text. +RETWEET = re.compile(r"\ART @([A-Za-z0-9_]+):") + +# The run of mentions X writes in front of a reply, to name who is being +# replied to. A handle is at most 15 characters of [A-Za-z0-9_], and the run +# can hold several of them. +# +# Why a pattern and not a field of the export. The live X API marks this run: +# display_text_range starts after it, and entities.user_mentions lists each +# recipient with its indices. The archive export does neither reliably. +# Measured over the 3195 posts of the PyBCN account and the 15 reply posts +# behind the galleries: display_text_range starts at 0 on every one of them, +# including the replies, so it cannot mark anything; and user_mentions is +# empty on 2 of the 15 (both replies to @MrSwats) and misses the second +# recipient of a third (@HybridTheory_GL), because a mention of an account +# that no longer resolves is dropped from entities but stays in the text. +# Trusting entities would therefore have left three captions with the prefix +# the import is here to remove. +# +# What the export does carry, on all 15 and on 6943 of the 7021 posts that +# open with a mention, is in_reply_to_status_id_str. That is the fact the +# rule uses: the prefix is removed only from a reply, which is the only post +# X writes one on. A post that is not a reply keeps a leading mention, +# because there it is the text the account wrote. +REPLY_PREFIX = re.compile(r"\A(?:@[A-Za-z0-9_]{1,15}\s+)+") + +# The YAML front matter of a Hugo page, the same shape bin/check-content +# reads: the first block between two "---" lines. +FRONT_MATTER = re.compile(r"\A---\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\Z)", re.S) + +# The length guard of a title match, and the comparison form itself, come +# from bin/titlematch.py: bin/import-videos matches the same titles against +# the title of a YouTube upload, and one rule cannot live in two files. +_NOT_SLUG = re.compile(r"[^a-z0-9]+") + + +def plural(count, noun): + """Return "1 photo" or "4 photos": a count a person reads out loud.""" + return f"{count} {noun}" if count == 1 else f"{count} {noun}s" + + +def fail(message): + """Stop the run with exit code 2: the script cannot do its job.""" + print(f"import-x-photos: {message}", file=sys.stderr) + raise SystemExit(2) + + +def check_private(path): + """Stop the run if path names private export data. + + This is the privacy rule in code rather than in a comment. Every read + goes through it, so a future change to the file selection cannot quietly + start opening a direct message archive. + """ + lowered = str(path).lower() + for marker in PRIVATE_MARKERS: + if marker in lowered: + fail(f"refusing to read private export data ({marker}): {path}") + + +def check_out(out): + """Stop the run if out is inside the site's content or theme tree.""" + for tree in ("content", "themes"): + guarded = ROOT / tree + if out == guarded or guarded in out.parents: + fail(f"--out is inside {tree}/, which this script never writes: {out}") + + +def read_tweets(account_dir): + """Return the tweet records of one account, with the media directory. + + Reads only the tweets files and reports the directory the photos sit in. + No other file of the export is opened. + """ + data_dir = account_dir / "extracted" / "data" + if not data_dir.is_dir(): + fail(f"no export data directory: {data_dir}") + + media_dir = data_dir / "tweets_media" + check_private(media_dir) + if not media_dir.is_dir(): + fail(f"no media directory: {media_dir}") + + paths = sorted(path for path in data_dir.iterdir() if TWEETS_FILE.match(path.name)) + if not paths: + fail(f"no tweets.js in {data_dir}") + + records = [] + for path in paths: + check_private(path) + text = path.read_text(encoding="utf-8") + if ASSIGNMENT not in text: + fail(f"not a JavaScript assignment: {path}") + body = text[text.index(ASSIGNMENT) + 1:] + try: + records.extend(json.loads(body)) + except json.JSONDecodeError as error: + fail(f"cannot parse {path}: {error}") + + return records, media_dir + + +def photos_of(tweet): + """Return the photo media of a tweet, in the order it published them. + + A tweet can also carry a video or an animated GIF. Only a photo is + imported, so the type filter is what decides whether a tweet counts. + """ + media = (tweet.get("extended_entities") or {}).get("media") or [] + return [item for item in media if item.get("type") == "photo"] + + +def strip_reply_prefix(text, is_reply): + """Return text without the mentions X wrote in front of a reply. + + Only the run at the very start comes off, and only on a reply. A mention + inside the sentence is text the account wrote ("by Jimena (@Jimena_y_yo)") + and stays where it is. + + The whole text is kept when the prefix is all there is. A caption is also + the alt text of the photo, and the gallery skips a photo with no caption, + so a post whose text is nothing but recipients would lose its photo. That + is a curation decision and not one this function may take in silence. + """ + if not is_reply: + return text + stripped = REPLY_PREFIX.sub("", text or "") + return stripped if stripped.strip() else text + + +def caption_of(full_text, urls=None, is_reply=False): + """Return the caption of a tweet: its text, with the t.co links resolved. + + A reply also loses the mentions X wrote in front of it. See REPLY_PREFIX: + they name who the post answers, they were never typed by the account, and + a caption that opens with "@MrSwats" says nothing about the photo under + it. + + canon() does the normalization, not prep(). The difference matters here. + prep() is built to compare two strings, so it also strips the Markdown + markup characters and the emoji, and in a caption that is destruction: + "#PyDayBCN" loses its hash, "@hectorcanto_dev" loses the underscore of a + real handle, and the applause of a thank-you tweet disappears. canon() + normalizes the typography and nothing else. + + X shortens every link in the text to a t.co URL, and the export keeps + what each one said in entities.urls. A t.co link that is NOT in that list + is the link to the attached media, which points back at the photo the + caption sits under, so it comes off. One that IS in the list is real text: + the wifi SSID of a 2019 tweet reads as a t.co link in full_text, and + dropping it leaves the sentence saying "use SSID" and nothing else. + """ + # X stores the text with its HTML entities, so a post that wrote "&" + # arrives as "&" and printed that way on the page. Unescape before + # anything else: a t.co URL carries no entity, and from here the caption + # is plain text, which the template escapes again when it prints it. + text = strip_reply_prefix(html.unescape(full_text or ""), is_reply) + for short, expanded in (urls or {}).items(): + text = text.replace(short, expanded) + text = canon(T_CO.sub(" ", text)) + # A caption becomes an alt attribute, which is one line. canon() collapses + # spaces and tabs but keeps a newline, because a newline matters in the + # body text it normalizes elsewhere. Here it does not: a tweet that broke + # its sentence over two lines reads as one sentence in a caption. + return re.sub(r"\s+", " ", text).strip() + + +def slugify(text): + """Return a file-name slug of text: lowercase ASCII words and hyphens. + + The accents are folded here, which canon() deliberately never does. That + is safe in this one place because the result is a file name and not + reading text: "Història" and "Historia" both slug to "historia", and the + caption next to it keeps the accent that the tweet published. + """ + folded = unicodedata.normalize("NFKD", text) + folded = "".join(character for character in folded + if not unicodedata.combining(character)) + slug = _NOT_SLUG.sub("-", folded.lower()).strip("-") + # The cut can land in the middle of a word and leave a trailing hyphen, + # so the strip runs again after it. + return slug[:SLUG_LIMIT].strip("-") or "photo" + + +def extension_of(media_url): + """Return the file extension of a media URL, lowercased.""" + return pathlib.PurePosixPath(media_url).suffix.lower() or ".jpg" + + +def local_path(media_dir, tweet_id, media_url): + """Return the exported file of one photo. + + The export names a media file after the tweet that published it plus the + last segment of the media URL. + """ + return media_dir / f"{tweet_id}-{pathlib.PurePosixPath(media_url).name}" + + +def read_talks(path): + """Return the talks of an event file, each with its caption match form. + + Only the id and the title are read. The id is what a photo stores, and + the title is the one field a caption can carry, because a live post + quotes the title of the talk it shows. + """ + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + fail(f"cannot read --event file: {error}") + + match = FRONT_MATTER.match(text) + if not match: + fail(f"--event file has no YAML front matter: {path}") + + try: + data = yaml.safe_load(match.group(1)) or {} + except yaml.YAMLError as error: + fail(f"cannot parse the front matter of {path}: {error}") + + records = data.get("talks") or [] + talks, unusable = [], [] + for position, record in enumerate(records, start=1): + if not isinstance(record, dict): + unusable.append(f"position {position} is not a talk") + continue + talk_id, title = record.get("id"), record.get("title") + if not talk_id or not title: + unusable.append(f"'{title or talk_id or position}' has no id or no title") + continue + talks.append({"id": talk_id, "title": title, "folded": fold(title)}) + + if not talks: + fail(f"no talk with an id and a title in {path}") + return talks, unusable + + +def match_talk(caption, talks): + """Return (talk id, ambiguous ids) for one caption. + + A talk matches when its folded title is contained in the folded caption, + which is what a live post does: it quotes the title and adds the speaker + ("Causal Inference: a brief introduction, by Ubaldo Hervas"). When two + talks match one caption, the return is no id and both of them: an + ambiguous link is worse than no link, because the gallery would point + the reader at the wrong talk and nothing on the page would say so. + """ + return match_title(caption, [(talk["id"], talk["folded"]) for talk in talks]) + + +def collect(export_dir, start, end): + """Return the own tweets and the excluded retweets of a date range. + + A tweet counts when it carries at least one photo and its created_at, + read as UTC, falls on or between the two dates. Both accounts are + walked, and the result is sorted chronologically over the two of them + together, because the numbering has to follow the event and not the + account. + """ + own, retweets = [], [] + + for account in ACCOUNTS: + account_dir = export_dir / account + if not account_dir.is_dir(): + fail(f"no export for account {account}: {account_dir}") + records, media_dir = read_tweets(account_dir) + + for record in records: + tweet = record.get("tweet") or {} + media = photos_of(tweet) + if not media: + continue + + moment = parsedate_to_datetime(tweet["created_at"]) + if not start <= moment.astimezone(datetime.timezone.utc).date() <= end: + continue + + full_text = tweet.get("full_text") or "" + entry = { + "account": account, + "id": tweet["id_str"], + "utc": moment.astimezone(datetime.timezone.utc), + "local": moment.astimezone(LOCAL_ZONE), + "media": media, + "media_dir": media_dir, + "full_text": full_text, + # Whether X wrote the recipients in front of the text. See + # REPLY_PREFIX: this is the one field of the export that says + # so on every post of the galleries. + "is_reply": bool(tweet.get("in_reply_to_status_id_str")), + # X shortens every link in the text to a t.co URL, including a + # token that only looks like one: the wifi SSID of one PyDay + # tweet reads as "https://t.co/..." in full_text. Removing the + # t.co links therefore removes real content, and entities.urls + # is the only place that still holds what it said. + "urls": {url["url"]: (url.get("expanded_url") or url["url"]) + for url in (tweet.get("entities") or {}).get("urls", []) + if url.get("url")}, + } + + retweet = RETWEET.match(full_text) + if retweet: + entry["source_account"] = retweet.group(1) + retweets.append(entry) + else: + entry["caption"] = caption_of(full_text, entry["urls"], + entry["is_reply"]) + own.append(entry) + + key = lambda entry: (entry["utc"], entry["id"]) # noqa: E731 + own.sort(key=key) + retweets.sort(key=key) + return own, retweets + + +def enumerate_photos(own, talks): + """Return one record per photo of the event, named, numbered, and matched. + + The name of a photo is its tweet's chronological number, the slug of the + caption, and the original extension. The number is per tweet, so the + photos of one tweet sort together; a second photo of the same tweet gets + a -2 suffix, a third -3, because they share the one caption. + + Every photo is named here, dropped or not, and the number counts the + tweets of the event and not the photos that survive. So a name never + moves when a filter changes, and the gap a dropped photo leaves in the + numbering is a record that something was dropped. + """ + photos = [] + taken = set() + + for number, tweet in enumerate(own, start=1): + stem = f"{number:02d}-{slugify(tweet['caption'])}" + talk, ambiguous = match_talk(tweet["caption"], talks) if talks else (None, []) + for index, media in enumerate(tweet["media"], start=1): + extension = extension_of(media["media_url_https"]) + name = stem if index == 1 else f"{stem}-{index}" + candidate = f"{name}{extension}" + # A collision cannot happen with the numbering above, but a + # duplicate name would silently overwrite a photo, so the loop + # guards it anyway. + suffix = index + while candidate in taken: + suffix += 1 + candidate = f"{stem}-{suffix}{extension}" + taken.add(candidate) + + photos.append({ + "file": candidate, + "index": index, + "caption": tweet["caption"], + "account": tweet["account"], + "id": tweet["id"], + "date": tweet["local"].date(), + "time": tweet["local"].strftime("%H:%M"), + "media": media, + "media_dir": tweet["media_dir"], + "full_text": tweet["full_text"], + "urls": tweet["urls"], + "talk": talk, + "ambiguous": ambiguous, + "drop": None, + "bytes": None, + }) + + return photos + + +def apply_filters(photos, one_per_tweet, patterns): + """Mark the photos that a filter drops, with the reason for each one. + + --one-per-tweet runs first, so an extra photo of a post is reported as + the extra it is, rather than as a pattern hit that happens to be true of + its inherited caption. A pattern is tried in the order it was given on + the command line, and the first one that matches is the reason recorded: + one reason per photo reads as a decision, a list of them does not. + """ + for photo in photos: + if one_per_tweet and photo["index"] > 1: + photo["drop"] = ("one-per-tweet", + f"photo {photo['index']} of the post, same caption as photo 1") + continue + for pattern, regex in patterns: + if regex.search(photo["caption"]): + photo["drop"] = ("drop", pattern) + break + + +def stage(photos, out_dir): + """Copy the photos that no filter dropped into out_dir. + + A photo the export names but does not carry is marked dropped too, with + the path that is missing, so it appears in the curation list instead of + just in a warning at the end of the run. + """ + staged, missing = [], [] + + for photo in photos: + if photo["drop"]: + continue + source = local_path(photo["media_dir"], photo["id"], photo["media"]["media_url_https"]) + check_private(source) + if not source.is_file(): + photo["drop"] = ("missing", str(source)) + missing.append(source) + continue + + target = out_dir / photo["file"] + shutil.copy2(source, target) + photo["bytes"] = target.stat().st_size + staged.append(photo) + + return staged, missing + + +def install(staged, out_dir, slug): + """Copy the staged photos into the theme and return the directory. + + This is the one write outside the staging directory, and it is only the + image files: content/ is never touched, so the front matter that names + them stays a human edit. + """ + target_dir = INSTALL_ROOT / slug + target_dir.mkdir(parents=True, exist_ok=True) + for photo in staged: + shutil.copy2(out_dir / photo["file"], target_dir / photo["file"]) + return target_dir + + +def yaml_string(text): + """Return text as a double-quoted YAML scalar.""" + escaped = text.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def tweet_url(account, tweet_id): + return f"https://x.com/{account}/status/{tweet_id}" + + +def write_photos_yaml(path, slug, own, staged): + """Write the front matter block to paste into the event file. + + No "credit" field is emitted: who a photo is credited to is what the + human curation decides. "talk" is emitted for a photo whose caption + carried the title of exactly one talk of the --event file, which is the + one link a caption can prove on its own. + """ + lines = ["photos:"] + if own: + first = own[0] + lines.append(f' thread: {yaml_string(tweet_url(first["account"], first["id"]))}') + lines.append(" items:") + for item in staged: + lines.append(f' - file: {slug}/{item["file"]}') + lines.append(f' caption: {yaml_string(item["caption"])}') + lines.append(f' source: {yaml_string(tweet_url(item["account"], item["id"]))}') + lines.append(f' time: {yaml_string(item["time"])}') + if item["talk"]: + lines.append(f' talk: {item["talk"]}') + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def normalization_notes(staged): + """Return the captions that differ from the raw text of the post. + + Three changes can do it. A t.co link is replaced by what entities.urls + says it stood for; the t.co link of the attached media is removed; and a + reply loses the mentions X wrote in front of it. All three are a visible + change to what the post said, so every one of them is reported for a + human to read back. + """ + notes = [] + for item in staged: + plain = canon(T_CO.sub(" ", item["full_text"])).strip() + plain = re.sub(r"\s+", " ", plain) + if plain != item["caption"]: + notes.append((item["file"], plain, item["caption"])) + return notes + + +def date_label(start, end): + """Return the heading form of the date range.""" + return start.isoformat() if start == end else f"{start.isoformat()} to {end.isoformat()}" + + +def write_report(path, start, end, slug, own, retweets, photos, staged, missing, + patterns, talks, event_path, unusable, installed): + """Write the report a human reads to curate the set. + + The curation list is the point of this file: every photo of the event in + order, with the caption, the time, the talk it was matched to, and, when + it was dropped, which option dropped it. That is enough to decide the + set without opening the export or the JSON behind it. + """ + retweet_photos = sum(len(entry["media"]) for entry in retweets) + accounts = {} + for entry in retweets: + source = entry["source_account"] + accounts[source] = accounts.get(source, 0) + len(entry["media"]) + + dropped_extra = [p for p in photos if p["drop"] and p["drop"][0] == "one-per-tweet"] + dropped_pattern = [p for p in photos if p["drop"] and p["drop"][0] == "drop"] + matched = [p for p in staged if p["talk"]] + ambiguous = [p for p in staged if p["ambiguous"]] + + lines = [ + f"# X photo import for {date_label(start, end)}", + "", + f"Staged into `{slug}/`. Nothing was written to `content/`: paste the" + " photos.yaml block into the event file by hand, and set the credits there.", + "", + ] + if installed: + lines += [f"Installed into `{installed.relative_to(ROOT)}/`," + f" {plural(len(staged), 'file')}." + f" The front matter is still a human edit.", ""] + else: + lines += ["Not installed: this run staged only. Add `--install` to copy the" + " surviving photos into the theme.", ""] + + lines += ["## Curation list", "", + "Every photo of the event, in the order it was posted. A number is the" + " position of the post, so a gap is a photo that a filter dropped.", ""] + for photo in photos: + if photo["drop"]: + reason, detail = photo["drop"] + if reason == "one-per-tweet": + status = f"dropped by --one-per-tweet ({detail})" + elif reason == "drop": + status = f"dropped by --drop `{detail}`" + else: + status = f"dropped, missing from the export ({detail})" + elif photo["talk"]: + status = f"staged, talk `{photo['talk']}`" + elif photo["ambiguous"]: + status = ("staged, no talk: the caption matches " + + ", ".join(f"`{talk_id}`" for talk_id in photo["ambiguous"]) + + ", so neither is linked") + elif talks: + status = "staged, no talk matched" + else: + status = "staged" + when = photo["time"] if start == end else f"{photo['date'].isoformat()} {photo['time']}" + lines.append(f"- `{photo['file']}` {when}: {status}") + lines.append(f" - caption: {photo['caption'] or '(empty)'}") + lines.append(f" - source: {tweet_url(photo['account'], photo['id'])}") + if not photos: + lines.append("No own photo in the date range.") + lines.append("") + + lines += ["## Totals", "", "| What | Count |", "|---|---|", + f"| Own posts with a photo | {len(own)} |", + f"| Photos in them | {len(photos)} |", + f"| Dropped by --one-per-tweet | {len(dropped_extra)} |", + f"| Dropped by --drop | {len(dropped_pattern)} |", + f"| Dropped, missing from the export | {len(missing)} |", + f"| Photos staged | {len(staged)} |", + f"| Staged photos matched to a talk | {len(matched)} |", + f"| Staged photos with an ambiguous title match | {len(ambiguous)} |", + f"| Retweet photos excluded | {retweet_photos} |", + ""] + + lines += ["## Dropped by a pattern", ""] + if not patterns: + lines.append("No --drop pattern was given.") + else: + lines.append("A pattern that dropped nothing is reported too, because a pattern" + " that matches nothing is usually a typo and not an empty set:") + lines.append("") + for pattern, _ in patterns: + hits = [p for p in dropped_pattern if p["drop"][1] == pattern] + lines.append(f"- `{pattern}`: {plural(len(hits), 'photo')}") + for photo in hits: + lines.append(f" - `{photo['file']}`: {photo['caption']}") + lines.append("") + + lines += ["## Talk matches", ""] + if not talks: + lines.append("No --event file was given, so no photo was matched to a talk.") + else: + short = [talk for talk in talks if len(talk["folded"]) < TITLE_MINIMUM] + lines.append(f"Read {len(talks)} talks from `{event_path}`. A caption matches a" + f" talk when the normalized title is contained in the normalized" + f" caption, and a title shorter than {TITLE_MINIMUM} characters is" + f" never matched, because a short title also appears inside" + f" unrelated prose.") + lines.append("") + if short: + lines.append("Too short to be matched, so a photo of one of these has to be" + " linked by hand:") + lines.append("") + lines += [f"- `{talk['id']}`: {talk['title']}" for talk in short] + lines.append("") + if unusable: + lines.append("Not usable as a match target:") + lines.append("") + lines += [f"- {note}" for note in unusable] + lines.append("") + if ambiguous: + lines.append("Ambiguous, left unlinked. More than one title fits the" + " caption, so the link would be a guess:") + lines.append("") + for photo in ambiguous: + lines.append(f"- `{photo['file']}`: " + + ", ".join(f"`{i}`" for i in photo["ambiguous"])) + lines.append(f" - caption: {photo['caption']}") + lines.append("") + unmatched = [p for p in staged if not p["talk"] and not p["ambiguous"]] + lines.append(f"Matched {len(matched)} of {len(staged)} staged photos." + f" The {len(unmatched)} below carry no title, so each one is a" + f" human decision:") + lines.append("") + lines += [f"- `{photo['file']}`: {photo['caption'] or '(empty)'}" + for photo in unmatched] + lines.append("") + + lines += ["## Retweets excluded", "", + f"- Retweets with at least one photo: {len(retweets)}", + f"- Photos in them: {retweet_photos}", + "", + "A retweet carries somebody else's photo, so the copyright is theirs.", + "Each account below has to give permission before any of its photos is" + " published.", + ""] + + if accounts: + lines.append("| Account | Photos |") + lines.append("|---|---|") + for account, count in sorted(accounts.items(), key=lambda pair: (-pair[1], pair[0])): + lines.append(f"| @{account} | {count} |") + else: + lines.append("No retweet with a photo in the date range.") + lines.append("") + + empty = [item["file"] for item in staged if not item["caption"]] + lines += ["## Empty captions", ""] + if empty: + lines.append("The tweet had no text left after the t.co links came off." + " Write a caption by hand:") + lines.append("") + lines += [f"- `{name}`" for name in empty] + else: + lines.append("None: every staged photo has a caption.") + lines.append("") + + notes = normalization_notes(staged) + lines += ["## Captions that differ from the text of the post", ""] + if notes: + lines.append("A t.co link was replaced by what it stood for, or removed because" + " it pointed at the photo itself, or the post is a reply and lost" + " the mentions X wrote in front of it. Read the two lines back:") + lines.append("") + for name, plain, caption in notes: + lines.append(f"- `{name}`") + lines.append(f" - tweet: {plain}") + lines.append(f" - caption: {caption}") + else: + lines.append("None.") + lines.append("") + + lost = [(item["file"], item["urls"]) for item in staged if item["urls"]] + lines += ["## Links removed from a caption", ""] + if lost: + lines.append("A t.co link comes off the caption, because it points at the photo" + " itself or reads as noise on the page. X also shortens a token that" + " only looks like a link, so one of these may be text the caption" + " needs. Put it back by hand:") + lines.append("") + for name, urls in lost: + lines.append(f"- `{name}`: " + ", ".join(urls)) + else: + lines.append("None.") + lines.append("") + + if missing: + lines += ["## Missing media files", ""] + lines.append("The export names these photos but does not carry the file:") + lines.append("") + lines += [f"- `{path_}`" for path_ in missing] + lines.append("") + + lines += ["## Files written", "", "| File | Time | Bytes |", "|---|---|---|"] + for item in staged: + lines.append(f'| `{item["file"]}` | {item["time"]} | {item["bytes"]} |') + lines.append("") + lines.append(f"Total: {len(staged)} files," + f" {sum(item['bytes'] for item in staged)} bytes.") + + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def parse_date(value, flag): + try: + return datetime.date.fromisoformat(value) + except ValueError: + fail(f"{flag} is not a YYYY-MM-DD date: {value}") + + +def compile_patterns(values): + """Return the --drop patterns as (text, regex) pairs. + + Case is ignored, because a caption is prose a human typed live: "Lunch" + and "lunch" are the same photo to drop. The text is kept next to the + regex so the report can name the pattern the way it was typed. + """ + patterns = [] + for value in values or []: + try: + patterns.append((value, re.compile(value, re.IGNORECASE))) + except re.error as error: + fail(f"--drop is not a valid regular expression: {value} ({error})") + return patterns + + +def main(): + parser = argparse.ArgumentParser( + description="Stage the X photos of one event for a human to curate.") + parser.add_argument("--date", required=True, help="the first event day, as YYYY-MM-DD") + parser.add_argument("--until", help="the last event day, as YYYY-MM-DD, for an event" + " that lasts more than one day (default: --date)") + parser.add_argument("--export", required=True, + help="the directory holding one subdirectory per account") + parser.add_argument("--out", required=True, help="the staging directory to write") + parser.add_argument("--slug", help="the media directory name used in photos.yaml" + " (default: the first date)") + parser.add_argument("--event", help="the event file whose talks a caption is matched" + " against") + parser.add_argument("--install", action="store_true", + help="also copy the staged photos into the theme") + parser.add_argument("--one-per-tweet", action="store_true", + help="keep only the first photo of each post") + parser.add_argument("--drop", action="append", metavar="PATTERN", + help="drop a photo whose caption matches this regular" + " expression, case insensitive. Repeatable") + args = parser.parse_args() + + start = parse_date(args.date, "--date") + end = parse_date(args.until, "--until") if args.until else start + if end < start: + fail(f"--until is before --date: {end.isoformat()} < {start.isoformat()}") + + export_dir = pathlib.Path(args.export).expanduser().resolve() + check_private(export_dir) + if not export_dir.is_dir(): + fail(f"no such export directory: {export_dir}") + + out_dir = pathlib.Path(args.out).expanduser().resolve() + check_out(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + slug = args.slug or start.isoformat() + patterns = compile_patterns(args.drop) + + talks, unusable, event_path = [], [], None + if args.event: + event_file = pathlib.Path(args.event).expanduser().resolve() + if not event_file.is_file(): + fail(f"no such --event file: {event_file}") + talks, unusable = read_talks(event_file) + try: + event_path = event_file.relative_to(ROOT) + except ValueError: + event_path = event_file + + own, retweets = collect(export_dir, start, end) + photos = enumerate_photos(own, talks) + apply_filters(photos, args.one_per_tweet, patterns) + staged, missing = stage(photos, out_dir) + installed = install(staged, out_dir, slug) if args.install else None + + write_photos_yaml(out_dir / "photos.yaml", slug, own, staged) + write_report(out_dir / "report.md", start, end, slug, own, retweets, photos, staged, + missing, patterns, talks, event_path, unusable, installed) + + retweet_photos = sum(len(entry["media"]) for entry in retweets) + matched = sum(1 for photo in staged if photo["talk"]) + print(f"{date_label(start, end)}: {plural(len(own), 'own post')}, " + f"{plural(len(photos), 'photo')}, {len(staged)} staged, " + f"{matched} matched to a talk, " + f"{plural(len(retweets), 'retweet')} excluded " + f"({plural(retweet_photos, 'photo')})") + if installed: + print(f"installed {plural(len(staged), 'file')} into {installed}") + print(f"wrote {out_dir}/photos.yaml and {out_dir}/report.md") + + if missing: + for path_ in missing: + print(f"missing media file: {path_}", file=sys.stderr) + return 1 + if not staged: + print(f"no own photo found for {date_label(start, end)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/list-channel b/bin/list-channel new file mode 100755 index 000000000..582663c2d --- /dev/null +++ b/bin/list-channel @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Print what the YouTube channel holds that data/channel_videos.yaml does not. + +This is how a new video gets noticed. When the board uploads a recording, +nothing on this site changes until a person writes the link on the talk of +its event page. This script names the video, so that step is a known step +and not a thing somebody finds by eye on a meetup page months later. + +The channel is read with yt-dlp, one call per tab that holds videos: the +videos tab and the streams tab (a live stream lands in streams and never +in videos). The shorts tab holds none and is not read. Nothing is cached: +every run reads the channel as it is. + +Two videos of the inventory show in no tab, because they were uploaded +unlisted and only a playlist reveals them. The inventory marks them with +unlisted: true, so their absence from the tabs is expected and reported as +such. A video the inventory holds, not marked unlisted, that no tab shows +is a different thing: it was removed from the channel, or made unlisted, +and a person has to open its URL and decide. Both directions are reported +and named apart. + +yt-dlp is not in this repository and not in CI, and this script is never +part of a check or of the build, because reading the channel is a network +call. Pass the binary with --yt-dlp, or let the script find it on PATH or +at the one place this site's maintainer keeps it. + +Exit codes: + 0 the inventory and the channel agree + 1 there is something for a person to do: a new video, an entry no tab + shows that is not marked unlisted, an unlisted entry a tab now + shows, or a title the channel changed + 2 the script could not run: no yt-dlp, no inventory, or the channel + could not be read + +Usage: + bin/list-channel [--yt-dlp PATH] [--yaml] + +--yaml prints each new video as an inventory entry ready to paste into +data/channel_videos.yaml, with placed: false and an empty reason, which +bin/check-content rejects until a person fills in the decision. +""" + +import argparse +import json +import pathlib +import shutil +import subprocess +import sys + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +INVENTORY = ROOT / "data" / "channel_videos.yaml" + +# The Python Barcelona channel. The inventory names it too, and the two have +# to agree, or the comparison below means nothing. +CHANNEL = "UCEhI2CfdT5--TYq47K4en4A" + +# The tabs of a channel that list videos. shorts holds none on this channel. +TABS = ("videos", "streams") + +# Where the maintainer of this site keeps yt-dlp, in a virtualenv of its +# own, when it is not on PATH. The message below says how to install it. +KNOWN_YT_DLP = pathlib.Path.home() / ".venvs" / "yt-dlp" / "bin" / "yt-dlp" + +INSTALL = ("yt-dlp is not installed. It is a Python package: " + "python3 -m venv ~/.venvs/yt-dlp && " + "~/.venvs/yt-dlp/bin/pip install yt-dlp, " + "then run bin/list-channel --yt-dlp ~/.venvs/yt-dlp/bin/yt-dlp") + +# yt-dlp waits on the network, and a channel of 90 videos takes seconds, so +# this is a stuck call and not a slow one. +TIMEOUT_SECONDS = 300 + + +def fail(message): + """Stop the run with exit code 2: the script cannot do its job.""" + print("list-channel: %s" % message, file=sys.stderr) + sys.exit(2) + + +def plural(count, noun): + """Return "1 video" or "4 videos": a count a person reads out loud.""" + return "%d %s" % (count, noun if count == 1 else noun + "s") + + +def find_yt_dlp(given): + """Return the path of yt-dlp, or stop with the message that says how to + install it.""" + if given: + if not given.is_file(): + fail("no such file: %s. %s" % (given, INSTALL)) + return given + found = shutil.which("yt-dlp") + if found: + return pathlib.Path(found) + if KNOWN_YT_DLP.is_file(): + return KNOWN_YT_DLP + fail(INSTALL) + + +def read_tab(yt_dlp, channel, tab): + """Return the entries of one tab of the channel: a list of (id, title). + + One network call. --flat-playlist lists the videos without reading each + one, and --dump-single-json gives the whole tab as one document. + """ + url = "https://www.youtube.com/channel/%s/%s" % (channel, tab) + try: + run = subprocess.run( + [str(yt_dlp), "--flat-playlist", "--dump-single-json", + "--no-warnings", url], + capture_output=True, text=True, timeout=TIMEOUT_SECONDS, check=False) + except subprocess.TimeoutExpired: + fail("yt-dlp did not answer in %d seconds for %s" % (TIMEOUT_SECONDS, url)) + if run.returncode != 0: + fail("yt-dlp failed on %s (exit %d): %s" + % (url, run.returncode, run.stderr.strip() or "no message")) + try: + data = json.loads(run.stdout) + except ValueError as error: + fail("yt-dlp wrote no JSON for %s: %s" % (url, error)) + if data.get("channel_id") not in (None, channel): + fail("yt-dlp read channel %s, not %s" % (data.get("channel_id"), channel)) + entries = [] + for entry in data.get("entries") or []: + if not isinstance(entry, dict) or not entry.get("id"): + continue + entries.append((entry["id"], (entry.get("title") or "").strip())) + return entries + + +def read_inventory(path): + """Return the inventory as {id: entry} for the channel videos, and the + channel id it names.""" + if not path.is_file(): + fail("no inventory at %s" % path) + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError as error: + fail("cannot parse %s: %s" % (path, error)) + videos = {} + for entry in data.get("videos") or []: + if isinstance(entry, dict) and entry.get("id"): + videos[str(entry["id"])] = entry + return videos, data.get("channel") + + +def entry_yaml(video_id, title): + """Return one inventory entry for a new video, ready to paste.""" + return "\n".join([ + " - id: %s" % json.dumps(video_id), + " title: %s" % json.dumps(title, ensure_ascii=False), + " placed: false", + ' reason: ""', + ]) + + +def main(): + parser = argparse.ArgumentParser( + description="Print what the YouTube channel holds that the inventory does not.") + parser.add_argument("--yt-dlp", type=pathlib.Path, default=None, + help="the yt-dlp binary (default: PATH, then %s)" % KNOWN_YT_DLP) + parser.add_argument("--inventory", type=pathlib.Path, default=INVENTORY, + help="the inventory (default: data/channel_videos.yaml)") + parser.add_argument("--yaml", action="store_true", + help="print each new video as an inventory entry to paste") + args = parser.parse_args() + + yt_dlp = find_yt_dlp(args.yt_dlp) + inventory, channel = read_inventory(args.inventory) + if channel and channel != CHANNEL: + fail("%s names channel %s and this script reads %s" + % (args.inventory, channel, CHANNEL)) + + listed, tab_of = {}, {} + counts = {} + for tab in TABS: + entries = read_tab(yt_dlp, CHANNEL, tab) + counts[tab] = len(entries) + for video_id, title in entries: + listed.setdefault(video_id, title) + tab_of.setdefault(video_id, tab) + + print("channel %s: %s, %s distinct; inventory: %s" + % (CHANNEL, + ", ".join("%d in %s" % (counts[tab], tab) for tab in TABS), + len(listed), "%d entries" % len(inventory))) + + new = sorted((video_id for video_id in listed if video_id not in inventory), + key=lambda video_id: (tab_of[video_id], listed[video_id])) + unlisted = sorted(video_id for video_id, entry in inventory.items() + if video_id not in listed and entry.get("unlisted")) + gone = sorted(video_id for video_id, entry in inventory.items() + if video_id not in listed and not entry.get("unlisted")) + surfaced = sorted(video_id for video_id, entry in inventory.items() + if video_id in listed and entry.get("unlisted")) + renamed = sorted(video_id for video_id, entry in inventory.items() + if video_id in listed and listed[video_id] + and str(entry.get("title") or "").strip() != listed[video_id]) + + print() + print("%s in the channel and not in the inventory: write the link on the " + "talk of its event page, then add the entry" + % plural(len(new), "video")) + for video_id in new: + print(" NEW %-12s %-8s %s" % (video_id, tab_of[video_id], listed[video_id])) + print(" https://www.youtube.com/watch?v=%s" % video_id) + if args.yaml and new: + print() + print("entries to paste under videos: in %s" % args.inventory) + for video_id in new: + print(entry_yaml(video_id, listed[video_id])) + + print() + print("%s in the inventory and in no tab, marked unlisted, which is expected" + % plural(len(unlisted), "video")) + for video_id in unlisted: + print(" unlisted %-12s %s" % (video_id, inventory[video_id].get("title"))) + + print("%s in the inventory and in no tab, not marked unlisted: removed from " + "the channel, or made unlisted; open the URL and decide" + % plural(len(gone), "video")) + for video_id in gone: + print(" GONE %-12s %s" % (video_id, inventory[video_id].get("title"))) + print(" https://www.youtube.com/watch?v=%s" % video_id) + + print("%s marked unlisted that a tab now shows: drop the unlisted mark" + % plural(len(surfaced), "video")) + for video_id in surfaced: + print(" PUBLIC %-12s %-8s %s" % (video_id, tab_of[video_id], listed[video_id])) + + print("%s whose title the channel changed: copy the new title into the " + "inventory" % plural(len(renamed), "video")) + for video_id in renamed: + print(" RENAMED %-12s inventory: %s" % (video_id, inventory[video_id].get("title"))) + print(" channel: %s" % listed[video_id]) + + return 1 if (new or gone or surfaced or renamed) else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 496ac4ba50bb551e9b4286c181dc946a46137ea8 Mon Sep 17 00:00:00 2001 From: David Arcos Date: Thu, 15 Oct 2026 09:59:38 +0200 Subject: [PATCH 4/5] Read a content adapter in check-html-safety, and cap an imported photo at 1000px bin/check-html-safety adds .gotmpl to the suffixes it scans. A content adapter, content/
/_content.gotmpl, is a template Hugo runs while it assembles the content, and it can write a page body through the content key of AddPage, which goes through goldmark like the body of a markdown file. No adapter is in the tree yet; the scan is ready for the first one. bin/fit-sources adds IMPORTED_PHOTO_CAP = 1000 for the photographs the archive imports. The gallery never shows one wider than 900px and the original is one click away in the post each one credits, so 1000 covers that at full detail and leaves the repository a fifth lighter. Its docstring also says that orpheus.png has been cut since the paragraph about it was written: 1120x269 and 16 KB, and a dry run over the set reports nothing. --- bin/check-html-safety | 7 +++++++ bin/fit-sources | 25 ++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/bin/check-html-safety b/bin/check-html-safety index f94680469..8060333ba 100755 --- a/bin/check-html-safety +++ b/bin/check-html-safety @@ -55,6 +55,13 @@ TEXT_SUFFIXES = ( ".pandoc", ".pdc", # text/pandoc ".txt", # text/plain, copied as a page resource ".svg", # image/svg+xml, XML that can carry