diff --git a/annotations/batches/batch-07.json b/annotations/batches/batch-07.json new file mode 100644 index 0000000..51a4a34 --- /dev/null +++ b/annotations/batches/batch-07.json @@ -0,0 +1,1121 @@ +{ + "counts": { + "natural": 100 + }, + "items": [ + { + "advisory_id": "GHSA-mpf5-3vph-q75r", + "condition_score": 0, + "content_hash": "7751f71aa5dbcb7df0f53da5e4b6a7ea3544f26a63fd0deb99c2240c828f961a", + "intended_use": "evaluation", + "leakage_group": "GHSA-2p9g-x3cv-5hh4", + "order": 0, + "pool": "dev", + "stratum": "natural", + "words": 56 + }, + { + "advisory_id": "GHSA-frgf-rv99-862x", + "condition_score": 1, + "content_hash": "d85c6d1a77826c01de8f4d06f8ced3be5e139651300130e9f7d657e81c13680f", + "intended_use": "evaluation", + "leakage_group": "GHSA-5jq3-8437-x35p", + "order": 1, + "pool": "dev", + "stratum": "natural", + "words": 33 + }, + { + "advisory_id": "GHSA-vfg9-phjp-9frw", + "condition_score": 1, + "content_hash": "619053a1ed222f0081815129569b7cc83c240512df39d74de9ff34f78c902514", + "intended_use": "evaluation", + "leakage_group": "GHSA-6fgp-29mf-chhc", + "order": 2, + "pool": "dev", + "stratum": "natural", + "words": 31 + }, + { + "advisory_id": "GHSA-mq26-g339-26xf", + "condition_score": 4, + "content_hash": "5a3d3b1a2ea1bec9cd25c5ace8996d7fcde08fc615ab9ca54c589db2d0015605", + "intended_use": "evaluation", + "leakage_group": "GHSA-27x4-j476-jp5f", + "order": 3, + "pool": "dev", + "stratum": "natural", + "words": 68 + }, + { + "advisory_id": "GHSA-29gw-9793-fvw7", + "condition_score": 3, + "content_hash": "3ecc99d6c0ac25e2a13312b5d52fc9bdf1e51d57facc4950e6c00dc2bd9c0637", + "intended_use": "evaluation", + "leakage_group": "GHSA-24qx-w28j-9m6p", + "order": 4, + "pool": "dev", + "stratum": "natural", + "words": 150 + }, + { + "advisory_id": "GHSA-m87f-39q9-6f55", + "condition_score": 4, + "content_hash": "d6294b7066b33a1d7610149fa987b6773c5c85dbc84548a969a8cf99861e6c93", + "intended_use": "evaluation", + "leakage_group": "GHSA-24qx-w28j-9m6p", + "order": 5, + "pool": "dev", + "stratum": "natural", + "words": 91 + }, + { + "advisory_id": "GHSA-pwgv-4x5q-6m9f", + "condition_score": 9, + "content_hash": "6017e184b19eedc91dced2407a913063ccf173569f73b90df221f68d70447db7", + "intended_use": "evaluation", + "leakage_group": "GHSA-27jp-wm6q-gp25", + "order": 6, + "pool": "dev", + "stratum": "natural", + "words": 1380 + }, + { + "advisory_id": "GHSA-9x4q-3gxw-849f", + "condition_score": 3, + "content_hash": "9c623dca8977954247756cd8e438a85e8dcbcb03f54bdc20e3a01daa4ce1facf", + "intended_use": "evaluation", + "leakage_group": "GHSA-24qx-w28j-9m6p", + "order": 7, + "pool": "dev", + "stratum": "natural", + "words": 165 + }, + { + "advisory_id": "GHSA-2vpq-fh52-j3wv", + "condition_score": 2, + "content_hash": "63786fce5ae3d8c2b25e3e01b7e91e6f1562547f6b8ec1f5908fc07726c95206", + "intended_use": "evaluation", + "leakage_group": "GHSA-2vpq-fh52-j3wv", + "order": 8, + "pool": "dev", + "stratum": "natural", + "words": 154 + }, + { + "advisory_id": "GHSA-f66q-9rf6-8795", + "condition_score": 10, + "content_hash": "5971f4bfb1ab39a080d709ae50740143bd2b0012ab6bfe0757b507fad26c7224", + "intended_use": "evaluation", + "leakage_group": "GHSA-672h-6x89-76m5", + "order": 9, + "pool": "dev", + "stratum": "natural", + "words": 2916 + }, + { + "advisory_id": "GHSA-rjxg-rpg3-9r89", + "condition_score": 2, + "content_hash": "fa90ad965ff39cb6d1d4c5dac5df52b1d5e29dc3c7a4cf90bb4c4326d6e2b206", + "intended_use": "evaluation", + "leakage_group": "GHSA-2h46-8gf5-fmxv", + "order": 10, + "pool": "dev", + "stratum": "natural", + "words": 147 + }, + { + "advisory_id": "GHSA-555p-6grf-mh7f", + "condition_score": 0, + "content_hash": "158f2b919cfc41827f933bb043d5f5aff495b71714db123b9679961c59d818ca", + "intended_use": "evaluation", + "leakage_group": "GHSA-35mr-4567-66vg", + "order": 11, + "pool": "dev", + "stratum": "natural", + "words": 350 + }, + { + "advisory_id": "GHSA-h9fp-p2p9-873q", + "condition_score": 6, + "content_hash": "2aa4a0eba9bf5195ebba891864e7bd7c7e179c721fa7d8d70215e41cdceab6d8", + "intended_use": "evaluation", + "leakage_group": "GHSA-38wq-6q2w-hcf9", + "order": 12, + "pool": "dev", + "stratum": "natural", + "words": 422 + }, + { + "advisory_id": "GHSA-m67m-3p5g-cw9j", + "condition_score": 7, + "content_hash": "9c5dde3e5af2bc771c428095f33369ae9e9ef58856890819bea99803815ef3e0", + "intended_use": "evaluation", + "leakage_group": "GHSA-2p9g-x3cv-5hh4", + "order": 13, + "pool": "dev", + "stratum": "natural", + "words": 323 + }, + { + "advisory_id": "GHSA-8f8m-wrvr-wcvf", + "condition_score": 0, + "content_hash": "624eb8109929f8e315b65431c922e1fe4c64ed75e71255660a1f2b5231f8c692", + "intended_use": "evaluation", + "leakage_group": "GHSA-22jm-4hxw-35jf", + "order": 14, + "pool": "dev", + "stratum": "natural", + "words": 123 + }, + { + "advisory_id": "GHSA-vc3v-ppc7-v486", + "condition_score": 3, + "content_hash": "5c8158b196498f068252c26ab381a1439128bba99899404ac69191829c3dd062", + "intended_use": "evaluation", + "leakage_group": "GHSA-2wgc-48g2-cj5w", + "order": 15, + "pool": "dev", + "stratum": "natural", + "words": 90 + }, + { + "advisory_id": "GHSA-cw7j-v52w-fp5r", + "condition_score": 3, + "content_hash": "3059e99a8be028a17b855b2565ea542438a7406e3d95038d0d017cd87cd12e26", + "intended_use": "evaluation", + "leakage_group": "GHSA-5662-2rj7-f2v6", + "order": 16, + "pool": "dev", + "stratum": "natural", + "words": 174 + }, + { + "advisory_id": "GHSA-9xwg-3r6f-jcx2", + "condition_score": 8, + "content_hash": "b6fc1d8a28435d808c6a1de13fe2c75acec0a19a78d6d0a604168e6334109073", + "intended_use": "evaluation", + "leakage_group": "GHSA-62q4-447f-wv8h", + "order": 17, + "pool": "dev", + "stratum": "natural", + "words": 497 + }, + { + "advisory_id": "GHSA-43jf-985q-588j", + "condition_score": 0, + "content_hash": "109c7ba03c2705be13b01c00f95a6894924c9c6467ee1c877357ada97e764860", + "intended_use": "evaluation", + "leakage_group": "GHSA-23hm-7w47-xw72", + "order": 18, + "pool": "dev", + "stratum": "natural", + "words": 104 + }, + { + "advisory_id": "GHSA-vp96-hxj8-p424", + "condition_score": 4, + "content_hash": "c44492bc9a531e8d7af874721a61651cfd051c97009a57bc45de019ed2e305ce", + "intended_use": "evaluation", + "leakage_group": "GHSA-2rcm-phc9-3945", + "order": 19, + "pool": "dev", + "stratum": "natural", + "words": 63 + }, + { + "advisory_id": "GHSA-fxff-wxxv-c2jc", + "condition_score": 0, + "content_hash": "db20e6ebec45798e5cb26ed0cc8af5a0acd36e4fe5ebd1b52ae877a699d718dc", + "intended_use": "evaluation", + "leakage_group": "GHSA-fxff-wxxv-c2jc", + "order": 20, + "pool": "dev", + "stratum": "natural", + "words": 44 + }, + { + "advisory_id": "GHSA-g9wf-5777-gq43", + "condition_score": 7, + "content_hash": "0dbc3fd34c863de5687c95bb01164d21a4b9cd4ed4303dcb1e6ac5116e61bc2c", + "intended_use": "evaluation", + "leakage_group": "GHSA-c87f-fq5g-63r2", + "order": 21, + "pool": "dev", + "stratum": "natural", + "words": 1488 + }, + { + "advisory_id": "GHSA-756h-r2c9-qp5j", + "condition_score": 1, + "content_hash": "91c781afcd6fc5005be25381d1594a607603787bd9dc2e3bb8784fec4acac482", + "intended_use": "evaluation", + "leakage_group": "GHSA-756h-r2c9-qp5j", + "order": 22, + "pool": "dev", + "stratum": "natural", + "words": 27 + }, + { + "advisory_id": "GHSA-h2fp-xgx6-xh6f", + "condition_score": 2, + "content_hash": "8187b3e511a990b7cea7db18654bbaf3b21dfd022d731cf5c85ad5b323b8a5af", + "intended_use": "evaluation", + "leakage_group": "GHSA-29vq-49wr-vm6x", + "order": 23, + "pool": "dev", + "stratum": "natural", + "words": 49 + }, + { + "advisory_id": "GHSA-7xr3-6ggc-wc9p", + "condition_score": 2, + "content_hash": "dd8b9de730d863ba644f89659d85b349c26796b54483615339e5e4525a7786d3", + "intended_use": "evaluation", + "leakage_group": "GHSA-7xr3-6ggc-wc9p", + "order": 24, + "pool": "dev", + "stratum": "natural", + "words": 71 + }, + { + "advisory_id": "GHSA-8c4w-v65p-jvcv", + "condition_score": 3, + "content_hash": "fe3d8370015d44f018fa0aeb7e5581145b76e7c6d6bf74dec255f1c553a94239", + "intended_use": "evaluation", + "leakage_group": "GHSA-22jm-4hxw-35jf", + "order": 25, + "pool": "dev", + "stratum": "natural", + "words": 72 + }, + { + "advisory_id": "GHSA-8qxj-f9rh-9fg2", + "condition_score": 0, + "content_hash": "1e15f59edda6c140925682894f3fc9c1c889ea26ce3c27a818853249a8b41344", + "intended_use": "evaluation", + "leakage_group": "GHSA-8qxj-f9rh-9fg2", + "order": 26, + "pool": "dev", + "stratum": "natural", + "words": 56 + }, + { + "advisory_id": "GHSA-pw27-w7w4-9qc7", + "condition_score": 1, + "content_hash": "0827c617a88fa8e7457d930e4facb862333472f4c2ccf486f7b54eec86877d77", + "intended_use": "evaluation", + "leakage_group": "GHSA-2655-q453-22f9", + "order": 27, + "pool": "dev", + "stratum": "natural", + "words": 45 + }, + { + "advisory_id": "GHSA-xgxc-v2qg-chmh", + "condition_score": 1, + "content_hash": "ddda3f85636b52bd5c35bf87498e918fbb5fb80250de498cd995359423631079", + "intended_use": "evaluation", + "leakage_group": "GHSA-2655-q453-22f9", + "order": 28, + "pool": "dev", + "stratum": "natural", + "words": 37 + }, + { + "advisory_id": "GHSA-7wqf-h36w-47mc", + "condition_score": 3, + "content_hash": "80e87eafd1236919742575d34508897c4719a132aeecb422435afecf5c24563f", + "intended_use": "evaluation", + "leakage_group": "GHSA-22hf-vx2v-gjff", + "order": 29, + "pool": "dev", + "stratum": "natural", + "words": 112 + }, + { + "advisory_id": "GHSA-966j-vmvw-g2g9", + "condition_score": 2, + "content_hash": "32f0d77aa912bd24eccbe06dea3880c9905de4dbd126fa71169dc23e33b10574", + "intended_use": "evaluation", + "leakage_group": "GHSA-27mf-ghqm-j3j8", + "order": 30, + "pool": "dev", + "stratum": "natural", + "words": 56 + }, + { + "advisory_id": "GHSA-gjrg-mpp7-g774", + "condition_score": 3, + "content_hash": "22671cc586f3457a3ade903e330c67935c75cfb85ad063b6a1f29a9c0db68c0b", + "intended_use": "evaluation", + "leakage_group": "GHSA-gjrg-mpp7-g774", + "order": 31, + "pool": "dev", + "stratum": "natural", + "words": 193 + }, + { + "advisory_id": "GHSA-jqqh-999x-w26w", + "condition_score": 1, + "content_hash": "01f4b4dba439b736f7c0c55ee546d3076af1cfedf108086efec3b7493313b20e", + "intended_use": "evaluation", + "leakage_group": "GHSA-66x7-2r56-fj77", + "order": 32, + "pool": "dev", + "stratum": "natural", + "words": 34 + }, + { + "advisory_id": "GHSA-p28x-hj68-7vfp", + "condition_score": 1, + "content_hash": "7d814e77c7920336fa335895abbdf0a9c8dcd240baef21bc0078e0b0df76709b", + "intended_use": "evaluation", + "leakage_group": "GHSA-4987-5p3p-9r27", + "order": 33, + "pool": "dev", + "stratum": "natural", + "words": 30 + }, + { + "advisory_id": "GHSA-j257-jfvv-h3x5", + "condition_score": 4, + "content_hash": "ff87668d96618ae336c540177c3e9372df6441975c5d82637f40028362ae4007", + "intended_use": "evaluation", + "leakage_group": "GHSA-23c7-6444-399m", + "order": 34, + "pool": "dev", + "stratum": "natural", + "words": 51 + }, + { + "advisory_id": "GHSA-jrm8-xgf3-fwqr", + "condition_score": 3, + "content_hash": "50516a06d4cd1ea01997322103a1d7c604825a01fa8cdeb7514ad30265ff00dd", + "intended_use": "evaluation", + "leakage_group": "GHSA-3p54-567p-2wpr", + "order": 35, + "pool": "dev", + "stratum": "natural", + "words": 345 + }, + { + "advisory_id": "GHSA-8wwf-2644-f8x4", + "condition_score": 0, + "content_hash": "ab1762ebad1a1afc815342dcddbc700e3a5c229f47a81f9500a7c3a9c9105fed", + "intended_use": "evaluation", + "leakage_group": "GHSA-8wwf-2644-f8x4", + "order": 36, + "pool": "dev", + "stratum": "natural", + "words": 43 + }, + { + "advisory_id": "GHSA-r8g9-w4f3-9crm", + "condition_score": 0, + "content_hash": "29ed6080135d73c68b3470f67b2cefa5b761ff232e7866421f04f1974fb70cd9", + "intended_use": "evaluation", + "leakage_group": "GHSA-9q62-r72g-pvv7", + "order": 37, + "pool": "dev", + "stratum": "natural", + "words": 29 + }, + { + "advisory_id": "GHSA-mxvw-cj37-8g2h", + "condition_score": 0, + "content_hash": "01c8a0f25738379db1af9fe9b38324edfbcb1d2e0bb20dd299c3151d8031b664", + "intended_use": "evaluation", + "leakage_group": "GHSA-35p3-6j45-prwm", + "order": 38, + "pool": "dev", + "stratum": "natural", + "words": 79 + }, + { + "advisory_id": "GHSA-8434-v7xw-8m9x", + "condition_score": 3, + "content_hash": "b695e2b7809577e82dc72a9aae38be937b42c9c2bb93c1ab5e6934f5a2b3e0ec", + "intended_use": "evaluation", + "leakage_group": "GHSA-8434-v7xw-8m9x", + "order": 39, + "pool": "dev", + "stratum": "natural", + "words": 90 + }, + { + "advisory_id": "GHSA-rpc6-h455-3rx5", + "condition_score": 1, + "content_hash": "8889ecbf85aa594679dde508c9247792190c6b9ed1a4f5da1e3b209228e5d446", + "intended_use": "evaluation", + "leakage_group": "GHSA-q4xr-rc97-m4xx", + "order": 40, + "pool": "dev", + "stratum": "natural", + "words": 60 + }, + { + "advisory_id": "GHSA-69w3-r845-3855", + "condition_score": 3, + "content_hash": "f38def764251901b662cce1f66a2620734f774744ba3697365d6556eb74ee7f7", + "intended_use": "evaluation", + "leakage_group": "GHSA-282v-666c-3fvg", + "order": 41, + "pool": "dev", + "stratum": "natural", + "words": 99 + }, + { + "advisory_id": "GHSA-r6h7-5pq2-j77h", + "condition_score": 0, + "content_hash": "e119504935330acd09ca273df7cca9d67608ebd9147e07c4abba9a8bf181a23c", + "intended_use": "evaluation", + "leakage_group": "GHSA-2pfh-q76x-gwvm", + "order": 42, + "pool": "dev", + "stratum": "natural", + "words": 69 + }, + { + "advisory_id": "GHSA-5wvp-7f3h-6wmm", + "condition_score": 4, + "content_hash": "35182efd918eab60944a618b619f1e40f8262bf614edc764506c80ec9609fc02", + "intended_use": "evaluation", + "leakage_group": "GHSA-5wvp-7f3h-6wmm", + "order": 43, + "pool": "dev", + "stratum": "natural", + "words": 128 + }, + { + "advisory_id": "GHSA-6667-f46p-pg88", + "condition_score": 1, + "content_hash": "d3dfe9420f5f636cac7b85e61ea30e8b5e607aa21e815312c52828b0bd315e9d", + "intended_use": "evaluation", + "leakage_group": "GHSA-2pfh-q76x-gwvm", + "order": 44, + "pool": "dev", + "stratum": "natural", + "words": 37 + }, + { + "advisory_id": "GHSA-p43w-g3c5-g5mq", + "condition_score": 0, + "content_hash": "d4082fafbc9e08145721286d9114162ee6c6f3d6def6f7728df3784dbe4ebec5", + "intended_use": "evaluation", + "leakage_group": "GHSA-3c5c-7235-994j", + "order": 45, + "pool": "dev", + "stratum": "natural", + "words": 21 + }, + { + "advisory_id": "GHSA-4vvc-r4p4-qgrr", + "condition_score": 0, + "content_hash": "e9a8d72e4d619f9fda11f794eea23d190687cde8c4e8a656236dda17bab28faf", + "intended_use": "evaluation", + "leakage_group": "GHSA-4vvc-r4p4-qgrr", + "order": 46, + "pool": "dev", + "stratum": "natural", + "words": 86 + }, + { + "advisory_id": "GHSA-pw3c-h7wp-cvhx", + "condition_score": 0, + "content_hash": "260db8a42e46b83e8e50885ccf4be20d1e443dfc34f333d7a4968963b74d11d5", + "intended_use": "evaluation", + "leakage_group": "GHSA-3c5c-7235-994j", + "order": 47, + "pool": "dev", + "stratum": "natural", + "words": 23 + }, + { + "advisory_id": "GHSA-ccgm-3xw4-h5p8", + "condition_score": 2, + "content_hash": "18ce893540062c667fa7f355fa78bc9fbec31973df0e89b19f837963458919a8", + "intended_use": "evaluation", + "leakage_group": "GHSA-ccgm-3xw4-h5p8", + "order": 48, + "pool": "dev", + "stratum": "natural", + "words": 26 + }, + { + "advisory_id": "GHSA-428g-f7cq-pgp5", + "condition_score": 2, + "content_hash": "b316f0ab6ef9754b6e43a955c308efa2006c1d62539d77458d994ed41c8f0531", + "intended_use": "evaluation", + "leakage_group": "GHSA-428g-f7cq-pgp5", + "order": 49, + "pool": "dev", + "stratum": "natural", + "words": 58 + }, + { + "advisory_id": "GHSA-4mqg-h5jf-j9m7", + "condition_score": 2, + "content_hash": "936b92cfaae2a6d55175b9ee9f173f7737df2d378185c601338259c74efb9640", + "intended_use": "evaluation", + "leakage_group": "GHSA-4mqg-h5jf-j9m7", + "order": 50, + "pool": "dev", + "stratum": "natural", + "words": 218 + }, + { + "advisory_id": "GHSA-2jg5-xgvv-4wq7", + "condition_score": 1, + "content_hash": "d065d53fc66dc618142f5e254a76a86b053c792707dd948f92391acdcd2c1acd", + "intended_use": "evaluation", + "leakage_group": "GHSA-2jg5-xgvv-4wq7", + "order": 51, + "pool": "dev", + "stratum": "natural", + "words": 71 + }, + { + "advisory_id": "GHSA-3pmh-24wp-xpf4", + "condition_score": 1, + "content_hash": "aec7050750b75ea10bb863522863fd22e2dc0b751d1529d8ec9f106543aebc61", + "intended_use": "evaluation", + "leakage_group": "GHSA-2p9g-x3cv-5hh4", + "order": 52, + "pool": "dev", + "stratum": "natural", + "words": 50 + }, + { + "advisory_id": "GHSA-rg9q-m8hv-xxr6", + "condition_score": 1, + "content_hash": "f7702a3a9c4d319cac3b6a2e722781850c7bffe28ef2c6f4cf97eded4949b8ad", + "intended_use": "evaluation", + "leakage_group": "GHSA-275c-w5mq-v5m2", + "order": 53, + "pool": "dev", + "stratum": "natural", + "words": 25 + }, + { + "advisory_id": "GHSA-4822-jvwx-w47h", + "condition_score": 4, + "content_hash": "1ad62d0473c9a664721d452d2e8ffa5d59e7447dbba6af75178e4aa2b9f75b00", + "intended_use": "evaluation", + "leakage_group": "GHSA-22p3-qrh9-cx32", + "order": 54, + "pool": "dev", + "stratum": "natural", + "words": 197 + }, + { + "advisory_id": "GHSA-mcpw-cp35-p3h8", + "condition_score": 0, + "content_hash": "debc8362d2f89b724dda81b5f1764c4a01b9affca186d2494778f23fbfee27bd", + "intended_use": "evaluation", + "leakage_group": "GHSA-mcpw-cp35-p3h8", + "order": 55, + "pool": "dev", + "stratum": "natural", + "words": 76 + }, + { + "advisory_id": "GHSA-hjr4-fhgp-23g9", + "condition_score": 0, + "content_hash": "8970718c053c277f7102616b5061235e03265c722fc8aad8135aaf8595d4cfbb", + "intended_use": "evaluation", + "leakage_group": "GHSA-hjr4-fhgp-23g9", + "order": 56, + "pool": "dev", + "stratum": "natural", + "words": 28 + }, + { + "advisory_id": "GHSA-63cw-57p8-fm3p", + "condition_score": 2, + "content_hash": "b052edc2cb89921db915b0332c18919e74b79f1d2fd8e419b91dffe4f90c5891", + "intended_use": "evaluation", + "leakage_group": "GHSA-2vj5-px25-gjrp", + "order": 57, + "pool": "dev", + "stratum": "natural", + "words": 116 + }, + { + "advisory_id": "GHSA-g95c-2jgm-hqc6", + "condition_score": 3, + "content_hash": "b981904b02da5182c36099d72d2c37d163e7e0bc214e33329e909cb9a3b037ed", + "intended_use": "evaluation", + "leakage_group": "GHSA-2h46-8gf5-fmxv", + "order": 58, + "pool": "dev", + "stratum": "natural", + "words": 164 + }, + { + "advisory_id": "GHSA-gw46-8559-cggp", + "condition_score": 1, + "content_hash": "0c3cd33a4b52010a2f810a014f0f4ef155324749b593f4968d9ba2e33b83f260", + "intended_use": "evaluation", + "leakage_group": "GHSA-3g56-2hh3-35ph", + "order": 59, + "pool": "dev", + "stratum": "natural", + "words": 39 + }, + { + "advisory_id": "GHSA-w446-h7vg-wv3p", + "condition_score": 3, + "content_hash": "3627948fd97b0f3ea2789426c61c7bf9b5c4e1c18b36bba4c689e4e6b5387931", + "intended_use": "evaluation", + "leakage_group": "GHSA-22jm-4hxw-35jf", + "order": 60, + "pool": "dev", + "stratum": "natural", + "words": 63 + }, + { + "advisory_id": "GHSA-4mrx-6fxm-8jpg", + "condition_score": 0, + "content_hash": "0db0b4d3bbaf3a137eb67933a4945026df4e5da1caa00d8f893f324be5008479", + "intended_use": "evaluation", + "leakage_group": "GHSA-22wc-c9wj-6q2v", + "order": 61, + "pool": "dev", + "stratum": "natural", + "words": 41 + }, + { + "advisory_id": "GHSA-jjw5-xxj6-pcv5", + "condition_score": 2, + "content_hash": "02b369dfe664760a01689047e7e3f9729869ceb4f92c11a7e9f1a9f50249b405", + "intended_use": "evaluation", + "leakage_group": "GHSA-jjw5-xxj6-pcv5", + "order": 62, + "pool": "dev", + "stratum": "natural", + "words": 60 + }, + { + "advisory_id": "GHSA-765j-9r45-w2q2", + "condition_score": 5, + "content_hash": "f79a96482691933b147014c9fd7594093ee7c8bc0d3fc852ed4eaef56b0a4a95", + "intended_use": "evaluation", + "leakage_group": "GHSA-2ccw-7px8-vmpf", + "order": 63, + "pool": "dev", + "stratum": "natural", + "words": 128 + }, + { + "advisory_id": "GHSA-4x4j-2g7c-83w6", + "condition_score": 3, + "content_hash": "57bae3e3a56e206397f7eaccce4998349a57a45b01efa661b5bb05d01ff7c01b", + "intended_use": "evaluation", + "leakage_group": "GHSA-3c5c-7235-994j", + "order": 64, + "pool": "dev", + "stratum": "natural", + "words": 503 + }, + { + "advisory_id": "GHSA-qhmp-h54x-38qr", + "condition_score": 4, + "content_hash": "4ee78bdd8f8f59e0132d59041c0556e26d956c6726012efe381f17a03b6d1c46", + "intended_use": "evaluation", + "leakage_group": "GHSA-856c-92hv-3vxx", + "order": 65, + "pool": "dev", + "stratum": "natural", + "words": 176 + }, + { + "advisory_id": "GHSA-q3qx-c6g2-7pw2", + "condition_score": 7, + "content_hash": "bcf408f6185b8afe5e482515e6fcdc939a9cf32a8c5add93db8e553fc1b92ae2", + "intended_use": "evaluation", + "leakage_group": "GHSA-27mf-ghqm-j3j8", + "order": 66, + "pool": "dev", + "stratum": "natural", + "words": 192 + }, + { + "advisory_id": "GHSA-7pq6-v88g-wf3w", + "condition_score": 3, + "content_hash": "31b7211900c61f89f2ca8b8430ca5028b095f15900abf911eb8b6265ee1ef7a3", + "intended_use": "evaluation", + "leakage_group": "GHSA-444r-2whx-3685", + "order": 67, + "pool": "dev", + "stratum": "natural", + "words": 123 + }, + { + "advisory_id": "GHSA-7wxf-r2qv-9xwr", + "condition_score": 4, + "content_hash": "5407bc24e42d759adee0d27cbb2ff6accbddb5dd62c0f5048af9a33e84810067", + "intended_use": "evaluation", + "leakage_group": "GHSA-7wxf-r2qv-9xwr", + "order": 68, + "pool": "dev", + "stratum": "natural", + "words": 105 + }, + { + "advisory_id": "GHSA-3qgw-p4fm-x7gf", + "condition_score": 0, + "content_hash": "bffb96f23173fcb42919bf71e49058a2fee71286e776285327b3f874e03d7e44", + "intended_use": "evaluation", + "leakage_group": "GHSA-23hm-7w47-xw72", + "order": 69, + "pool": "dev", + "stratum": "natural", + "words": 127 + }, + { + "advisory_id": "GHSA-p4v8-rw59-93cq", + "condition_score": 4, + "content_hash": "efb3ade14b3dbeba688a0e3d53d5380684a67da4fe9ce4cd9960cc7c17373a56", + "intended_use": "evaluation", + "leakage_group": "GHSA-23m2-mghx-vqmf", + "order": 70, + "pool": "dev", + "stratum": "natural", + "words": 142 + }, + { + "advisory_id": "GHSA-jh82-c5jw-pxpc", + "condition_score": 2, + "content_hash": "1dd735537b9e9ee95e912b2a66cdafd268bd0825404906e6981fcd5253092c82", + "intended_use": "evaluation", + "leakage_group": "GHSA-22p9-r2f5-22mf", + "order": 71, + "pool": "dev", + "stratum": "natural", + "words": 348 + }, + { + "advisory_id": "GHSA-mvf6-hwxh-7v76", + "condition_score": 0, + "content_hash": "292c76468fe4fc38943a3cc6fa806e1a0bbb7d7b891d846da3fa94368ca2fa2f", + "intended_use": "evaluation", + "leakage_group": "GHSA-mvf6-hwxh-7v76", + "order": 72, + "pool": "dev", + "stratum": "natural", + "words": 35 + }, + { + "advisory_id": "GHSA-mcmc-c59m-pqq8", + "condition_score": 9, + "content_hash": "163ddce4cb5f09b33008f9f5fad5ec4e7353a18fa01317355b656da69b543cd4", + "intended_use": "evaluation", + "leakage_group": "GHSA-hw9r-6m78-w6h3", + "order": 73, + "pool": "dev", + "stratum": "natural", + "words": 412 + }, + { + "advisory_id": "GHSA-6wcr-wcqm-3mfh", + "condition_score": 2, + "content_hash": "ba2f2b855a7c2d9f0faa380850d5130fd2c0576fe585bdc8b8adf951e7a74772", + "intended_use": "evaluation", + "leakage_group": "GHSA-2655-q453-22f9", + "order": 74, + "pool": "dev", + "stratum": "natural", + "words": 49 + }, + { + "advisory_id": "GHSA-67cx-rhhq-mfhq", + "condition_score": 6, + "content_hash": "c92dbabafefffe0299eb2fd5a90223044493194e4055dd1dd35da9ba69701ba7", + "intended_use": "evaluation", + "leakage_group": "GHSA-2v95-h47v-g4x9", + "order": 75, + "pool": "dev", + "stratum": "natural", + "words": 165 + }, + { + "advisory_id": "GHSA-373w-rj84-pv6x", + "condition_score": 3, + "content_hash": "784c26aac5f8cdba9d2a5c89caf6e1a3accc884071545b63741c95068098c1bb", + "intended_use": "evaluation", + "leakage_group": "GHSA-373w-rj84-pv6x", + "order": 76, + "pool": "dev", + "stratum": "natural", + "words": 94 + }, + { + "advisory_id": "GHSA-3j5x-7ccf-ppgm", + "condition_score": 2, + "content_hash": "b591a67a21d03b834698f2e6478fba68866b2763d42fa8fa298bedec69ae7fb4", + "intended_use": "evaluation", + "leakage_group": "GHSA-3j5x-7ccf-ppgm", + "order": 77, + "pool": "dev", + "stratum": "natural", + "words": 41 + }, + { + "advisory_id": "GHSA-5rp4-749p-vx26", + "condition_score": 0, + "content_hash": "1e71e2c63814d7f40caf3f80e1034b9567da012c4105ffdff9c108f8250995b5", + "intended_use": "evaluation", + "leakage_group": "GHSA-22hf-vx2v-gjff", + "order": 78, + "pool": "dev", + "stratum": "natural", + "words": 27 + }, + { + "advisory_id": "GHSA-7xxv-wpxj-mx5v", + "condition_score": 0, + "content_hash": "6c65f7413554fd7cb6e032d32b84266511bde240e54a191a67b36b0ac94d6efd", + "intended_use": "evaluation", + "leakage_group": "GHSA-7xxv-wpxj-mx5v", + "order": 79, + "pool": "dev", + "stratum": "natural", + "words": 66 + }, + { + "advisory_id": "GHSA-q9h8-gpw5-c95c", + "condition_score": 3, + "content_hash": "ca1ca6294625c9235f006f22f243e41e2c2146cd3dcb7e57108dacf89de08c0b", + "intended_use": "evaluation", + "leakage_group": "GHSA-22p3-qrh9-cx32", + "order": 80, + "pool": "dev", + "stratum": "natural", + "words": 45 + }, + { + "advisory_id": "GHSA-2q8v-3gqq-4f8p", + "condition_score": 6, + "content_hash": "579ee3ff48a8532b765e34d7387bbab20d7b456d2511b97065f5d37cca0753a2", + "intended_use": "evaluation", + "leakage_group": "GHSA-22wc-c9wj-6q2v", + "order": 81, + "pool": "dev", + "stratum": "natural", + "words": 811 + }, + { + "advisory_id": "GHSA-cj7g-h7rf-h8j9", + "condition_score": 4, + "content_hash": "171f5f0c2bbb3e42c5b8d1328d6e601d94293ccea12a6baf4b6a159ce20d33b6", + "intended_use": "evaluation", + "leakage_group": "GHSA-299q-3p96-5898", + "order": 82, + "pool": "dev", + "stratum": "natural", + "words": 122 + }, + { + "advisory_id": "GHSA-5jrq-582v-9767", + "condition_score": 0, + "content_hash": "3b1b91294928e578ba31c368d91250fb4b103e275de93f408b8556224c3321a2", + "intended_use": "evaluation", + "leakage_group": "GHSA-5jrq-582v-9767", + "order": 83, + "pool": "dev", + "stratum": "natural", + "words": 21 + }, + { + "advisory_id": "GHSA-v4w5-p2hg-8fh6", + "condition_score": 1, + "content_hash": "6cd97425cb954767063747b460850bce86c74163b58fe6a24ba16232b0bcbdf3", + "intended_use": "evaluation", + "leakage_group": "GHSA-2xpw-w6gg-jr37", + "order": 84, + "pool": "dev", + "stratum": "natural", + "words": 100 + }, + { + "advisory_id": "GHSA-jjjr-3jcw-f8v6", + "condition_score": 2, + "content_hash": "6012e0fcf7747ee4e908d6a21b9d2eb4deab8be1d4fd86df1fcb16878e26fc8d", + "intended_use": "evaluation", + "leakage_group": "GHSA-23m2-mghx-vqmf", + "order": 85, + "pool": "dev", + "stratum": "natural", + "words": 165 + }, + { + "advisory_id": "GHSA-3vpf-mcj7-5h38", + "condition_score": 5, + "content_hash": "1d031f50076f1016591829a12803a3704815a53f8b72d718a793c61ab0bab485", + "intended_use": "evaluation", + "leakage_group": "GHSA-2h46-8gf5-fmxv", + "order": 86, + "pool": "dev", + "stratum": "natural", + "words": 269 + }, + { + "advisory_id": "GHSA-qjmc-wwmw-cq9r", + "condition_score": 0, + "content_hash": "9b11d3c6205c9dd5adcd378a01b9adc85feaa87914e3582a6348aa4d872534ac", + "intended_use": "evaluation", + "leakage_group": "GHSA-2w93-qwpp-vgvj", + "order": 87, + "pool": "dev", + "stratum": "natural", + "words": 35 + }, + { + "advisory_id": "GHSA-4j5j-58j7-6c3w", + "condition_score": 3, + "content_hash": "778239d74aadad285aea50e003a21ccbf51397a63b7c13bc4aa3e4b1c68fbf0d", + "intended_use": "evaluation", + "leakage_group": "GHSA-35mr-4567-66vg", + "order": 88, + "pool": "dev", + "stratum": "natural", + "words": 51 + }, + { + "advisory_id": "GHSA-5jc6-h9w7-jm3p", + "condition_score": 6, + "content_hash": "57456000bf2605b2d39dc2a3547d26dbb8f3d8ba128affd56d16300a8f8fd3e6", + "intended_use": "evaluation", + "leakage_group": "GHSA-3p54-567p-2wpr", + "order": 89, + "pool": "dev", + "stratum": "natural", + "words": 328 + }, + { + "advisory_id": "GHSA-7c2f-r6gc-h92h", + "condition_score": 3, + "content_hash": "435123770617aa7649430c53ef3af6aa9187c870b5ceded0f63ab74765de40aa", + "intended_use": "evaluation", + "leakage_group": "GHSA-22hf-vx2v-gjff", + "order": 90, + "pool": "dev", + "stratum": "natural", + "words": 98 + }, + { + "advisory_id": "GHSA-hxxf-235m-72v3", + "condition_score": 2, + "content_hash": "eadd27198c68dc62878d1561b1acc6c011765193306ce198334aa29cb1241918", + "intended_use": "evaluation", + "leakage_group": "GHSA-282v-666c-3fvg", + "order": 91, + "pool": "dev", + "stratum": "natural", + "words": 107 + }, + { + "advisory_id": "GHSA-vp6v-whfm-rv3g", + "condition_score": 3, + "content_hash": "d3b58b11d93ff42fa9d725cf220956e156f86bc4c8069ce4d4036dab5c433c2b", + "intended_use": "evaluation", + "leakage_group": "GHSA-22p3-qrh9-cx32", + "order": 92, + "pool": "dev", + "stratum": "natural", + "words": 243 + }, + { + "advisory_id": "GHSA-9x6q-5423-w5v9", + "condition_score": 1, + "content_hash": "f254bb806b4de9e178cbf04265b2251bd50bc50cd433706120b64015e544ee24", + "intended_use": "evaluation", + "leakage_group": "GHSA-2pfh-q76x-gwvm", + "order": 93, + "pool": "dev", + "stratum": "natural", + "words": 28 + }, + { + "advisory_id": "GHSA-63xm-rx5p-xvqr", + "condition_score": 0, + "content_hash": "9e31d6d602b871ea302d536920c7b690e126ba97ef28a16824afa1d4ac092604", + "intended_use": "evaluation", + "leakage_group": "GHSA-23hm-7w47-xw72", + "order": 94, + "pool": "dev", + "stratum": "natural", + "words": 111 + }, + { + "advisory_id": "GHSA-2ch8-f849-pjg3", + "condition_score": 1, + "content_hash": "476eb47d24e4dc0c9512ce4bd8823f126969d9372b503c38c9a535649e2f2af7", + "intended_use": "evaluation", + "leakage_group": "GHSA-2ch8-f849-pjg3", + "order": 95, + "pool": "dev", + "stratum": "natural", + "words": 49 + }, + { + "advisory_id": "GHSA-5m98-qgg9-wh84", + "condition_score": 6, + "content_hash": "deed10580a94ea694b8b25287ba311cf8af5e1aab66aa0a4748f023fcafb444b", + "intended_use": "evaluation", + "leakage_group": "GHSA-27mf-ghqm-j3j8", + "order": 96, + "pool": "dev", + "stratum": "natural", + "words": 167 + }, + { + "advisory_id": "GHSA-fpmr-qmgh-42x2", + "condition_score": 3, + "content_hash": "544f501faf697889e1e6afc96d4373fe126e52917ca05fb74a2010216bdf8b75", + "intended_use": "evaluation", + "leakage_group": "GHSA-299q-3p96-5898", + "order": 97, + "pool": "dev", + "stratum": "natural", + "words": 54 + }, + { + "advisory_id": "GHSA-vxq7-64xx-v4gw", + "condition_score": 4, + "content_hash": "fbf96c3166cdccff60aef935a38cffc26584b5aec9156c197a53796cd30a5695", + "intended_use": "evaluation", + "leakage_group": "GHSA-2xpw-w6gg-jr37", + "order": 98, + "pool": "dev", + "stratum": "natural", + "words": 240 + }, + { + "advisory_id": "GHSA-3m2g-v7jf-7fxc", + "condition_score": 3, + "content_hash": "2ae7d9f14a0fb1b2f057eacfc4656364d6a17476e8e8398d33826dbd79a246d9", + "intended_use": "evaluation", + "leakage_group": "GHSA-299q-3p96-5898", + "order": 99, + "pool": "dev", + "stratum": "natural", + "words": 88 + } + ], + "params": { + "max_words": null, + "n": 100, + "name": "batch-07", + "natural_share": 0.3, + "per_group_cap": 3, + "pool": "dev", + "seed": 20261020, + "strategy": "natural" + }, + "sample_version": "batch-07", + "source_commit": "1912cbd6ce9d4cb141636adc4abafb4eb77a93e6", + "split_assignment_sha256": "41d7354e1a13378667dc5cda74e3ed932f770af94cbde6e12460863981290067", + "split_version": "pool-v1" +} diff --git a/configs/experiments/ablations.toml b/configs/experiments/ablations.toml index 353d7a5..6b0c898 100644 --- a/configs/experiments/ablations.toml +++ b/configs/experiments/ablations.toml @@ -1,16 +1,27 @@ -# Ablations and learning curves on the chosen backbone, one factor at a time, three seeds -# each. Set base_model/revision to the winner of the DistilBERT vs DeBERTa comparison. -# uv run triggermap experiment --config configs/experiments/ablations.toml --gold +# Ablations and learning curve for DeBERTa-v3-small, one factor at a time, three seeds each. +# The recipe is the best grid cell of neural run 3 (sqrt span-tag weights, 5e-5, up to 20 +# epochs, no early stop before epoch 6). Run it on a GPU: +# CONFIGS="ablations" bash scripts/gpu/run_neural.sh [train] -model_id = "ablation" +model_id = "ablation-1" base_model = "microsoft/deberta-v3-small" revision = "a36c739020e01763fe789b4b85e2df55d6180012" +max_length = 512 +stride = 64 +context_tokens = 48 +use_context = true +group_loss_weight = 1.0 +bio_weighting = "sqrt" learning_rate = 5e-5 -epochs = 6 +epochs = 20 batch_size = 8 gradient_accumulation = 2 -patience = 2 +warmup_ratio = 0.1 +weight_decay = 0.01 +patience = 3 +min_epochs = 6 +device = "auto" mlflow_experiment = "triggermap-ablations" [grid] @@ -28,9 +39,9 @@ name = "no-group" # group objective removed group_loss_weight = 0.0 [[runs]] -name = "docs-125" # learning curve -max_train_documents = 125 +name = "docs-60" # learning curve (the train split has 239 documents) +max_train_documents = 60 [[runs]] -name = "docs-250" -max_train_documents = 250 +name = "docs-120" +max_train_documents = 120 diff --git a/docs/annotation_plan.md b/docs/annotation_plan.md index fe6f14c..7f848a2 100644 --- a/docs/annotation_plan.md +++ b/docs/annotation_plan.md @@ -13,8 +13,9 @@ in `splits/pool-v1.json` (see [`splits.md`](splits.md)). Every batch is committe | `batches/batch-04.json` | test | 138 | natural, ≤ 2 per leakage group | test | **last**, after guidelines are frozen | | `batches/batch-05.json` | train | 261 | 183 uncertainty from `crf-batches-01-02` + 78 natural, ≤ 3 per leakage group | train | next (drawn 2026-11-05) | | `batches/batch-06.json` | dev | 60 | natural, ≤ 3 per leakage group | dev | done under v0.2 (`batch-06`; model drafts, no `manual` records) | +| `batches/batch-07.json` | dev | 100 | natural, ≤ 3 per leakage group | dev (calibration, thresholds, regression checks) | by hand, before batch-04 (drawn 2026-11-07) | -Totals when complete: train 39 + 100 + 100 + 261 = 500; dev 38 + 112 + 60 = 210; +Totals when complete: train 39 + 100 + 100 + 261 = 500; dev 38 + 112 + 60 + 100 = 310; test 12 + 138 = 150. Batch-06 was added to give dev hand-labelled documents, but it was also submitted from model drafts; only 10 dev documents are `manual`. @@ -75,6 +76,24 @@ from model drafts. Batch-06 is 60 natural dev documents for hand labelling. Its Smaller sittings are fine: a partial export can be imported, and each later import replaces the earlier file. +## Hand-labelled dev and test batches (batch-07, batch-04) + +The release gates in [`targets.md`](targets.md) are judged on `manual` records only. Two batches +make them judgeable, labelled by hand with the procedure above: + +1. **`batch-07` (100 dev documents) first.** Calibration, review thresholds and + release-to-release regression checks are fitted and measured on hand-labelled dev records. + There are only 10 so far. +2. **`batch-04` (138 test documents) last.** With the 4 hand-labelled pilot test records, this + gives 142 test documents. Nobody tunes anything on them. They are scored once, in a declared + release evaluation that is logged in [`test-access-log.md`](test-access-log.md). + +Each batch gets its own new Label Studio project, created with no predictions and no model +backend. Task files: `data/label_studio/batch-07_tasks.json` and +`data/label_studio/batch-04_tasks.json` (`annotate tasks --sample …`, advisory text only). +Import with `--annotation-version batch-07` or `batch-04`. Every record must come out +`manual`. + ## Batch-05 (uncertainty sample) `batch-05` draws 261 unsampled train-pool documents: 183 where `crf-batches-01-02` is least diff --git a/docs/training_on_gpu.md b/docs/training_on_gpu.md index 854d9f8..44ab504 100644 --- a/docs/training_on_gpu.md +++ b/docs/training_on_gpu.md @@ -53,6 +53,24 @@ with the CUDA 12.6 wheels: On Colab or a rented machine the same commands work (without the leading `!` in a shell). The archives are then written to `./gpu-out`. +## Ablations and learning curve + +`configs/experiments/ablations.toml` runs the best recipe from run 3 (DeBERTa, `sqrt`, 5e-5) +with one factor changed at a time, three seeds each, 15 runs: +- `no-context`: no heading or list context; +- `no-group`: no group objective; +- `docs-60` and `docs-120`: learning-curve budgets. + +On a T4 it takes about 1–1.5 hours: + +```bash +!git clone https://github.com/professor3333/TriggerMap.git && cd TriggerMap && CONFIGS="ablations" bash scripts/gpu/run_neural.sh +``` + +Only `triggermap-neural-results.tgz` is needed back. `score_neural.sh` picks one best run per +grid, which suits model comparisons but not ablations, so these results are scored per variant +instead. + ## Scoring on the development machine ```bash