Resolve shared catalogue conflicts with one command

docs/DEVELOPER_WORKFLOW.md

@@ -40,7 +40,7 @@ It fetches Modwerk's `main` (from `upstream` when your fork has it, else `origin
 
 These stumbling blocks have each cost a contributor and the owner a repair round:
 
-- **main moved.** An author update merges automatically only when its CI ran against the current `main`; the scope job's name records both commits. When `main` moves first, merge it, run `pr:ready` again and push.
+- **main moved.** An author update merges automatically only when its CI ran against the current `main`; the scope job's name records both commits. When `main` moves first, merge it, run `pr:ready` again and push. If the merge stops on the shared catalogue files every Digitakt/Digitone module PR appends to (`src/catalog/machine-modules.json`, `src/community/module-changelogs.json`, the Elekloader `catalog.json` and lock), run `npm run catalog:resolve`. It merges them entry by entry and recomputes the catalogue hashes. It refuses an entry both sides changed, and any other conflict. Then commit and run `pr:ready`.
 - **Stale generated files.** `src/catalog/module-documents.json` and the compatibility files go stale whenever `main` moves under your branch. `pr:ready` regenerates them; commit what it names. Never edit them by hand.
 - **Letter case.** macOS and Windows find `Foo.s` when you wrote `foo.s`; Linux CI does not. `modules:check` refuses a manifest path in the wrong case, but not an `.include` line inside an assembly source, so check those yourself.
 - **A merge that breaks the code.** Merging `main` can leave a duplicate declaration or a stale call without any conflict marker. `pr:ready` runs the full check after the merge for exactly this reason.

package.json

@@ -18,6 +18,7 @@
     "test": "vitest run",
     "check": "node scripts/check.mjs",
     "pr:ready": "node scripts/pr-ready.mjs",
+    "catalog:resolve": "node scripts/catalog-resolve.mjs",
     "deploy": "npm run build && wrangler deploy --config wrangler.worker.jsonc",
     "dev:community": "wrangler dev --config wrangler.worker.jsonc --port 8788",
     "db:local": "wrangler d1 migrations apply octamod-community --config wrangler.worker.jsonc --local",

scripts/catalog-resolve.mjs

@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Every Digitakt/Digitone module PR appends to the same shared catalogue files, so after one merges the others
+// conflict with main. After `git merge main` stops on those conflicts, this resolves them entry by entry:
+//   npm run catalog:resolve        then review, `git commit` and `npm run pr:ready`
+// It reads the merge's three sides as Git objects and runs no module code.
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const CATALOG = 'vendor/elekloader/catalog/catalog.json', LOCK = 'vendor/elekloader/elekloader.lock.json'
+// Each file's JSON indent and its lists merged per entry. Each entry depends only on its own module, so merging
+// entries gives what the generators would write. The Octatrack catalogue is left out on purpose: its source revision
+// binds every compiled package, so two Octatrack PRs need a rebuild, not a merge.
+const FILES = {
+  'src/catalog/machine-modules.json': { indent: 2, lists: { modules: row => row.machine + '/' + row.id } },
+  'src/community/module-changelogs.json': { indent: 2, lists: {} },
+  [CATALOG]: { indent: 1, lists: { cores: pin => pin.file, mods: pin => pin.file } },
+  [LOCK]: { indent: 1, lists: {} },
+}
+const sha = text => createHash('sha256').update(text).digest('hex')
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b)
+const plain = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+
+/** Three-way JSON merge: a change on one side wins; objects merge by key and `lists` by entry id; any other value
+ * changed differently on both sides throws. Order: theirs (main), then what only ours (the PR) adds. */
+export function merge3(base, ours, theirs, lists, path) {
+  if (same(ours, theirs) || same(ours, base)) return theirs
+  if (same(theirs, base)) return ours
+  const id = lists?.[path.split('.').pop()]
+  if (id && [base ?? [], ours, theirs].every(Array.isArray)) {
+    const index = list => new Map(list.map(row => [id(row), row]))
+    const b = index(base ?? []), o = index(ours), t = index(theirs)
+    return [...new Set([...t.keys(), ...o.keys()])].map(key => merge3(b.get(key), o.get(key), t.get(key), null, path + '[' + key + ']')).filter(row => row !== undefined)
+  }
+  if (lists && [base, ours, theirs].every(plain)) {
+    const out = {}
+    for (const key of new Set([...Object.keys(theirs), ...Object.keys(ours)])) {
+      const value = merge3(base[key], ours[key], theirs[key], lists, path + '.' + key)
+      if (value !== undefined) out[key] = value
+    }
+    return out
+  }
+  throw new Error(path + ' changed differently on both sides')
+}
+
+// Whole-catalogue hashes and counts always differ between two PRs; blank them and derive them after the merge.
+function blank(path, value) {
+  if (value === undefined) return value
+  if (path === CATALOG) value.revision = ''
+  if (path === LOCK) Object.assign(value.catalog, { revision: '', sha256: '', cores: 0, mods: 0 })
+  return value
+}
+
+/** Resolves an in-progress `git merge` in `cwd` whose only conflicts are in the shared catalogue files. */
+export function resolveCatalogMerge(cwd) {
+  const git = (...args) => execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 })
+  const conflicted = git('diff', '--name-only', '--diff-filter=U', '-z').split('\0').filter(Boolean)
+  const other = conflicted.filter(path => !(path in FILES))
+  if (other.length) throw new Error('Conflicts outside the shared catalogue need a person: ' + other.join(', '))
+  const refs = [git('merge-base', 'HEAD', 'MERGE_HEAD').trim(), 'HEAD', 'MERGE_HEAD']
+  const read = (ref, path) => git('ls-tree', '--name-only', ref, '--', path).trim() === path ? git('show', ref + ':' + path) : undefined
+  const merged = {}
+  for (const [path, { indent, lists }] of Object.entries(FILES)) {
+    const [base, ours, theirs] = refs.map(ref => read(ref, path))
+    if (ours === undefined && theirs === undefined) continue
+    // Refuse rather than silently reformat a file whose layout changed on main.
+    if (theirs !== undefined && JSON.stringify(JSON.parse(theirs), null, indent) + '\n' !== theirs) throw new Error(path + ' on main is not in the layout this resolver writes')
+    const value = merge3(...[base, ours, theirs].map(text => blank(path, text === undefined ? undefined : JSON.parse(text))), lists, path)
+    if (value === undefined) throw new Error(path + ' was deleted on one side')
+    merged[path] = value
+  }
+  merged['src/catalog/machine-modules.json']?.modules.sort((a, b) => a.machine.localeCompare(b.machine) || a.id.localeCompare(b.id))
+  const text = path => JSON.stringify(merged[path], null, FILES[path].indent) + '\n'
+  const catalog = merged[CATALOG]
+  if (catalog) {
+    catalog.revision = sha(JSON.stringify({ schema: catalog.schema, kind: catalog.kind, cores: catalog.cores, mods: catalog.mods }))
+    if (merged[LOCK]) Object.assign(merged[LOCK].catalog, { revision: catalog.revision, sha256: sha(text(CATALOG)), cores: catalog.cores.length, mods: catalog.mods.length })
+  }
+  for (const path of Object.keys(merged)) { writeFileSync(resolve(cwd, path), text(path)); git('add', '--', path) }
+  if (git('diff', '--name-only', '--diff-filter=U').trim()) throw new Error('Unresolved conflicts remain')
+  return Object.keys(merged)
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) console.log('Resolved ' + resolveCatalogMerge(process.cwd()).join(', ') + '. Review them, git commit, then npm run pr:ready.')

scripts/catalog-resolve.test.mjs

@@ -0,0 +1,60 @@
+import { describe, expect, it } from 'vitest'
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, resolve } from 'node:path'
+import { merge3, resolveCatalogMerge } from './catalog-resolve.mjs'
+
+const sha = text => createHash('sha256').update(text).digest('hex')
+const row = id => ({ machine: 'digitakt', id, version: '1.0.0' })
+const pin = id => ({ file: id + '.elemod', sha256: id, id })
+
+// One repository state with these module ids in all four shared files.
+function write(dir, ids, notes = {}) {
+  const put = (path, value, indent) => { mkdirSync(dirname(resolve(dir, path)), { recursive: true }); writeFileSync(resolve(dir, path), JSON.stringify(value, null, indent) + '\n') }
+  put('src/catalog/machine-modules.json', { schemaVersion: 3, modules: ids.map(row) }, 2)
+  put('src/community/module-changelogs.json', { schemaVersion: 1, modules: Object.fromEntries(ids.map(id => [id, [{ version: '1.0.0', changes: [notes[id] ?? 'First release.'] }]])) }, 2)
+  put('vendor/elekloader/catalog/catalog.json', { schema: 1, kind: 'elekloader-catalog', revision: ids.join(), cores: [], mods: ids.map(pin) }, 1)
+  put('vendor/elekloader/elekloader.lock.json', { schema: 1, kind: 'elekloader-kit-lock', catalog: { revision: ids.join(), sha256: ids.join(), cores: 0, mods: ids.length } }, 1)
+}
+function conflictedRepo(main, pr, notes = {}) {
+  const dir = mkdtempSync(resolve(tmpdir(), 'catalog-resolve-'))
+  const git = (...args) => execFileSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@t', ...args], { cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
+  git('init', '-q', '-b', 'main'); write(dir, ['a']); git('add', '-A'); git('commit', '-qm', 'base')
+  git('checkout', '-qb', 'pr'); write(dir, pr, notes.pr); git('commit', '-qam', 'pr')
+  git('checkout', '-q', 'main'); write(dir, main, notes.main); git('commit', '-qam', 'main')
+  git('checkout', '-q', 'pr')
+  expect(() => git('merge', '-q', 'main')).toThrow()
+  return { dir, git }
+}
+
+describe('catalog resolve', () => {
+  it('keeps main’s order, appends the PR’s new entries and keeps one-sided changes', () => {
+    const lists = { modules: entry => entry.id }
+    expect(merge3({ v: 1, modules: [{ id: 'a' }] }, { v: 1, modules: [{ id: 'a' }, { id: 'c' }] }, { v: 2, modules: [{ id: 'a' }, { id: 'b' }] }, lists, 'file'))
+      .toEqual({ v: 2, modules: [{ id: 'a' }, { id: 'b' }, { id: 'c' }] })
+    expect(() => merge3({ modules: [{ id: 'a', v: 1 }] }, { modules: [{ id: 'a', v: 2 }] }, { modules: [{ id: 'a', v: 3 }] }, lists, 'file')).toThrow('file.modules[a] changed differently on both sides')
+  })
+
+  it('resolves two module additions and derives the catalogue revision and lock', () => {
+    const { dir, git } = conflictedRepo(['a', 'b'], ['a', 'c'])
+    try {
+      resolveCatalogMerge(dir)
+      const json = path => JSON.parse(readFileSync(resolve(dir, path), 'utf8'))
+      expect(json('src/catalog/machine-modules.json').modules.map(entry => entry.id)).toEqual(['a', 'b', 'c'])
+      expect(Object.keys(json('src/community/module-changelogs.json').modules)).toEqual(['a', 'b', 'c'])
+      const text = readFileSync(resolve(dir, 'vendor/elekloader/catalog/catalog.json'), 'utf8'), catalog = JSON.parse(text)
+      expect(catalog.mods.map(entry => entry.id)).toEqual(['a', 'b', 'c'])
+      expect(catalog.revision).toBe(sha(JSON.stringify({ schema: 1, kind: 'elekloader-catalog', cores: [], mods: catalog.mods })))
+      expect(json('vendor/elekloader/elekloader.lock.json').catalog).toEqual({ revision: catalog.revision, sha256: sha(text), cores: 0, mods: 3 })
+      expect(git('diff', '--name-only', '--diff-filter=U')).toBe('')
+    } finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+
+  it('leaves a module’s notes changed on both sides to a person', () => {
+    const { dir } = conflictedRepo(['a', 'b'], ['a', 'c'], { main: { a: 'Main note.' }, pr: { a: 'PR note.' } })
+    try { expect(() => resolveCatalogMerge(dir)).toThrow('modules.a changed differently on both sides') }
+    finally { rmSync(dir, { recursive: true, force: true }) }
+  })
+})