diff --git a/src/lib/ai-edition/document/audioTracks.ts b/src/lib/ai-edition/document/audioTracks.ts index b0055cb3a..5af2592ef 100644 --- a/src/lib/ai-edition/document/audioTracks.ts +++ b/src/lib/ai-edition/document/audioTracks.ts @@ -19,7 +19,11 @@ import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema"; import { isGeneratedAssetId } from "../timeline/clip-parts"; -import { anchorRegionsWithDerivedMs, clampSpanAgainstNeighbours } from "../timeline/timelineMap"; +import { + anchorRegionsWithDerivedMs, + clampSpanAgainstNeighbours, + hasCompleteClipAnchor, +} from "../timeline/timelineMap"; /** Every fragment of one user-visible track shares this key. */ export function trackGroupId(track: AxcutAudioTrack): string { @@ -75,14 +79,79 @@ export function reanchorAudioTracks( clips: AxcutClip[], makeId: () => string, ): AxcutAudioTrack[] { - // Coalesce back to one raw span per track FIRST: re-anchoring the stored - // fragments individually would re-ventilate each one and multiply them. + // Coalesce back to one raw span per take FIRST: re-anchoring the stored + // fragments individually would re-ventilate each one and multiply them. Per + // take, not per `trackId`: the edit may have pulled one take apart. // // Joined at the PILL level, before ventilation, for the same reason: ventilation // deliberately produces fragments that meet and whose offsets continue, so joining // after it would undo the split it just made. - return joinContiguousTakes(collapseTracksToPills(tracks)).flatMap((track) => - anchorAudioTrackFragments(track, clips, makeId), + return joinContiguousTakes(collapseTracksToPills(splitDetachedTakes(tracks, clips))).flatMap( + (track) => anchorAudioTrackFragments(track, clips, makeId), + ); +} + +/** + * Give each stretch of a take that no longer hangs together a `trackId` of its own (#1011). + * + * A structural clip edit carries every fragment with its own clip, so after a reorder the + * fragments of one take may no longer meet, or meet with the file jumping. Folded into ONE span + * per `trackId`, as every reader folds them, the take swallowed whatever clip now sat between its + * pieces: re-anchoring that span gave the clip audio it never had and ran `offsetMs` past the end + * of the file. Cut where it came apart, each piece keeps its clip, its length and its `offsetMs`, + * and is a take of its own, keyed by its first fragment. + */ +function splitDetachedTakes(tracks: AxcutAudioTrack[], clips: AxcutClip[]): AxcutAudioTrack[] { + const assetOf = new Map(clips.map((clip) => [clip.id, clip.assetId])); + const takes = new Map(); + for (const track of tracks) { + const key = trackGroupId(track); + const bucket = takes.get(key); + if (bucket) bucket.push(track); + else takes.set(key, [track]); + } + return [...takes.values()].flatMap((fragments) => { + const ordered = [...fragments].sort((a, b) => a.startMs - b.startMs); + let head = ordered[0]; + return ordered.map((fragment, index) => { + const previous = ordered[index - 1]; + if (previous && !hangsTogether(previous, fragment, assetOf)) head = fragment; + return { ...fragment, trackId: head.id }; + }); + }); +} + +/** + * The file runs on from `left` into `right`, and nothing plays between them: they meet on the + * ruler, or sit either side of an inserted word. + * + * Running on includes keeping the offset: a loop never advances it, and re-ventilating an orphan + * or an unanchored take copies it verbatim into every piece, which re-anchoring then repairs. + * Within a millisecond, because a re-laid boundary rounds to whole ms. + */ +function hangsTogether( + left: AxcutAudioTrack, + right: AxcutAudioTrack, + assetOf: Map, +): boolean { + const runsOn = + Math.abs(left.offsetMs - right.offsetMs) <= 1 || + (!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1); + return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf)); +} + +/** One stretch of one media, cut in two: what an inserted word leaves either side of it. */ +function mediaRunsOn( + left: AxcutAudioTrack, + right: AxcutAudioTrack, + assetOf: Map, +): boolean { + if (!hasCompleteClipAnchor(left) || !hasCompleteClipAnchor(right)) return false; + const assetId = assetOf.get(left.clipId); + return ( + assetId !== undefined && + assetId === assetOf.get(right.clipId) && + Math.abs(left.sourceEndSec - right.sourceStartSec) < 0.001 ); } @@ -109,21 +178,35 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] { } /** Same file, meeting on the ruler, and the file's own timecode continuing across the join — - * plus every payload the two would otherwise have to disagree about. */ + * plus every payload the two would otherwise have to disagree about. A loop's offset never + * advances, so its file says nothing about the join: two loops at one offset are one take when + * neither fades across the seam, which is the seam splitting one leaves (#1011). */ function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean { const spanMs = left.endMs - left.startMs; return ( left.assetId === right.assetId && left.kind === right.kind && - !left.loop && - !right.loop && + left.loop === right.loop && left.gainDb === right.gainDb && left.muted === right.muted && - Math.abs(left.endMs - right.startMs) < 1 && - Math.abs(left.offsetMs + spanMs - right.offsetMs) < 1 + Math.abs(left.endMs - right.startMs) <= 1 && + (left.loop + ? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0 + : Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1) ); } +/** + * A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what + * is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and + * keeps its offset; a region with no file passes through. + */ +export function cutTakeHead(region: T, cutSec: number): T { + const take = region as T & { offsetMs?: unknown; loop?: unknown }; + if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region; + return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) }; +} + /** * The user-visible tracks: fragments folded back into one span per `trackId`, * carrying the FIRST fragment's payload (its `offsetMs` is the track's real diff --git a/src/lib/ai-edition/document/insertion.test.ts b/src/lib/ai-edition/document/insertion.test.ts index 23efb0dae..2581c7f0a 100644 --- a/src/lib/ai-edition/document/insertion.test.ts +++ b/src/lib/ai-edition/document/insertion.test.ts @@ -179,6 +179,76 @@ describe("removeGeneratedClips", () => { }); }); +describe("a music bed over the cut (#1011)", () => { + // 2–8 s of c1, so the word typed at 4 s cuts it in two. + const withBed = () => + doc({ + audioTracks: [ + { + id: "bed", + trackId: "bed", + assetId: "music", + kind: "music", + clipId: "c1", + sourceStartSec: 2, + sourceEndSec: 8, + startMs: 2000, + endMs: 8000, + durationSec: 30, + offsetMs: 0, + gainDb: -18, + loop: false, + fadeInMs: 1000, + fadeOutMs: 1000, + muted: false, + label: "bed", + origin: "user", + }, + ], + } as Partial); + /** Each piece of the bed in ruler order: c1, the word, or the right half (a minted id). */ + const spans = (d: AxcutDocument) => + [...d.audioTracks] + .sort((a, b) => a.startMs - b.startMs) + .map((t) => [ + t.clipId?.startsWith("clip_") ? "right" : t.clipId, + t.startMs, + t.endMs, + t.offsetMs, + ]); + + it("plays on through the word, as one take", () => { + const next = withInsertion(withBed()); + expect(spans(next)).toEqual([ + ["c1", 2000, 4000, 0], + ["ext:synth_1", 4000, 4150, 2000], + ["right", 4150, 8150, 2150], + ]); + expect(new Set(next.audioTracks.map((t) => t.trackId)).size).toBe(1); + }); + + it("stays on the recording when the word is deleted", () => { + // The rejoined clip used to be re-cut against the clips as they were before the join, + // which put the right half's id back, and the bed lost everything after the word. Each + // piece keeps its place in the file: what played under the word stays played. + const back = removeGeneratedClips(withInsertion(withBed()), ["synth_1"]); + expect(spans(back)).toEqual([ + ["c1", 2000, 4000, 0], + ["c1", 4000, 8000, 2150], + ]); + }); + + it("stays on the recording when the word is dragged away", () => { + // The same join, reached by a reorder. The word takes the bed it carried with it. + const moved = moveClip(withInsertion(withBed()), "ext:synth_1", 2, "user", ""); + expect(spans(moved)).toEqual([ + ["c1", 2000, 4000, 0], + ["c1", 4000, 8000, 2150], + ["ext:synth_1", 10_000, 10_150, 2000], + ]); + }); +}); + describe("deleting the amber clip from the TIMELINE, not from the transcript", () => { // The trash icon on the clip calls `removeClip`, and so does the agent's tool. Neither // goes through `removeGeneratedClips`, so if the media were only dropped there, the diff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.ts index a4eb137f3..a33f1c6e7 100644 --- a/src/lib/ai-edition/document/timeline.test.ts +++ b/src/lib/ai-edition/document/timeline.test.ts @@ -5,6 +5,7 @@ import { type AxcutTrimRange, axcutSchemaVersion, } from "../schema"; +import { audioLanePills } from "./audioTracks"; import { buildTimelineFromIntervals, duplicateClip, @@ -25,6 +26,7 @@ import { setClipSourceRange, subtractInterval, timelineIntervals, + withClipsChanged, } from "./timeline"; function makeDoc(overrides: Partial = {}): AxcutDocument { @@ -1081,7 +1083,7 @@ describe("duplicateClip / moveClip", () => { }); // Left out on purpose, see `duplicateClip`: an imported take is not a modifier of the - // clip, and the audio fold (#1011) has no safe place for a copied fragment. + // clip. it("duplicateClip leaves an imported audio take on the original only", () => { const take: AxcutDocument["audioTracks"][number] = { id: "vo", @@ -1264,6 +1266,369 @@ describe("duplicateClip / moveClip", () => { }); }); +// #1011. A take is stored as one fragment per clip it covers, and a structural edit carries +// each fragment with its own clip. Folding every fragment of a `trackId` back into one span +// afterwards, first to last, swallowed whatever clip had come to sit between them. +describe("imported audio stays on the clips it was laid over (#1011)", () => { + type AudioTrack = AxcutDocument["audioTracks"][number]; + const fragment = ( + over: Pick< + AudioTrack, + "id" | "trackId" | "clipId" | "sourceStartSec" | "sourceEndSec" | "startMs" | "endMs" + > & + Partial, + ): AudioTrack => ({ + assetId: "bed", + kind: "music", + durationSec: 30, + offsetMs: 0, + gainDb: -18, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", + ...over, + }); + /** Every fragment in ruler order: which file, on which clip, where, and from where in the file. */ + const spans = (doc: AxcutDocument) => + [...doc.audioTracks] + .sort((a, b) => a.startMs - b.startMs) + .map((t) => [t.assetId, t.clipId, t.startMs, t.endMs, t.offsetMs]); + /** What each clip carries: the file, the stretch of the clip, the place in the file. */ + const carried = (doc: AxcutDocument) => + doc.audioTracks + .map((t) => + [ + t.clipId, + t.assetId, + Math.round((t.sourceStartSec ?? Number.NaN) * 1000), + Math.round((t.sourceEndSec ?? Number.NaN) * 1000), + t.offsetMs, + ].join(" "), + ) + .sort(); + const musicLane = (doc: AxcutDocument) => + audioLanePills(doc.audioTracks, "music").map((p) => [p.startMs, p.endMs]); + + describe("the report's project", () => { + // Three clips; a 30 s bed across the first junction, a looping 4 s track across the + // second, both music. + const reported = () => + makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [ + makeClip({ + id: "b6cdac", + assetId: "rec_a", + sourceEndSec: 102.1, + timelineEndSec: 102.1, + }), + makeClip({ + id: "63fedd", + assetId: "rec_b", + sourceEndSec: 61.87, + timelineStartSec: 102.1, + timelineEndSec: 163.97, + }), + makeClip({ + id: "d51b65", + assetId: "rec_a", + sourceEndSec: 102.1, + timelineStartSec: 163.97, + timelineEndSec: 266.07, + }), + ], + }, + audioTracks: [ + fragment({ + id: "bed", + trackId: "bed", + clipId: "b6cdac", + sourceStartSec: 89.869, + sourceEndSec: 102.1, + startMs: 89_869, + endMs: 102_100, + fadeInMs: 1000, + }), + fragment({ + id: "bed_2", + trackId: "bed", + clipId: "63fedd", + sourceStartSec: 0, + sourceEndSec: 17.769, + startMs: 102_100, + endMs: 119_869, + offsetMs: 12_231, + fadeOutMs: 1000, + }), + fragment({ + id: "loop", + trackId: "loop", + assetId: "loop", + durationSec: 4, + loop: true, + clipId: "63fedd", + sourceStartSec: 17.769, + sourceEndSec: 61.87, + startMs: 119_869, + endMs: 163_970, + fadeInMs: 1000, + }), + fragment({ + id: "loop_2", + trackId: "loop", + assetId: "loop", + durationSec: 4, + loop: true, + clipId: "d51b65", + sourceStartSec: 0, + sourceEndSec: 36.007, + startMs: 163_970, + endMs: 199_977, + fadeOutMs: 1000, + }), + ], + }); + + it("moves each fragment with its own clip when the middle clip goes to the end", () => { + const before = reported(); + const moved = moveClip(before, "63fedd", 2); + // Was: the bed stretched over d51b65 at offset 12231, and the loop pushed to 221.969– + // 385.939 s, past the 266.07 s end. + expect(spans(moved)).toEqual([ + ["bed", "b6cdac", 89_869, 102_100, 0], + ["loop", "d51b65", 102_100, 138_107, 0], + ["bed", "63fedd", 204_200, 221_969, 12_231], + ["loop", "63fedd", 221_969, 266_070, 0], + ]); + expect(carried(moved)).toEqual(carried(before)); + // Four takes now, side by side on the one music row: nothing for the lane to push. + expect(musicLane(moved)).toEqual([ + [89_869, 102_100], + [102_100, 138_107], + [204_200, 221_969], + [221_969, 266_070], + ]); + }); + + it("puts both takes back together when the clip moves back", () => { + const before = reported(); + const back = moveClip(moveClip(before, "63fedd", 2), "63fedd", 1); + expect(spans(back)).toEqual(spans(before)); + expect(carried(back)).toEqual(carried(before)); + expect(musicLane(back)).toEqual([ + [89_869, 119_869], + [119_869, 199_977], + ]); + expect(back.audioTracks.map((t) => [t.trackId, t.fadeInMs, t.fadeOutMs])).toEqual([ + ["bed", 1000, 0], + ["bed", 0, 1000], + ["loop", 1000, 0], + ["loop", 0, 1000], + ]); + }); + }); + + it("keeps each half's place in the file when the two clips under a take swap", () => { + // The halves still meet on the ruler; only the file says they are no longer one take. + const doc = makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [ + makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }), + makeClip({ + id: "clip_b", + assetId: "rec_b", + sourceEndSec: 10, + timelineStartSec: 10, + timelineEndSec: 20, + }), + ], + }, + audioTracks: [ + fragment({ + id: "bed", + trackId: "bed", + clipId: "clip_a", + sourceStartSec: 0, + sourceEndSec: 10, + startMs: 0, + endMs: 10_000, + }), + fragment({ + id: "bed_2", + trackId: "bed", + clipId: "clip_b", + sourceStartSec: 0, + sourceEndSec: 10, + startMs: 10_000, + endMs: 20_000, + offsetMs: 10_000, + }), + ], + }); + expect(spans(moveClip(doc, "clip_a", 1))).toEqual([ + ["bed", "clip_b", 0, 10_000, 10_000], + ["bed", "clip_a", 10_000, 20_000, 0], + ]); + }); + + describe("a bed across three clips", () => { + // A: rec_a 0–10 s, B: rec_b 0–10 s, C: rec_a 20–30 s; the bed covers 5–25 s on the ruler. + const threeClips = () => + makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [ + makeClip({ id: "clip_a", assetId: "rec_a", sourceEndSec: 10, timelineEndSec: 10 }), + makeClip({ + id: "clip_b", + assetId: "rec_b", + sourceEndSec: 10, + timelineStartSec: 10, + timelineEndSec: 20, + }), + makeClip({ + id: "clip_c", + assetId: "rec_a", + sourceStartSec: 20, + sourceEndSec: 30, + timelineStartSec: 20, + timelineEndSec: 30, + }), + ], + }, + audioTracks: [ + fragment({ + id: "bed", + trackId: "bed", + clipId: "clip_a", + sourceStartSec: 5, + sourceEndSec: 10, + startMs: 5000, + endMs: 10_000, + }), + fragment({ + id: "bed_2", + trackId: "bed", + clipId: "clip_b", + sourceStartSec: 0, + sourceEndSec: 10, + startMs: 10_000, + endMs: 20_000, + offsetMs: 5000, + }), + fragment({ + id: "bed_3", + trackId: "bed", + clipId: "clip_c", + sourceStartSec: 20, + sourceEndSec: 25, + startMs: 20_000, + endMs: 25_000, + offsetMs: 15_000, + }), + ], + }); + + it("gives a clip inserted inside it none of it", () => { + const before = threeClips(); + const added = makeClip({ + id: "clip_new", + assetId: "rec_c", + sourceEndSec: 4, + timelineEndSec: 4, + }); + const [a, ...rest] = before.timeline.clips; + const next = withClipsChanged(before, [a, added, ...rest]); + expect(carried(next)).toEqual(carried(before)); + expect(spans(next)).toEqual([ + ["bed", "clip_a", 5000, 10_000, 0], + ["bed", "clip_b", 14_000, 24_000, 5000], + ["bed", "clip_c", 24_000, 29_000, 15_000], + ]); + }); + + it("gives a duplicated clip none of it", () => { + const before = threeClips(); + const next = duplicateClip(before, "clip_a"); + expect(carried(next)).toEqual(carried(before)); + }); + + it("keeps the rest of it on its own footage when a clip under it is deleted", () => { + const next = removeClip(threeClips(), "clip_b"); + expect(spans(next)).toEqual([ + ["bed", "clip_a", 5000, 10_000, 0], + ["bed", "clip_c", 10_000, 15_000, 15_000], + ]); + }); + + it("cuts the file with the footage when a clip under it is narrowed", () => { + // Two seconds off clip_b's head take the two seconds of bed that played over them: + // the jump in the file is where the footage was cut, and clip_c keeps its music. + const next = setClipSourceRange(threeClips(), "clip_b", 2, 10); + expect(spans(next)).toEqual([ + ["bed", "clip_a", 5000, 10_000, 0], + ["bed", "clip_b", 10_000, 18_000, 7000], + ["bed", "clip_c", 18_000, 23_000, 15_000], + ]); + expect(musicLane(next)).toEqual([ + [5000, 10_000], + [10_000, 23_000], + ]); + }); + }); + + it("still advances the offset of a fragment a rebuild re-cuts across two clips", () => { + // The repair the fold was added for: re-ventilating an orphan copies `offsetMs` + // verbatim into every piece, and the second piece must pick up where the first stops. + const doc = makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [ + makeClip({ id: "clip_a", sourceEndSec: 10, timelineEndSec: 10 }), + makeClip({ + id: "clip_b", + sourceStartSec: 20, + sourceEndSec: 30, + timelineStartSec: 10, + timelineEndSec: 20, + }), + ], + }, + audioTracks: [ + fragment({ + id: "bed", + trackId: "bed", + clipId: "clip_b", + sourceStartSec: 22, + sourceEndSec: 28, + startMs: 12_000, + endMs: 18_000, + offsetMs: 1000, + }), + ], + }); + const rebuilt = replaceTimeline( + doc, + [ + { startSec: 0, endSec: 10 }, + { startSec: 20, endSec: 24 }, + { startSec: 26, endSec: 30 }, + ], + "split clip_b", + ); + expect(spans(rebuilt).map(([, , start, end, offset]) => [start, end, offset])).toEqual([ + [12_000, 14_000, 1000], + [14_000, 18_000, 3000], + ]); + expect(musicLane(rebuilt)).toEqual([[12_000, 18_000]]); + }); +}); + function makeZoom(overrides: Partial = {}) { return { id: "z1", diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 53b7c1120..0e3a6046a 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -25,6 +25,7 @@ import { } from "../timeline/timelineMap"; import { dropTrimPillsByIds, trimAppliesToClip } from "../timeline/trim-mapping"; import { + cutTakeHead, dropUnusedGeneratedMedia, reanchorAudioTracks, removeAudioTrack, @@ -475,7 +476,7 @@ function rederiveAnchoredRegion< if (!span) return []; return [ { - ...region, + ...cutTakeHead(region, sourceStartSec - region.sourceStartSec), sourceStartSec, sourceEndSec, startMs: Math.round(span.startSec * 1000), @@ -1007,8 +1008,7 @@ export function moveClip( // The modifiers anchored to the clip are copied the same way, across every collection // `mapAllRegionCollections` walks; `withClipsChanged` then places the copies on the new // clip. Imported audio is left out on purpose: a take is media on its own lane, not an -// effect on the clip, and a copied fragment cannot be placed safely while every structural -// edit folds each `trackId` back into one span (`reanchorAudioTracks`, #1011). +// effect on the clip. export function duplicateClip( document: AxcutDocument, clipId: string, @@ -1275,15 +1275,20 @@ function joinable(left: AxcutClip, right: AxcutClip): boolean { * zoom, an annotation and an audio take all name a clip the same way, and an id that no * longer exists has to stop being named. */ function reanchorRows(document: AxcutDocument, absorbed: Map): AxcutDocument { - const moved = mapAllRegionCollections(document, (regions) => + const relabel = (regions: T[]): T[] => regions.map((region) => hasCompleteClipAnchor(region) && absorbed.has(region.clipId) ? { ...region, clipId: absorbed.get(region.clipId) as string } : region, - ), - ); + ); + const moved = mapAllRegionCollections(document, relabel); return { ...moved, + // Relabelled and nothing more. The walk above also re-cuts every take, but against the + // clips as they were, which put the absorbed id straight back and the rederive that + // follows dropped what the absorbed clip carried (#1011). That rederive re-cuts them + // against the joined clips. + audioTracks: relabel(document.audioTracks), timeline: { ...moved.timeline, trimRanges: moved.timeline.trimRanges.map((trim) =>