Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 93 additions & 10 deletions src/lib/ai-edition/document/audioTracks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@

import type { AxcutAudioTrack, AxcutClip, AxcutDocument } from "../schema";
import { isGeneratedAssetId } from "../timeline/clip-parts";
import { anchorRegionsWithDerivedMs, clampSpanAgainstNeighbours } from "../timeline/timelineMap";
import {
anchorRegionsWithDerivedMs,
clampSpanAgainstNeighbours,
hasCompleteClipAnchor,
} from "../timeline/timelineMap";

/** Every fragment of one user-visible track shares this key. */
export function trackGroupId(track: AxcutAudioTrack): string {
Expand Down Expand Up @@ -75,14 +79,79 @@ export function reanchorAudioTracks(
clips: AxcutClip[],
makeId: () => string,
): AxcutAudioTrack[] {
// Coalesce back to one raw span per track FIRST: re-anchoring the stored
// fragments individually would re-ventilate each one and multiply them.
// Coalesce back to one raw span per take FIRST: re-anchoring the stored
// fragments individually would re-ventilate each one and multiply them. Per
// take, not per `trackId`: the edit may have pulled one take apart.
//
// Joined at the PILL level, before ventilation, for the same reason: ventilation
// deliberately produces fragments that meet and whose offsets continue, so joining
// after it would undo the split it just made.
return joinContiguousTakes(collapseTracksToPills(tracks)).flatMap((track) =>
anchorAudioTrackFragments(track, clips, makeId),
return joinContiguousTakes(collapseTracksToPills(splitDetachedTakes(tracks, clips))).flatMap(
(track) => anchorAudioTrackFragments(track, clips, makeId),
);
}

/**
* Give each stretch of a take that no longer hangs together a `trackId` of its own (#1011).
*
* A structural clip edit carries every fragment with its own clip, so after a reorder the
* fragments of one take may no longer meet, or meet with the file jumping. Folded into ONE span
* per `trackId`, as every reader folds them, the take swallowed whatever clip now sat between its
* pieces: re-anchoring that span gave the clip audio it never had and ran `offsetMs` past the end
* of the file. Cut where it came apart, each piece keeps its clip, its length and its `offsetMs`,
* and is a take of its own, keyed by its first fragment.
*/
function splitDetachedTakes(tracks: AxcutAudioTrack[], clips: AxcutClip[]): AxcutAudioTrack[] {
const assetOf = new Map(clips.map((clip) => [clip.id, clip.assetId]));
const takes = new Map<string, AxcutAudioTrack[]>();
for (const track of tracks) {
const key = trackGroupId(track);
const bucket = takes.get(key);
if (bucket) bucket.push(track);
else takes.set(key, [track]);
}
return [...takes.values()].flatMap((fragments) => {
const ordered = [...fragments].sort((a, b) => a.startMs - b.startMs);
let head = ordered[0];
return ordered.map((fragment, index) => {
const previous = ordered[index - 1];
if (previous && !hangsTogether(previous, fragment, assetOf)) head = fragment;
return { ...fragment, trackId: head.id };
});
});
}

/**
* The file runs on from `left` into `right`, and nothing plays between them: they meet on the
* ruler, or sit either side of an inserted word.
*
* Running on includes keeping the offset: a loop never advances it, and re-ventilating an orphan
* or an unanchored take copies it verbatim into every piece, which re-anchoring then repairs.
* Within a millisecond, because a re-laid boundary rounds to whole ms.
*/
function hangsTogether(
left: AxcutAudioTrack,
right: AxcutAudioTrack,
assetOf: Map<string, string>,
): boolean {
const runsOn =
Math.abs(left.offsetMs - right.offsetMs) <= 1 ||
(!right.loop && Math.abs(left.offsetMs + (left.endMs - left.startMs) - right.offsetMs) <= 1);
return runsOn && (Math.abs(left.endMs - right.startMs) <= 1 || mediaRunsOn(left, right, assetOf));
}

/** One stretch of one media, cut in two: what an inserted word leaves either side of it. */
function mediaRunsOn(
left: AxcutAudioTrack,
right: AxcutAudioTrack,
assetOf: Map<string, string>,
): boolean {
if (!hasCompleteClipAnchor(left) || !hasCompleteClipAnchor(right)) return false;
const assetId = assetOf.get(left.clipId);
return (
assetId !== undefined &&
assetId === assetOf.get(right.clipId) &&
Math.abs(left.sourceEndSec - right.sourceStartSec) < 0.001
);
}

Expand All @@ -109,21 +178,35 @@ function joinContiguousTakes(pills: AxcutAudioTrack[]): AxcutAudioTrack[] {
}

/** Same file, meeting on the ruler, and the file's own timecode continuing across the join —
* plus every payload the two would otherwise have to disagree about. */
* plus every payload the two would otherwise have to disagree about. A loop's offset never
* advances, so its file says nothing about the join: two loops at one offset are one take when
* neither fades across the seam, which is the seam splitting one leaves (#1011). */
function takesJoin(left: AxcutAudioTrack, right: AxcutAudioTrack): boolean {
const spanMs = left.endMs - left.startMs;
return (
left.assetId === right.assetId &&
left.kind === right.kind &&
!left.loop &&
!right.loop &&
left.loop === right.loop &&
left.gainDb === right.gainDb &&
left.muted === right.muted &&
Math.abs(left.endMs - right.startMs) < 1 &&
Math.abs(left.offsetMs + spanMs - right.offsetMs) < 1
Math.abs(left.endMs - right.startMs) <= 1 &&
(left.loop
? left.offsetMs === right.offsetMs && left.fadeOutMs === 0 && right.fadeInMs === 0
: Math.abs(left.offsetMs + spanMs - right.offsetMs) <= 1)
);
}

/**
* A clip edit that cuts a fragment's head off cuts the same stretch off its take's file, so what
* is left goes on playing the audio over its own footage (#1011). A loop restarts on its own and
* keeps its offset; a region with no file passes through.
*/
export function cutTakeHead<T extends object>(region: T, cutSec: number): T {
const take = region as T & { offsetMs?: unknown; loop?: unknown };
if (!(cutSec > 0) || typeof take.offsetMs !== "number" || take.loop === true) return region;
return { ...region, offsetMs: take.offsetMs + Math.round(cutSec * 1000) };
}

/**
* The user-visible tracks: fragments folded back into one span per `trackId`,
* carrying the FIRST fragment's payload (its `offsetMs` is the track's real
Expand Down
70 changes: 70 additions & 0 deletions src/lib/ai-edition/document/insertion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,76 @@ describe("removeGeneratedClips", () => {
});
});

describe("a music bed over the cut (#1011)", () => {
// 2–8 s of c1, so the word typed at 4 s cuts it in two.
const withBed = () =>
doc({
audioTracks: [
{
id: "bed",
trackId: "bed",
assetId: "music",
kind: "music",
clipId: "c1",
sourceStartSec: 2,
sourceEndSec: 8,
startMs: 2000,
endMs: 8000,
durationSec: 30,
offsetMs: 0,
gainDb: -18,
loop: false,
fadeInMs: 1000,
fadeOutMs: 1000,
muted: false,
label: "bed",
origin: "user",
},
],
} as Partial<AxcutDocument>);
/** Each piece of the bed in ruler order: c1, the word, or the right half (a minted id). */
const spans = (d: AxcutDocument) =>
[...d.audioTracks]
.sort((a, b) => a.startMs - b.startMs)
.map((t) => [
t.clipId?.startsWith("clip_") ? "right" : t.clipId,
t.startMs,
t.endMs,
t.offsetMs,
]);

it("plays on through the word, as one take", () => {
const next = withInsertion(withBed());
expect(spans(next)).toEqual([
["c1", 2000, 4000, 0],
["ext:synth_1", 4000, 4150, 2000],
["right", 4150, 8150, 2150],
]);
expect(new Set(next.audioTracks.map((t) => t.trackId)).size).toBe(1);
});

it("stays on the recording when the word is deleted", () => {
// The rejoined clip used to be re-cut against the clips as they were before the join,
// which put the right half's id back, and the bed lost everything after the word. Each
// piece keeps its place in the file: what played under the word stays played.
const back = removeGeneratedClips(withInsertion(withBed()), ["synth_1"]);
expect(spans(back)).toEqual([
["c1", 2000, 4000, 0],
["c1", 4000, 8000, 2150],
]);
});

it("stays on the recording when the word is dragged away", () => {
// The same join, reached by a reorder. The word takes the bed it carried with it.
const moved = moveClip(withInsertion(withBed()), "ext:synth_1", 2, "user", "");
expect(spans(moved)).toEqual([
["c1", 2000, 4000, 0],
["c1", 4000, 8000, 2150],
["ext:synth_1", 10_000, 10_150, 2000],
]);
});
});

describe("deleting the amber clip from the TIMELINE, not from the transcript", () => {
// The trash icon on the clip calls `removeClip`, and so does the agent's tool. Neither
// goes through `removeGeneratedClips`, so if the media were only dropped there, the
Expand Down
Loading
Loading