diff --git a/.gitignore b/.gitignore index a1822ee05..24ad425eb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,4 @@ workbench/fixtures/ /aur_ci /aur_ci.pub /aur_known_hosts +tmp_handoff.md diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index e51f0f8af..b2674b85d 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -5,7 +5,7 @@ use crate::ffi::*; use crate::regions::SpeedSegment; -use crate::scene::SceneAudio; +use crate::scene::{SceneAudio, SceneAudioTrack}; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -1381,6 +1381,142 @@ pub fn assemble_concatenated_pcm( output } +/// Mix imported audio tracks (issue #350) over the assembled programme. +/// +/// Each track is decoded across its trim window — already resampled to 48 kHz +/// stereo by `decode_clip_audio`, the same path a clip's own audio takes — scaled +/// by its per-track gain (the same `10^(dB/20)` law as `finish_audio`), and summed +/// into the programme at `start_sec`. The programme length is NOT extended: a +/// track that runs past the video is truncated to it, so the audio and video +/// streams stay the same length for the muxer. +/// +/// The decode window is capped up front at the room left in the programme after +/// `start_sec`, and a track starting at/after the end is skipped without decoding. +/// `decode_clip_audio` preallocates from the window, so this keeps a long track +/// pinned near a short programme's end from buffering (and clamping away) hours of +/// PCM. `trim_end_sec` must therefore be concrete — the renderer sends +/// `trimEnd ?? durationSec`. +/// +/// A track whose file has no decodable audio is skipped — the same degradation a +/// stream-less clip gets. +pub fn mix_external_tracks(mut programme: PlanarPcm, tracks: &[SceneAudioTrack]) -> PlanarPcm { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if programme_len == 0 { + return programme; + } + for track in tracks { + let offset = (track.start_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + // A track that starts at or past the programme end contributes nothing — + // skip it before decoding anything. + if offset >= programme_len { + continue; + } + let trim_start = track.trim_start_sec.max(0.0); + let Some(trim_end_full) = track.trim_end_sec else { + // Without a concrete end there is no safe window to decode (see the doc + // comment); the renderer always resolves one, so this only guards a + // hand-written scene. + continue; + }; + // Cap the decode window at the room left in the programme. Everything past + // `offset` that overflows is discarded by `overlay_track_pcm` anyway, so + // decoding it only wastes time and memory — a three-hour track placed at + // second 9 of a ten-second export must not buffer three hours of PCM. + let remaining_sec = (programme_len - offset) as f64 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + let trim_end = trim_end_full.min(trim_start + remaining_sec); + if trim_end <= trim_start { + continue; + } + let decoded = match decode_clip_audio(&track.path, trim_start, trim_end) { + Ok(Some(pcm)) => pcm, + _ => continue, + }; + // The app's own range is -60..+12 dB (the inspector slider); clamping at + // -12 here floored every quiet bed at a tenth of the attenuation asked for. + let gain = 10.0f32.powf(track.gain_db.clamp(-60.0, 12.0) / 20.0); + overlay_track_pcm( + &mut programme, + &decoded, + offset, + gain, + track.fade_in_sec.max(0.0), + track.fade_out_sec.max(0.0), + ); + } + programme +} + +/// Sum one decoded track into the programme at `offset` samples, scaled by `gain`, +/// truncated at the programme's end. Split out of `mix_external_tracks` so the +/// placement/gain/clamp math is testable without ffmpeg, exactly like +/// `mix_aligned_tracks` is split from the decode above. +fn overlay_track_pcm( + programme: &mut PlanarPcm, + decoded: &PlanarPcm, + offset: usize, + gain: f32, + fade_in_sec: f64, + fade_out_sec: f64, +) { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if offset >= programme_len { + return; + } + let room = programme_len - offset; + // The ramps are measured against the DECODED length, not the room left in the + // programme: a track running past the end is cut off there, and a fade-out + // timed to the cut would ramp down over audio the export never reaches. + let decoded_len = decoded.iter().map(Vec::len).max().unwrap_or(0); + let (fade_in, fade_out) = resolve_fade_samples(decoded_len, fade_in_sec, fade_out_sec); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let Some(source) = decoded.get(channel) else { + continue; + }; + let count = source.len().min(room); + let dst = &mut programme[channel]; + for k in 0..count { + dst[offset + k] += source[k] * gain * fade_envelope(k, decoded_len, fade_in, fade_out); + } + } +} + +/// Fade lengths in samples, reduced to fit inside `len`. +/// +/// Fades that do not fit share the window in proportion rather than being clamped +/// independently: clamping each to the length first would turn an asymmetric pair +/// into a symmetric one, losing the shape asked for. Kept identical to the app's +/// `resolveFadeSecs` so the preview and the render agree. +fn resolve_fade_samples(len: usize, fade_in_sec: f64, fade_out_sec: f64) -> (usize, usize) { + if len == 0 { + return (0, 0); + } + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let mut fade_in = fade_in_sec.max(0.0) * rate; + let mut fade_out = fade_out_sec.max(0.0) * rate; + let total = fade_in + fade_out; + if total > len as f64 && total > 0.0 { + let scale = len as f64 / total; + fade_in *= scale; + fade_out *= scale; + } + (fade_in.round() as usize, fade_out.round() as usize) +} + +/// Linear ramp factor at sample `k` of a `len`-sample track. +fn fade_envelope(k: usize, len: usize, fade_in: usize, fade_out: usize) -> f32 { + let mut v = 1.0f32; + if fade_in > 0 && k < fade_in { + v = v.min(k as f32 / fade_in as f32); + } + if fade_out > 0 && len > k { + let remaining = len - k; + if remaining <= fade_out { + v = v.min(remaining as f32 / fade_out as f32); + } + } + v +} + /// Encodeur AAC attaché au muxer avant son header. Les paquets utilisent le même interleaver /// que la vidéo ; les pts restent en unités échantillon jusqu'au rescale vers l'AVStream. pub(crate) struct AacEncoder { @@ -1692,6 +1828,68 @@ mod tests { assert_eq!(mixed[1], vec![0.25, -0.5, 0.75]); } + // Imported audio track overlay (issue #350). + #[test] + fn overlay_sums_at_offset_with_gain() { + let mut programme = planar(&[0.1, 0.1, 0.1, 0.1]); + // ×2 gain, placed at sample offset 1. + overlay_track_pcm(&mut programme, &planar(&[0.2, 0.2]), 1, 2.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.1, 0.5, 0.5, 0.1]); + assert_eq!(programme[1], vec![0.1, 0.5, 0.5, 0.1]); + } + + #[test] + fn overlay_truncates_a_track_that_runs_past_the_programme() { + let mut programme = planar(&[0.0, 0.0, 0.0]); + // A 4-sample track placed at offset 2 has room for only 1 sample. + overlay_track_pcm(&mut programme, &planar(&[1.0, 1.0, 1.0, 1.0]), 2, 1.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.0, 0.0, 1.0]); + } + + #[test] + fn overlay_past_the_end_is_a_no_op() { + let mut programme = planar(&[0.3, 0.3]); + overlay_track_pcm(&mut programme, &planar(&[1.0]), 5, 1.0, 0.0, 0.0); + assert_eq!(programme[0], vec![0.3, 0.3]); + } + + #[test] + fn mix_external_tracks_skips_empty_windows() { + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 0.0, + gain_db: 0.0, + trim_start_sec: 2.0, + trim_end_sec: Some(1.0), // end <= start: empty window, never decoded + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + // The empty window is skipped before any decode, so the programme is + // untouched even though the path does not exist. + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + + #[test] + fn mix_external_tracks_skips_a_track_that_starts_past_the_programme() { + // 2 samples = ~0.00004 s of programme at 48 kHz; the track starts at 1 s, so + // its offset is past the end. It must be skipped before any decode is + // attempted (the path does not exist), never buffering its window. + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 1.0, + gain_db: 0.0, + trim_start_sec: 0.0, + trim_end_sec: Some(3600.0), + fade_in_sec: 0.0, + fade_out_sec: 0.0, + }]; + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + #[test] fn single_track_is_not_clamped() { // Promesse de non-régression : une source mono-piste ressort telle quelle, y compris @@ -1784,6 +1982,73 @@ mod tests { assert!((loud[0][0] - ceiling).abs() < 1e-6); } + #[test] + fn fades_that_fit_are_left_alone() { + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let (fin, fout) = resolve_fade_samples(rate as usize, 0.1, 0.2); + assert_eq!(fin, (0.1 * rate).round() as usize); + assert_eq!(fout, (0.2 * rate).round() as usize); + } + + #[test] + fn fades_too_long_for_the_track_share_it_in_proportion() { + // 6 s + 4 s of fade on a 2 s track → 1.2 s / 0.8 s, not a clamped 1 s / 1 s. + // Mirrors `resolveFadeSecs` on the app side; the two must agree or the + // preview and the render shape the same track differently. + let rate = AUDIO_OUTPUT_SAMPLE_RATE as f64; + let len = (2.0 * rate) as usize; + let (fin, fout) = resolve_fade_samples(len, 6.0, 4.0); + assert_eq!(fin, (1.2 * rate).round() as usize); + assert_eq!(fout, (0.8 * rate).round() as usize); + assert!(fin + fout <= len + 1); + } + + #[test] + fn a_fade_in_longer_than_the_track_still_reaches_full_volume() { + // Left unreduced this holds the gain near zero for the whole track — the + // layer exports silent. + let (fin, fout) = resolve_fade_samples(100, 10.0, 0.0); + assert_eq!((fin, fout), (100, 0)); + assert!((fade_envelope(99, 100, fin, fout) - 0.99).abs() < 1e-3); + } + + #[test] + fn the_envelope_ramps_at_both_edges_and_holds_between() { + assert_eq!(fade_envelope(0, 100, 10, 10), 0.0); + assert!((fade_envelope(5, 100, 10, 10) - 0.5).abs() < 1e-6); + assert_eq!(fade_envelope(50, 100, 10, 10), 1.0); + assert!((fade_envelope(95, 100, 10, 10) - 0.5).abs() < 1e-6); + } + + #[test] + fn overlay_applies_the_fade_over_the_decoded_length() { + // The ramps are measured against the DECODED length, not the room left in + // the programme: a fade-out timed to the programme's end would ramp down + // over audio the export never reaches. + let mut programme = planar(&[0.0, 0.0, 0.0, 0.0]); + let decoded = planar(&[1.0, 1.0, 1.0, 1.0]); + // A 4-sample fade-in at 48 kHz is far below one sample of real time, so + // ask for the whole decoded length in seconds. + let four = 4.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + overlay_track_pcm(&mut programme, &decoded, 0, 1.0, four, 0.0); + assert_eq!(programme[0][0], 0.0); + assert!(programme[0][1] > 0.0 && programme[0][1] < 1.0); + assert!(programme[0][3] > programme[0][1]); + } + + #[test] + fn a_track_gain_below_the_output_bound_is_honoured() { + // The per-track gain range is the inspector's -60..+12, NOT the project + // output trim's ±12: clamping here at -12 floored every quiet bed at a + // tenth of the attenuation asked for. + let mut programme = planar(&[0.0]); + let decoded = planar(&[1.0]); + let gain = 10.0f32.powf(-40.0 / 20.0); + overlay_track_pcm(&mut programme, &decoded, 0, gain, 0.0, 0.0); + assert!((programme[0][0] - gain).abs() < 1e-9); + assert!(programme[0][0] < 10.0f32.powf(-12.0 / 20.0)); + } + #[test] fn output_is_clipped_to_full_scale_and_keeps_its_length() { // The trim can push a hot signal past full scale; the timeline must come back the diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 235a32917..9a58a5443 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -21,7 +21,7 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -459,6 +459,12 @@ pub fn run_composited_multi( let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene so the + // mix step below owns them. Empty for a project with no imported audio. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -552,7 +558,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index c7cf571bc..f8afc95b5 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,7 +30,7 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -1078,6 +1078,11 @@ pub fn run_composited_multi( // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1146,7 +1151,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 5fff82056..a3d1a7d68 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,7 +3,7 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -1344,6 +1344,11 @@ unsafe fn run_multi_inner( // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1477,7 +1482,7 @@ unsafe fn run_multi_inner( out_fps as f64, ); let assembled_audio = finish_audio( - assemble_concatenated_pcm(&clip_pcm, &audio_plan), + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), audio_settings, ); audio_encoder.encode(&assembled_audio, octx)?; diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index ff300d475..7a4905a2f 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -432,6 +432,39 @@ pub struct SceneAudio { pub gain_db: f32, } +/// One imported audio track (issue #350) mixed over the assembled programme — +/// voiceover / BGM / SFX. Deliberately a SEPARATE `Scene` field rather than a +/// member of `SceneAudio`, so `SceneAudio` stays `Copy` and the pipelines keep +/// copying it out of a borrow unchanged. +/// +/// `start_sec` is the track's head on the OUTPUT programme; `trim_start_sec` / +/// `trim_end_sec` window the source file (both source seconds). The renderer +/// resolves `start_sec` from the track's raw timeline position — equal to it when +/// the project has no trims/speed, which is the case this first cut mixes exactly. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudioTrack { + pub path: String, + #[serde(default)] + pub start_sec: f64, + #[serde(default)] + pub gain_db: f32, + #[serde(default)] + pub trim_start_sec: f64, + #[serde(default)] + pub trim_end_sec: Option, + /// Ramp lengths at this entry's own edges, in seconds. The app puts them only + /// on the pieces that touch the track's real start and end, so a split or + /// looping track fades once instead of at every cut or repeat. + /// + /// `#[serde(default)]` for the usual reason: a payload from a build that + /// predates the field must degrade to "no fade", not fail the whole scene. + #[serde(default)] + pub fade_in_sec: f64, + #[serde(default)] + pub fade_out_sec: f64, +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -500,6 +533,10 @@ pub struct Scene { /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] pub audio: SceneAudio, + /// Imported audio tracks mixed over the programme (issue #350). `#[serde(default)]`: + /// absent from every scene written before this, and from a project with none. + #[serde(default)] + pub audio_tracks: Vec, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6cbdb97c9..bf3d12289 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -280,6 +280,63 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(first.project.primaryAssetId); expect(after.assets).toHaveLength(2); }); + + // Issue #350 — external audio import (voiceover / BGM / SFX). + it("appends an audio asset without claiming the primary slot", async () => { + const doc = await service.createProject("P"); + const updated = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover.mp3", + kind: "audio", + }); + expect(updated.assets).toHaveLength(1); + expect(updated.assets[0]?.kind).toBe("audio"); + // An audio-only file must never become the project's primary asset, even + // when it is the first file added to an otherwise-empty project. + expect(updated.project.primaryAssetId).toBeUndefined(); + }); + + it("keeps the existing video primary when an audio track is added", async () => { + const doc = await service.createProject("P"); + const withVideo = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const primary = withVideo.project.primaryAssetId; + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/bgm.wav", + kind: "audio", + }); + expect(withAudio.project.primaryAssetId).toBe(primary); + expect(withAudio.assets).toHaveLength(2); + }); + + it("rejects unsupported audio extensions", async () => { + const doc = await service.createProject("P"); + await expect( + service.addAsset(doc.project.id, { path: "/tmp/clip.mp4", kind: "audio" }), + ).rejects.toBeInstanceOf(ProjectFileError); + }); + + it("accepts a recorded .webm take as audio", async () => { + // MediaRecorder writes a voiceover as webm/opus — the same extension a + // screen recording uses. The caller has already declared the kind here, + // so this gate must take it; only the import PICKER, which has nothing + // but the extension to go on, still refuses .webm as audio. + const doc = await service.createProject("P"); + const next = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover-2026.webm", + kind: "audio", + }); + expect(next.assets.at(-1)).toMatchObject({ kind: "audio" }); + // ...and it must not have claimed the primary (video-only) slot. + expect(next.project.primaryAssetId).toBeUndefined(); + }); + + it("accepts a video extension under the default kind but not as audio", async () => { + const doc = await service.createProject("P"); + // The same extension routing works in reverse: an .mp3 is fine as audio + // but rejected as video (covered above), and an .mp4 is the opposite. + await expect( + service.addAsset(doc.project.id, { path: "/tmp/a.mp3", kind: "audio" }), + ).resolves.toBeDefined(); + }); }); describe("removeAsset", () => { @@ -363,6 +420,57 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(b.assets[1]?.id); }); + // Issue #350 — an audio overlay can never be primary. + it("passes primary to the next VIDEO asset, never to an audio asset", async () => { + const doc = await service.createProject("P"); + const video = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + await service.addAsset(doc.project.id, { path: "/tmp/music.mp3", kind: "audio" }); + const primaryId = video.project.primaryAssetId; + expect(primaryId).toBeTruthy(); + // Removing the only video leaves just the audio asset; primary must clear, + // not fall to the audio one. + const after = await service.removeAsset(doc.project.id, primaryId ?? ""); + expect(after.project.primaryAssetId).toBeUndefined(); + expect(after.assets).toHaveLength(1); + expect(after.assets[0]?.kind).toBe("audio"); + }); + + it("drops audioTracks that referenced a removed audio asset", async () => { + const doc = await service.createProject("P"); + await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/music.mp3", + kind: "audio", + }); + const audioId = withAudio.assets.find((a) => a.kind === "audio")?.id ?? ""; + expect(audioId).toBeTruthy(); + const withTrack = await service.saveProject({ + ...withAudio, + audioTracks: [ + { + id: "trk_1", + assetId: audioId, + kind: "music", + startMs: 0, + endMs: 10_000, + durationSec: 10, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "music", + origin: "user", + }, + ], + }); + expect(withTrack.audioTracks).toHaveLength(1); + const after = await service.removeAsset(doc.project.id, audioId); + expect(after.audioTracks).toEqual([]); + expect(after.assets.some((a) => a.id === audioId)).toBe(false); + }); + it("resequences other assets and rederives their anchored regions", async () => { const created = await service.createProject("P"); const withA = await service.addAsset(created.project.id, { path: "/tmp/a.mp4" }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 3c93e3bc0..9c9b958b9 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -38,6 +38,9 @@ export interface ProjectSummary { export interface AddAssetInput { path: string; label?: string; + // "audio" imports an external voiceover / BGM / SFX file (issue #350). + // Defaults to "video" when omitted, so existing callers are unaffected. + kind?: "video" | "audio"; } export class DocumentNotFoundError extends Error { @@ -72,6 +75,35 @@ function isSupportedVideoPath(filePath: string): boolean { return SUPPORTED_VIDEO_EXTENSIONS.has(ext); } +// Imported audio (issue #350). Decoding is handled downstream by the same +// WebCodecs / ffmpeg paths that read a video's audio track, so this list is the +// container formats decodeAudioData and the compositor can open. +// What may be filed as an AUDIO asset. Deliberately WIDER than the import +// picker's list in `electron/ipc/handlers.ts`: this gate runs when the caller +// has already declared `kind: "audio"`, so it only has to reject files that +// could not carry audio at all, whereas the picker has to guess from the +// extension alone and must not offer a video as audio. +// +// `.webm` is exactly that difference. An in-editor voiceover take is written by +// MediaRecorder as webm/opus — the same extension a screen recording uses — so +// the picker rightly refuses it while this gate must accept it. +const SUPPORTED_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".oga", + ".opus", + ".webm", +]); + +function isSupportedAudioPath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + return SUPPORTED_AUDIO_EXTENSIONS.has(ext); +} + function safeProjectId(raw: string): string { // ponytail: project ids are uuid-prefixed strings (e.g. "proj_"). Reject // anything that smells like path traversal before we ever touch the disk. @@ -283,7 +315,15 @@ export class DocumentService { if (!input.path) { throw new ProjectFileError("Asset path is required.", projectId); } - if (!isSupportedVideoPath(input.path)) { + const kind = input.kind ?? "video"; + if (kind === "audio") { + if (!isSupportedAudioPath(input.path)) { + throw new ProjectFileError( + `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`, + projectId, + ); + } + } else if (!isSupportedVideoPath(input.path)) { throw new ProjectFileError( `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`, projectId, @@ -300,18 +340,24 @@ export class DocumentService { } const asset: AxcutAsset = { id: createId("asset"), - kind: "video", + kind, label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, sizeBytes, cameraTrack: null, }; + // An audio import is an overlay, never the thing the timeline is built + // around, so it must not claim the empty primaryAssetId slot — otherwise the + // first file dropped into a fresh project (a BGM track) would become its + // primary asset and the editor would try to lay out clips from a file with + // no video. + const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId; const next: AxcutDocument = { ...doc, assets: [...doc.assets, asset], project: { ...doc.project, - ...(doc.project.primaryAssetId ? {} : { primaryAssetId: asset.id }), + ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), updatedAt: new Date().toISOString(), }, }; @@ -324,9 +370,13 @@ export class DocumentService { throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId); } const assets = doc.assets.filter((a) => a.id !== assetId); + // Primary is the thing the timeline is built around, so it must fall to the + // next VIDEO asset — never an audio overlay (issue #350), which can't be + // primary (see addAsset). Falling back to `assets[0]` would hand primary to + // an audio asset when the removed one was the last video. const primaryAssetId = doc.project.primaryAssetId === assetId - ? (assets[0]?.id ?? undefined) + ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) : doc.project.primaryAssetId; const withoutAssetClips = doc.timeline.clips .filter((clip) => clip.assetId === assetId) @@ -334,6 +384,9 @@ export class DocumentService { const next: AxcutDocument = { ...withoutAssetClips, assets, + // Drop imported audio tracks that referenced the removed asset — they + // would otherwise dangle, pointing at an asset the document no longer has. + audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), timeline: { ...withoutAssetClips.timeline, trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index e140a4e37..6de7a3cc4 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -289,6 +289,22 @@ interface Window { name?: string; canceled?: boolean; }>; + // Import an external audio file from the timeline toolbar (issue #350). + openAudioFilePicker: () => Promise<{ + success: boolean; + path?: string; + name?: string; + canceled?: boolean; + message?: string; + }>; + // Persist an in-editor voiceover take (raw MediaRecorder bytes) under the + // recordings dir, so it outlives the session like every other asset. + saveRecordedVoiceover: (data: ArrayBuffer) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + }>; setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; setCurrentRecordingSession: ( session: import("../src/lib/recordingSession").RecordingSession | null, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index aa2014670..f404daa84 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -107,6 +107,9 @@ const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([ ".ts", ]); const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +// See the save-recorded-voiceover handler: an upper bound on renderer-supplied +// bytes written to disk, well past any plausible take. +const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024; const nativeMacCaptureEvents = new EventEmitter(); // Enumeration walks every display and window and grabs a thumbnail of each, so it @@ -186,6 +189,34 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +// Imported audio (issue #350). Kept separate from the video set so the two +// pickers stay honest — an audio picker must not approve a video path and vice +// versa. A SUBSET of SUPPORTED_AUDIO_EXTENSIONS in the document service, which +// also accepts `.webm`: that gate is told the kind by its caller, while this one +// only has the extension to go on and `.webm` is far more often a video. +const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".opus", +]); + +function hasAllowedImportAudioExtension(filePath: string): boolean { + return ALLOWED_IMPORT_AUDIO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +// Video OR audio. The type-specific pickers stay honest (see the audio set's +// comment), but the generic media READS — peaks, binary, file-info, chunk — serve +// whichever kind the document points at, so they must accept both. Gating them on +// video alone dropped every imported audio path once `approvedPaths` was empty +// (a project reopen), and the waveform was lost for good (issue #350). +function hasAllowedImportMediaExtension(filePath: string): boolean { + return hasAllowedImportVideoExtension(filePath) || hasAllowedImportAudioExtension(filePath); +} + function runProcess( command: string, args: string[], @@ -282,8 +313,13 @@ async function prepareSupplementalPreviewAudioTrack(videoPath: string) { return { success: true, path: pathToFileURL(outputPath).toString() }; } -async function approveReadableVideoPath( - filePath?: string | null, +// Shared core behind the media path approvers. `hasAllowedExtension` is the ONLY +// thing that differs between video and audio imports, so it is the single knob: +// an already-approved path passes regardless, otherwise the extension gate, +// optional trusted-dir confinement, and a stat check decide whether to approve. +async function approveReadableMediaPath( + filePath: string | null | undefined, + hasAllowedExtension: (p: string) => boolean, trustedDirs?: string[], ): Promise { const normalizedPath = normalizeVideoSourcePath(filePath); @@ -295,7 +331,7 @@ async function approveReadableVideoPath( return normalizedPath; } - if (!hasAllowedImportVideoExtension(normalizedPath)) { + if (!hasAllowedExtension(normalizedPath)) { return null; } @@ -322,6 +358,29 @@ async function approveReadableVideoPath( return normalizedPath; } +function approveReadableVideoPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportVideoExtension, trustedDirs); +} + +function approveReadableAudioPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportAudioExtension, trustedDirs); +} + +// For the generic media reads that accept either kind — NOT for the pickers, +// which must stay type-specific (see `hasAllowedImportMediaExtension`). +function approveReadableAvPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportMediaExtension, trustedDirs); +} + function resolveRecordingOutputPath(fileName: string): string { const trimmed = fileName.trim(); if (!trimmed) { @@ -3590,6 +3649,8 @@ export function registerIpcHandlers( } }); + // The media tab imports VIDEO (it arranges clips). Audio is imported from the + // timeline toolbar instead (issue #350) — see `open-audio-file-picker` below. ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -3636,6 +3697,84 @@ export function registerIpcHandlers( } }); + // Import an external audio file (voiceover / BGM / SFX) — issue #350. Driven by + // the timeline's "Add audio" tool: audio is a timeline overlay (like an + // annotation), not a media-tab clip, so it has its own audio-only picker and the + // renderer adds it as a kind:"audio" asset + track at the playhead. + ipcMain.handle("open-audio-file-picker", async () => { + try { + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.selectAudio"), + defaultPath: RECORDINGS_DIR, + filters: [ + { + name: mainT("dialogs", "fileDialogs.audioFiles"), + extensions: ["mp3", "wav", "m4a", "aac", "flac", "ogg", "opus"], + }, + { name: mainT("dialogs", "fileDialogs.allFiles"), extensions: ["*"] }, + ], + properties: ["openFile"], + }, + getMainWindow(), + ); + const result = await dialog.showOpenDialog(dialogOptions); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + const normalizedPath = await approveReadableAudioPath(result.filePaths[0]); + if (!normalizedPath) { + return { + success: false, + message: "Selected file is not a supported readable audio file", + }; + } + + return { + success: true, + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to open audio file picker:", error); + return { + success: false, + message: "Failed to open audio file picker", + error: String(error), + }; + } + }); + + // In-editor voiceover recording: the renderer hands over the raw MediaRecorder + // blob (webm/opus) and gets back the path it landed at, under the recordings + // dir so it lives with the project's other media and survives relaunches. + ipcMain.handle("save-recorded-voiceover", async (_event, data: ArrayBuffer) => { + try { + if (!(data instanceof ArrayBuffer) || data.byteLength === 0) { + return { success: false, message: "Empty recording" }; + } + // A cap, because this writes renderer-supplied bytes straight to disk. An + // hour of Opus is a few tens of MB, so 512 MB is far past any real take + // and still refuses a runaway or malformed payload before it is buffered. + if (data.byteLength > MAX_RECORDED_VOICEOVER_BYTES) { + return { success: false, message: "Recording too large" }; + } + await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + const fileName = `voiceover-${new Date().toISOString().replace(/[:.]/g, "-")}.webm`; + const target = path.join(RECORDINGS_DIR, fileName); + await fs.writeFile(target, Buffer.from(data)); + return { success: true, path: target }; + } catch (error) { + console.error("Failed to save recorded voiceover:", error); + return { + success: false, + message: "Failed to save recorded voiceover", + error: String(error), + }; + } + }); + ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { // showItemInFolder returns nothing, it throws on error @@ -3661,7 +3800,7 @@ export function registerIpcHandlers( ipcMain.handle("read-binary-file", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, @@ -3691,7 +3830,7 @@ export function registerIpcHandlers( // recording above that can never be loaded whole — see read-file-chunk). ipcMain.handle("get-readable-file-info", async (_, filePath: string) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, @@ -3727,7 +3866,7 @@ export function registerIpcHandlers( async (_, filePath: string, durationSec: number): Promise => { try { // Same approval gate as every other read of a renderer-supplied path. - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, message: "File path is not approved" }; } @@ -3751,7 +3890,7 @@ export function registerIpcHandlers( // do (2 GiB cap) and a 16 GB machine cannot hold for multi-GB recordings. ipcMain.handle("read-file-chunk", async (_, filePath: string, offset: number, length: number) => { try { - const normalizedPath = await approveReadableVideoPath(filePath); + const normalizedPath = await approveReadableAvPath(filePath); if (!normalizedPath) { return { success: false, diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 5d4992c10..7b01e0b2f 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -489,6 +489,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.projectId, request.payload.path, request.payload.label, + request.payload.kind, ), ); case "document.removeAsset": diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 0fbbccc9c..90088781a 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -151,9 +151,17 @@ export class AiEditionService { } } - async addAsset(projectId: string, path: string, label?: string): Promise { - const document = await this.options.documents.addAsset(projectId, { path, label }); - const assetId = document.project.primaryAssetId ?? document.assets.at(-1)?.id ?? ""; + async addAsset( + projectId: string, + path: string, + label?: string, + kind?: "video" | "audio", + ): Promise { + const document = await this.options.documents.addAsset(projectId, { path, label, kind }); + // The just-added asset is always the last one; primaryAssetId is only a + // fallback for the video case and would point at the wrong asset for an + // audio import (which never claims primary), so prefer the tail. + const assetId = document.assets.at(-1)?.id ?? document.project.primaryAssetId ?? ""; return { assetId, document }; } diff --git a/electron/preload.ts b/electron/preload.ts index 6aff16407..7873bef90 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -276,6 +276,12 @@ contextBridge.exposeInMainWorld("electronAPI", { openVideoFilePicker: () => { return ipcRenderer.invoke("open-video-file-picker"); }, + openAudioFilePicker: () => { + return ipcRenderer.invoke("open-audio-file-picker"); + }, + saveRecordedVoiceover: (data: ArrayBuffer) => { + return ipcRenderer.invoke("save-recorded-voiceover", data); + }, setCurrentVideoPath: (path: string) => { return ipcRenderer.invoke("set-current-video-path", path); }, diff --git a/src/components/ai-edition/EditorEmptyState.test.tsx b/src/components/ai-edition/EditorEmptyState.test.tsx index acb513e4b..be7f130b7 100644 --- a/src/components/ai-edition/EditorEmptyState.test.tsx +++ b/src/components/ai-edition/EditorEmptyState.test.tsx @@ -53,6 +53,7 @@ const sampleDoc = vi.hoisted( }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }), ); diff --git a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx index c5b45637c..52eb7b2ad 100644 --- a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx +++ b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx @@ -77,6 +77,7 @@ const DOC: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/components/ai-edition/ExportDialog.test.ts b/src/components/ai-edition/ExportDialog.test.ts index 3aa8d85b5..ed2f1ab1a 100644 --- a/src/components/ai-edition/ExportDialog.test.ts +++ b/src/components/ai-edition/ExportDialog.test.ts @@ -57,6 +57,7 @@ function doc(assets: AxcutAsset[], clips: AxcutClip[]): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index b371d100c..58d2ad83b 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -14,7 +14,7 @@ import { replaceTimeline as replaceTimelineOp, } from "@/lib/ai-edition/document/timeline"; import { isModalOpen } from "@/lib/ai-edition/modalGuard"; -import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; +import { type AxcutAudioTrack, type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useAssetTranscriptions, @@ -44,6 +44,7 @@ import { import { Preview } from "./Preview"; import type { TrimTarget } from "./RightPanes"; import { importPendingRecording } from "./recordingImport"; +import { AddAudioLayerDialog } from "./v4/AddAudioLayerDialog"; import v4 from "./v4/EditorShellV4.module.css"; import { type EditorMode, EditorTopBar } from "./v4/EditorTopBar"; import { type Facet, FloatingInspector } from "./v4/FloatingInspector"; @@ -73,6 +74,10 @@ interface SeekTarget { * handlers that need it. The store write cadence is unchanged — `currentTimeSec` * is still the source of truth, still updated every frame. */ +// Stable empty list: a fresh `[]` each render would churn the preview's audio +// element set on every playhead tick. +const NO_AUDIO_TRACKS: AxcutAudioTrack[] = []; + function NativePlaybackSync({ visibleClips, clips, @@ -427,7 +432,16 @@ export function NewEditorShell() { const handleDropAsset = useCallback( (assetId: string) => enqueueTimelineWrite(() => { - const at = useProjectStore.getState().document?.timeline.clips.length ?? 0; + const doc = useProjectStore.getState().document; + // An audio asset has no video, so it must never become a clip (issue + // #350) — it goes on the audio lane as a track. Adding it "to the + // timeline" reuses its existing track if it already has one (importing + // already placed one) so the same file can't stack up duplicate lanes. + if (doc?.assets.find((a) => a.id === assetId)?.kind === "audio") { + if (doc.audioTracks.some((t) => t.assetId === assetId)) return Promise.resolve(); + return tl.addAudioTrack(assetId).then(() => undefined); + } + const at = doc?.timeline.clips.length ?? 0; return tl.insertClipAt(assetId, at); }).catch((error) => { toast.error(te("mediaStage.couldNotAddAsset"), { @@ -770,6 +784,56 @@ export function NewEditorShell() { }); }, []); + // Voiceover recording (the one audio gesture that is not a file import). The + // dialog owns the mic; the shell owns the transport and the placement. + const [voiceoverFlow, setVoiceoverFlow] = useState<{ maxDurationSec: number } | null>(null); + // The playhead as it was when RECORDING STARTED. Recording plays the video so + // the user can narrate what they see, which means the live playhead has moved + // on by the take's own length by the time the take ends — reading it then + // placed every voiceover one full take-length to the right of where it was + // spoken. Captured on the way in, used on the way out. + const voiceoverStartSecRef = useRef(0); + + const openVoiceoverFlow = useCallback(() => { + const doc = useProjectStore.getState().document; + if (!doc) return; + const total = doc.timeline.clips.reduce((max, c) => Math.max(max, c.timelineEndSec), 0); + const playhead = useProjectStore.getState().currentTimeSec; + voiceoverStartSecRef.current = playhead; + // Recording stops itself at the end of the timeline: a take can never + // outlive the video it was recorded over. + setVoiceoverFlow({ maxDurationSec: Math.max(0.5, total - playhead) }); + }, []); + + // Silences the timeline's own audio tracks for the duration of a take — see + // where it is passed to the preview. + const [voiceoverRecording, setVoiceoverRecording] = useState(false); + + const handleVoiceoverRecordingStart = useCallback(() => { + // Re-capture: the user may have scrubbed between opening the dialog and + // hitting Record, and playback starts from wherever the playhead is now. + voiceoverStartSecRef.current = useProjectStore.getState().currentTimeSec; + setVoiceoverRecording(true); + if (videoElement?.paused) void videoElement.play().catch(() => undefined); + }, [videoElement]); + + const handleVoiceoverRecordingStop = useCallback(() => { + setVoiceoverRecording(false); + videoElement?.pause(); + }, [videoElement]); + + const handleVoiceoverReady = useCallback( + async (assetId: string, durationSec: number) => { + setVoiceoverFlow(null); + await tl.addAudioTrack(assetId, voiceoverStartSecRef.current, { + kind: "voiceover", + durationSec, + spanSec: durationSec, + }); + }, + [tl], + ); + const pasteRegion = useCallback(async () => { const doc = useProjectStore.getState().document; if (!doc) return; @@ -786,6 +850,21 @@ export function NewEditorShell() { return; } + // Validate before building anything. The clipboard outlives the project, so + // a track copied in one project and pasted in another would reference an + // asset that only exists back where it came from — a pill that plays + // nothing and exports nothing. Audio is the only kind carrying a reference + // out of the document today; the next one belongs here too, rather than in + // its own branch below. + const referencedAssetId = (snapshot.region as { assetId?: unknown }).assetId; + if ( + typeof referencedAssetId === "string" && + !doc.assets.some((a) => a.id === referencedAssetId) + ) { + toast.error(te("regionClipboard.pasteAssetMissing")); + return; + } + const { anchorRegionsWithDerivedMs } = await import("@/lib/ai-edition/timeline/timelineMap"); const { createId } = await import("@/lib/ai-edition/document/ids"); @@ -793,6 +872,30 @@ export function NewEditorShell() { const timeMs = Math.round(useProjectStore.getState().currentTimeSec * 1000); const src = snapshot.region as { startMs: number; endMs: number }; const prefix = snapshot.kind === "annotation" ? "ann" : snapshot.kind; + + // Audio re-ventilates through its own anchorer, which advances each + // fragment's source offset — the generic one would copy the offset into + // every fragment and restart the file at each cut. + if (snapshot.kind === "audio") { + const { anchorAudioTrackFragments } = await import("@/lib/ai-edition/document/audioTracks"); + const track = { + ...(snapshot.region as unknown as AxcutAudioTrack), + id: createId("audio"), + trackId: undefined, + startMs: timeMs, + endMs: timeMs + (Number(src.endMs) - Number(src.startMs)), + }; + const fragments = anchorAudioTrackFragments(track, doc.timeline.clips, () => + createId("audio"), + ); + if (fragments.length === 0) return; + await saveDocument( + { ...doc, audioTracks: [...doc.audioTracks, ...fragments] }, + { history: true }, + ); + toast.success("Region pasted"); + return; + } const pasted = { ...snapshot.region, id: createId(prefix), @@ -842,7 +945,7 @@ export function NewEditorShell() { // `tl` belongs here now that the trim branch calls tl.addTrim: useTimeline // returns a fresh object each render, so memoizing on saveDocument alone // would paste through a callback holding a stale document. - }, [saveDocument, tl]); + }, [saveDocument, tl, te]); // Copy the SELECTED pill. Reads the same arrays the lanes render, so what gets // copied is what the user is looking at — the old version dug into the raw @@ -870,6 +973,22 @@ export function NewEditorShell() { return; } + // An audio track is stored as one fragment per clip it covers; the user + // copied the PILL, so collapse it back before it goes on the clipboard. + if (sel.kind === "audio") { + const { collapseTracksToPills, trackGroupId } = await import( + "@/lib/ai-edition/document/audioTracks" + ); + const [pill] = collapseTracksToPills( + tl.audioTracks.filter((t) => trackGroupId(t) === sel.id), + ); + if (!pill) return; + copyRegion({ kind: "audio", region: pill as unknown as Record }); + setCopiedClipId(null); + toast.success("Region copied"); + return; + } + const source = sel.kind === "zoom" ? tl.zoomRegions @@ -948,6 +1067,14 @@ export function NewEditorShell() { } if (tl.selection) { void tl.removeRegion(tl.selection.kind, tl.selection.id); + return; + } + // An audio track is selected through its OWN channel, not `selection` + // (the two are mutually exclusive — see addAudioTrack), so it needs its + // own branch here or Delete does nothing on the one lane that looks + // exactly like every other. + if (tl.selectedAudioTrackId) { + void tl.removeAudioTrack(tl.selectedAudioTrackId); } }; @@ -1032,6 +1159,18 @@ export function NewEditorShell() { void tl.addAnnotation(newRegionDurationSec()); return; } + // Unlike its neighbours this opens a file picker rather than dropping a region at + // the playhead — there is nothing to size, so it takes no duration (issue #350). + if (matchesShortcut(e, shortcuts.addAudio, isMac)) { + e.preventDefault(); + void tl.addAudio(); + return; + } + if (matchesShortcut(e, shortcuts.addVoiceover, isMac)) { + e.preventDefault(); + openVoiceoverFlow(); + return; + } if (matchesShortcut(e, shortcuts.addSpeed, isMac)) { e.preventDefault(); void tl.addSpeed(newRegionDurationSec()); @@ -1235,6 +1374,16 @@ export function NewEditorShell() { hasProject={hasProject} hasAsset={hasAsset} videoSources={videoSources} + // Imported audio tracks (issue #350). `videoSources` already + // resolves a URL for every asset (audio included), so it doubles as + // the audio source list; VirtualPreview looks each track up by assetId. + // Nothing already on the timeline plays while a take is being + // recorded. On speakers it bleeds straight into the microphone + // and lands in the new take; even on headphones, narrating over + // an earlier voiceover is not what the button offers. The video + // itself keeps playing — that is what the user is narrating to. + audioTracks={voiceoverRecording ? NO_AUDIO_TRACKS : tl.audioTracks} + audioSources={videoSources} clips={clips} zoomRegions={tl.zoomRegions} speedRegions={tl.speedRegions} @@ -1324,6 +1473,7 @@ export function NewEditorShell() { onTogglePlay={togglePlay} onPrevClip={handlePrevClip} onNextClip={handleNextClip} + onAddVoiceover={openVoiceoverFlow} onEditClip={setEditClipTarget} /> @@ -1383,6 +1533,21 @@ export function NewEditorShell() { onChoose={handleConfirmUnsaved} /> setExportOpen(false)} document={document} /> + { + // Belt and braces: the recorder's own stop handler clears this, but a + // flow that ends any other way must not leave the timeline muted. + setVoiceoverRecording(false); + setVoiceoverFlow(null); + }} + onComplete={(assetId, durationSec) => { + void handleVoiceoverReady(assetId, durationSec); + }} + onRecordingStart={handleVoiceoverRecordingStart} + onRecordingStop={handleVoiceoverRecordingStop} + /> ); } diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index aaf04f2db..cd6868e75 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -3,6 +3,7 @@ import type { CameraFullscreenRegion, ZoomFocus } from "@/components/video-edito import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAnnotationRegion, + AxcutAudioTrack, AxcutClip, AxcutTrimRange, AxcutZoomRegion, @@ -21,6 +22,12 @@ interface PreviewProps { hasProject: boolean; hasAsset: boolean; videoSources: VideoSource[]; + /** Imported audio tracks and the (unfiltered) asset URLs they resolve to + * (issue #350). Passed straight through to VirtualPreview — unlike the video + * `previewSources` below, these are NOT narrowed to clip-referenced assets, + * since an audio track has no clip. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; @@ -52,6 +59,8 @@ export function Preview({ hasProject, hasAsset, videoSources, + audioTracks = [], + audioSources = [], clips, zoomRegions, speedRegions, @@ -178,6 +187,8 @@ export function Preview({ <> ; interface PreviewCanvasProps { videoSources: VideoSource[]; + /** Imported audio tracks + their asset URLs (issue #350), forwarded to + * VirtualPreview. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 2e15f7b12..19946e085 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -13,6 +13,7 @@ import { Layout as LayoutIcon, Loader2, MousePointerClick, + Music, Sliders, Trash2, } from "lucide-react"; @@ -38,6 +39,7 @@ import defaultCursorPreviewUrl from "@/assets/cursors/Cursor=Default.svg"; import GradientEditor, { type GradientEditorState } from "@/components/ui/gradient-editor"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; +import { collapseTracksToPills, trackGroupId } from "@/lib/ai-edition/document/audioTracks"; import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; import type { AxcutAsset, @@ -52,6 +54,7 @@ import { } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; +import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { buildAggregatedSections, type ClipSection, @@ -2303,6 +2306,166 @@ export function AudioPane() { ); } +type TimelineApi = ReturnType; + +// Per-track controls for the selected imported audio track (issue #350). Shown by +// the inspector in place of the facet when an audio track is selected (see +// FloatingInspector). The header is the generic "Audio track"; the body leads +// with the file name, then the volume (a local live value during the drag, +// committed as one undo step on release), then a delete button styled like the +// region panes' (position and mute are edited on the lane itself). +// Longest fade the inspector offers. Past a few seconds a fade stops reading as +// a fade and starts reading as a level change, and the track's own span caps it +// anyway (`resolveFadeSecs` reduces one that does not fit). +const FADE_MAX_MS = 5000; + +export function AudioTrackPane({ tl }: { tl: TimelineApi }) { + const ts = useScopedT("settings"); + const trackId = tl.selectedAudioTrackId; + // The document stores one clip-anchored fragment per clip the track covers; + // the inspector edits the user-visible TRACK, so collapse first. Editing a + // single fragment would let the halves of a split take disagree. + const track = trackId + ? collapseTracksToPills(tl.audioTracks.filter((t) => trackGroupId(t) === trackId))[0] + : undefined; + const asset = track ? tl.assets.find((a) => a.id === track.assetId) : undefined; + // Live-drag values; null means "show the committed value". + const [liveGain, setLiveGain] = useState(null); + const [liveFadeIn, setLiveFadeIn] = useState(null); + const [liveFadeOut, setLiveFadeOut] = useState(null); + // Drop the live value when the selected track changes: a drag released outside + // the input never fires onCommit, so without this an uncommitted -10 dB from + // track A would show as track B's gain the moment B is selected. + // biome-ignore lint/correctness/useExhaustiveDependencies: trackId is the trigger, not a read — the body only resets the live value. + useEffect(() => { + setLiveGain(null); + setLiveFadeIn(null); + setLiveFadeOut(null); + }, [trackId]); + if (!track) return null; + const fileName = track.label || asset?.label || asset?.originalPath?.split(/[\\/]/).pop() || ""; + + // Match the region panes' danger-outlined delete button (see SelectionPane). + const deleteBtnStyle: CSSProperties = { + display: "flex", + width: "100%", + alignItems: "center", + justifyContent: "center", + gap: 7, + padding: "9px 14px", + borderRadius: 10, + border: "1px solid var(--danger)", + background: "var(--danger-soft)", + color: "var(--danger)", + font: "600 13px var(--font-display)", + cursor: "pointer", + }; + + return ( + } + helpText={ts("audioTrack.help")} + > +
+ {fileName} +
+
+ setLiveGain(value)} + onCommit={() => { + if (liveGain !== null) void tl.setAudioTrackGain(track.id, liveGain); + setLiveGain(null); + }} + /> + { + if (liveFadeIn !== null) void tl.updateAudioTrack(track.id, { fadeInMs: liveFadeIn }); + setLiveFadeIn(null); + }} + /> + { + if (liveFadeOut !== null) + void tl.updateAudioTrack(track.id, { fadeOutMs: liveFadeOut }); + setLiveFadeOut(null); + }} + /> +
+
+ {ts("audioTrack.mute")} + void tl.updateAudioTrack(track.id, { muted: v })} + /> +
+
+ {ts("audioTrack.loop")} + void tl.setAudioTrackLoop(track.id, v)} + /> +
+ + +
+ ); +} + // ─── Cursor ─────────────────────────────────────────────────────── function safeAssetUrl(relativePath: string): string { diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts index 208bf05a2..e1d6fec6d 100644 --- a/src/components/ai-edition/VirtualPreview.audio.test.ts +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; +import { projectRawTimelineSecToPlayback } from "@/lib/ai-edition/document/timeline"; +import type { AxcutAudioTrack, AxcutClip, AxcutTrimRange } from "@/lib/ai-edition/schema"; import { applyPreviewAudioSettings, type PreviewAudioGraph, resolveAudioTrackPlayback, + resolveTimelineAudioPlayback, + timelineAudioFadeAt, } from "./VirtualPreview"; /** Minimal stand-in: the function only ever touches `gain.gain.value`. */ @@ -80,3 +84,227 @@ describe("applyPreviewAudioSettings", () => { expect(graph.gain.gain.value).toBeCloseTo(0.5, 4); }); }); + +describe("resolveTimelineAudioPlayback", () => { + // A 6s track placed 10s into the RAW timeline, playing the source from 2s in. + const track: AxcutAudioTrack = { + id: "t1", + assetId: "a1", + kind: "music", + startMs: 10_000, + endMs: 16_000, + durationSec: 20, + offsetMs: 2000, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", + }; + const spanSec = (t: AxcutAudioTrack) => (t.endMs - t.startMs) / 1000; + + // Second arg is the track head projected to output seconds; with no trims it is + // just the raw head (10), so these read the same as before the output-space + // change — the projection is exercised separately below. + it("maps the playhead to a source position offset by the track offset", () => { + // 3s into the track's span → 2 (offset) + 3 = 5s of source. + expect(resolveTimelineAudioPlayback(13, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 5, + shouldPlay: true, + }); + }); + + it("does not play before the track starts, parked at the in-point", () => { + expect(resolveTimelineAudioPlayback(9, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 2, + shouldPlay: false, + }); + }); + + it("does not play past the end of its span, parked at the out-point", () => { + expect(resolveTimelineAudioPlayback(16, 10, track, spanSec(track))).toEqual({ + targetTimeSec: 8, + shouldPlay: false, + }); + }); + + it("goes silent when the file runs out before the span does", () => { + // A 5s file under a 10s span: at 6s in there is no source left, and the + // element holds at the end rather than restarting. + const short = { ...track, endMs: 20_000, offsetMs: 0, durationSec: 5 }; + expect(resolveTimelineAudioPlayback(14, 10, short, spanSec(short))).toEqual({ + targetTimeSec: 4, + shouldPlay: true, + }); + expect(resolveTimelineAudioPlayback(16, 10, short, spanSec(short)).shouldPlay).toBe(false); + }); + + it("folds a looping track back into its window, in phase with the export", () => { + // 4s of source (offset 0, 4s file) under a 10s span. + const looped = { ...track, endMs: 20_000, offsetMs: 0, durationSec: 4, loop: true }; + const span = spanSec(looped); + expect(resolveTimelineAudioPlayback(13, 10, looped, span).targetTimeSec).toBeCloseTo(3, 6); + // 5s in is 1s into the second repeat — the export's second mix entry agrees. + expect(resolveTimelineAudioPlayback(15, 10, looped, span).targetTimeSec).toBeCloseTo(1, 6); + expect(resolveTimelineAudioPlayback(15, 10, looped, span).shouldPlay).toBe(true); + // Past the span it stops, however much file is left. + expect(resolveTimelineAudioPlayback(21, 10, looped, span).shouldPlay).toBe(false); + }); + + // The regression: an interior trim must NOT skip the track's own content, so the + // preview stays byte-for-byte with `audio::mix_external_tracks`, which overlays the + // decoded window contiguously. Same scenario as Etienne's review and the projection's + // own test: a 10s clip with raw 2..4 cut, background track at raw head 0 spanning 0..10. + it("plays contiguously across an interior trim, matching the export", () => { + const clip: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }; + const trim: AxcutTrimRange = { + id: "trim_1", + assetId: "asset_1", + startSec: 2, + endSec: 4, + origin: "user", + reason: "", + }; + const bgm: AxcutAudioTrack = { + ...track, + id: "bgm", + assetId: "a2", + startMs: 0, + endMs: 10_000, + durationSec: 10, + offsetMs: 0, + }; + const project = (rawSec: number) => projectRawTimelineSecToPlayback([clip], [trim], rawSec); + const outputStart = project(bgm.startMs / 1000); // 0 + + // Raw playhead 5 sits 1s past the 2s cut → output 3. The track is a contiguous + // block, so it must be at source 3 — NOT source 5, which the old raw-space + // `local` produced (the 2s desync). + expect(resolveTimelineAudioPlayback(project(5), outputStart, bgm, spanSec(bgm))).toEqual({ + targetTimeSec: 3, + shouldPlay: true, + }); + // Just before the cut is unaffected: raw 1 → output 1 → source 1. + expect( + resolveTimelineAudioPlayback(project(1), outputStart, bgm, spanSec(bgm)).targetTimeSec, + ).toBeCloseTo(1, 6); + }); +}); + +describe("resolveTimelineAudioPlayback under a trim", () => { + const clip: AxcutClip = { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 20, + timelineStartSec: 0, + timelineEndSec: 20, + wordRefs: [], + origin: "user", + reason: "", + }; + const trim: AxcutTrimRange = { + id: "trim_1", + assetId: "asset_1", + startSec: 4, + endSec: 8, + origin: "user", + reason: "", + }; + const project = (rawSec: number) => projectRawTimelineSecToPlayback([clip], [trim], rawSec); + + const buried: AxcutAudioTrack = { + id: "buried", + assetId: "a1", + kind: "music", + startMs: 5000, + endMs: 7000, + durationSec: 30, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 0, + fadeOutMs: 0, + muted: false, + label: "", + origin: "user", + }; + + it("never plays a track buried inside the trim", () => { + // Both ends project onto the cut, so the track's OUTPUT span is zero. Read + // off the raw span instead it stayed 2s long and played at the boundary, + // with nothing on screen to explain the sound. + const outStart = project(buried.startMs / 1000); + const outSpan = project(buried.endMs / 1000) - outStart; + expect(outSpan).toBeCloseTo(0, 6); + for (const raw of [3, 5, 6, 9, 12]) { + expect(resolveTimelineAudioPlayback(project(raw), outStart, buried, outSpan).shouldPlay).toBe( + false, + ); + } + }); + + it("plays a track that merely crosses the trim, for the length that survives", () => { + const crossing = { ...buried, id: "crossing", startMs: 2000, endMs: 12_000 }; + const outStart = project(crossing.startMs / 1000); + const outSpan = project(crossing.endMs / 1000) - outStart; + // Raw 2..12 with raw 4..8 cut leaves 6s of programme. + expect(outSpan).toBeCloseTo(6, 6); + expect(resolveTimelineAudioPlayback(project(3), outStart, crossing, outSpan).shouldPlay).toBe( + true, + ); + // Just past the end of what survives. + expect( + resolveTimelineAudioPlayback(outStart + 6.1, outStart, crossing, outSpan).shouldPlay, + ).toBe(false); + }); +}); + +describe("timelineAudioFadeAt", () => { + const track: AxcutAudioTrack = { + id: "t1", + assetId: "a1", + kind: "music", + startMs: 0, + endMs: 10_000, + durationSec: 20, + offsetMs: 0, + gainDb: 0, + loop: false, + fadeInMs: 1000, + fadeOutMs: 2000, + muted: false, + label: "", + origin: "user", + }; + + it("ramps in and out over the track's own edges", () => { + expect(timelineAudioFadeAt(track, 0, 10)).toBe(0); + expect(timelineAudioFadeAt(track, 0.5, 10)).toBeCloseTo(0.5, 6); + expect(timelineAudioFadeAt(track, 5, 10)).toBe(1); + expect(timelineAudioFadeAt(track, 9, 10)).toBeCloseTo(0.5, 6); + expect(timelineAudioFadeAt(track, 10, 10)).toBe(0); + }); + + it("is silent when muted", () => { + expect(timelineAudioFadeAt({ ...track, muted: true }, 5, 10)).toBe(0); + }); + + it("still reaches full volume when a fade is longer than the span", () => { + // Unreduced, the ramp never completes and the track plays near-silent. + const long = { ...track, fadeInMs: 20_000, fadeOutMs: 0 }; + expect(timelineAudioFadeAt(long, 2, 2)).toBe(1); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.playback.test.tsx b/src/components/ai-edition/VirtualPreview.playback.test.tsx index f0389fe2a..f4ffe88a0 100644 --- a/src/components/ai-edition/VirtualPreview.playback.test.tsx +++ b/src/components/ai-edition/VirtualPreview.playback.test.tsx @@ -207,3 +207,176 @@ describe("VirtualPreview playback across a clip boundary", () => { expect(video.pauseCalls).toHaveLength(0); }); }); + +// Issue #350 — imported audio tracks follow the RAW virtual playhead. The +// decision math is unit-tested in VirtualPreview.audio.test.ts; here we prove the +// rAF loop applies it to the mounted