Skip to content

Clip source offsetting #1003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Brendonovich merged 4 commits into from
Sep 15, 2025
Merged

Clip source offsetting #1003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e069097
working camera offsetting
Brendonovich Sep 11, 2025
516d0f1
audio offsett works too
Brendonovich Sep 11, 2025
7cc6c17
fix properly
Brendonovich Sep 11, 2025
288b006
find by segment index
Brendonovich Sep 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 128 additions & 11 deletions apps/desktop/src/routes/editor/ConfigSidebar.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { NumberField } from "@kobalte/core";
import {
Collapsible,
Collapsible as KCollapsible,
Expand Down Expand Up @@ -31,7 +32,7 @@ import {
Suspense,
type ValidComponent,
} from "solid-js";
import { createStore } from "solid-js/store";
import { createStore, produce } from "solid-js/store";
import { Dynamic } from "solid-js/web";
import toast from "solid-toast";
import colorBg from "~/assets/illustrations/color.webp";
Expand All @@ -43,6 +44,7 @@ import { generalSettingsStore } from "~/store";
import {
type BackgroundSource,
type CameraShape,
type ClipOffsets,
commands,
type SceneSegment,
type StereoMode,
Expand Down Expand Up @@ -2317,9 +2319,33 @@ function ClipSegmentConfig(props: {
segmentIndex: number;
segment: TimelineSegment;
}) {
const { setProject, setEditorState, project, projectActions } =
const { setProject, setEditorState, project, projectActions, meta } =
useEditorContext();

// Get current clip configuration
const clipConfig = () =>
project.clips?.find((c) => c.index === props.segmentIndex);
const offsets = () => clipConfig()?.offsets || {};

function setOffset(type: keyof ClipOffsets, offset: number) {
if (Number.isNaN(offset)) return;

setProject(
produce((proj) => {
const clips = (proj.clips ??= []);
let clip = clips.find(
(clip) => clip.index === (props.segment.recordingSegment ?? 0),
);
if (!clip) {
clip = { index: 0, offsets: {} };
clips.push(clip);
}

clip.offsets[type] = offset / 1000;
}),
);
}

return (
<>
<div class="flex flex-row justify-between items-center">
Expand Down Expand Up @@ -2348,20 +2374,111 @@ function ClipSegmentConfig(props: {
Delete
</EditorButton>
</div>
<ComingSoonTooltip>
<Field name="Hide Cursor" disabled value={<Toggle disabled />} />
</ComingSoonTooltip>
<ComingSoonTooltip>
<Field
name="Disable Smooth Cursor Movement"
disabled
value={<Toggle disabled />}

<div class="space-y-1">
<h3 class="font-medium text-gray-12">Clip Settings</h3>
<p class="text-gray-11">
These settings apply to all segments for the current clip
</p>
</div>

{meta().hasSystemAudio && (
<SourceOffsetField
name="System Audio Offset"
value={offsets().system_audio}
onChange={(offset) => {
setOffset("system_audio", offset);
}}
/>
</ComingSoonTooltip>
)}
{meta().hasMicrophone && (
<SourceOffsetField
name="Microphone Offset"
value={offsets().mic}
onChange={(offset) => {
setOffset("mic", offset);
}}
/>
)}
{meta().hasCamera && (
<SourceOffsetField
name="Camera Offset"
value={offsets().camera}
onChange={(offset) => {
setOffset("camera", offset);
}}
/>
)}

{/*<ComingSoonTooltip>
<Field name="Hide Cursor" disabled value={<Toggle disabled />} />
</ComingSoonTooltip>
<ComingSoonTooltip>
<Field
name="Disable Smooth Cursor Movement"
disabled
value={<Toggle disabled />}
/>
</ComingSoonTooltip>*/}
</>
);
}

function SourceOffsetField(props: {
name: string;
// seconds
value?: number;
onChange: (value: number) => void;
}) {
const rawValue = () => Math.round((props.value ?? 0) * 1000);

const [value, setValue] = createSignal(rawValue().toString());

return (
<Field name={props.name}>
<div class="flex flex-row items-center justify-between w-full -mt-2">
<div class="flex flex-row space-x-1 items-end">
<NumberField.Root
value={value()}
onChange={setValue}
rawValue={rawValue()}
onRawValueChange={(v) => {
props.onChange(v);
}}
>
<NumberField.Input
onBlur={() => {
if (!rawValue() || value() === "" || Number.isNaN(rawValue())) {
setValue("0");
props.onChange(0);
}
}}
class="w-[5rem] p-[0.375rem] border rounded-[0.5rem] bg-gray-1 focus-visible:outline-none"
/>
</NumberField.Root>
<span class="text-gray-11">ms</span>
</div>
<div class="text-gray-11 flex flex-row space-x-1">
{[-100, -10, 10, 100].map((v) => (
<button
type="button"
onClick={() => {
const currentValue = rawValue() + v;
props.onChange(currentValue);
setValue(currentValue.toString());
}}
class="text-gray-11 hover:text-gray-12 text-xs px-1 py-0.5 bg-gray-1 border border-gray-3 rounded"
>
{Math.sign(v) > 0 ? "+" : "-"}
{Math.abs(v)}ms
</button>
))}
</div>
</div>
</Field>
);
}

function SceneSegmentConfig(props: {
segmentIndex: number;
segment: SceneSegment;
Expand Down
31 changes: 16 additions & 15 deletions apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,21 +355,22 @@ export function ClipTrack(
createEventListener(e.currentTarget, "mouseup", (e) => {
dispose();

// If there's only one segment, don't open the clip config panel
// since there's nothing to configure - just let the normal click behavior happen
const hasOnlyOneSegment = segments().length === 1;

if (hasOnlyOneSegment) {
// Clear any existing selection (zoom, layout, etc.) when clicking on a clip
// This ensures the sidebar updates properly
setEditorState("timeline", "selection", null);
} else {
// When there are multiple segments, show the clip configuration
setEditorState("timeline", "selection", {
type: "clip",
index: i(),
});
}
// // If there's only one segment, don't open the clip config panel
// // since there's nothing to configure - just let the normal click behavior happen
// const hasOnlyOneSegment = segments().length === 1;

// if (hasOnlyOneSegment) {
// // Clear any existing selection (zoom, layout, etc.) when clicking on a clip
// // This ensures the sidebar updates properly
// setEditorState("timeline", "selection", null);
// } else {
// When there are multiple segments, show the clip configuration
setEditorState("timeline", "selection", {
type: "clip",
index: i(),
});

// }
props.handleUpdatePlayhead(e);
});
});
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/src/utils/createPresets.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export function createPresets() {
const config = { ...preset.config };
// @ts-expect-error we reeeally don't want the timeline in the preset
config.timeline = undefined;
config.clips = undefined;

await updatePresets((store) => {
store.presets.push({ name: preset.name, config });
Expand Down
4 changes: 3 additions & 1 deletion apps/desktop/src/utils/tauri.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,8 @@ export type CaptionSettings = { enabled: boolean; font: string; size: number; co
export type CaptionsData = { segments: CaptionSegment[]; settings: CaptionSettings }
export type CaptureDisplay = { id: DisplayId; name: string; refresh_rate: number }
export type CaptureWindow = { id: WindowId; owner_name: string; name: string; bounds: LogicalBounds; refresh_rate: number }
export type ClipConfiguration = { index: number; offsets: ClipOffsets }
export type ClipOffsets = { camera?: number; mic?: number; system_audio?: number }
export type CommercialLicense = { licenseKey: string; expiryDate: number | null; refresh: number; activatedOn: number }
Comment on lines +352 to 354
Copy link
Contributor

@coderabbitai coderabbitai bot Sep 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Verification agent

🧩 Analysis chain

Do not manually edit this generated file; ensure these types come from tauri-specta codegen.

If these additions weren’t produced by codegen, please revert and re-generate. Also verify Rust-side shapes: TS shows optional fields (ClipOffsets.camera/mic/system_audio and ProjectConfiguration.clips?), which must match Rust (Option vs plain). Mismatch will desync IPC contracts.

To confirm, please check the Rust definitions:

🏁 Script executed:

#!/bin/bash
# Verify Rust shapes for ProjectConfiguration.clips and ClipOffsets
fd -t f configuration.rs crates | xargs -I{} rg -n -C3 'struct\s+ProjectConfiguration|clips\s*:' {}
fd -t f configuration.rs crates | xargs -I{} rg -n -C3 'struct\s+ClipOffsets|impl\s+Default\s+for\s+ClipOffsets' {}
# Also locate get_segment_time to confirm index semantics
rg -n 'fn\s+get_segment_time' -C3

Length of output: 1593

Do not edit generated tauri.ts — regenerate from tauri-specta; Rust↔TS shapes mismatch

  • ClipOffsets mismatch: Rust (crates/project/src/configuration.rs, ~lines 590–596) defines camera/mic/system_audio as non-optional f32 with #[serde(default)]; TS currently has these as optional (camera?: number, etc.) — regenerate so TS fields are non-optional numbers.
  • ProjectConfiguration.clips mismatch: Rust (crates/project/src/configuration.rs, ~lines 606–622) has pub clips: Vec (#[serde(default)]); TS should use ClipConfiguration[] (not optional).
  • Index semantics: get_segment_time signature (crates/project/src/configuration.rs, lines 511 & 640) is fn get_segment_time(&self, frame_time: f64) -> Option<(f64, u32)> — segment index is a numeric u32; ensure ClipConfiguration.index is numeric and aligned.
  • Action: if apps/desktop/src/utils/tauri.ts was edited manually, revert and re-run tauri-specta codegen to resync IPC contracts.
🤖 Prompt for AI Agents 
In apps/desktop/src/utils/tauri.ts around lines 352 to 354, the generated
TypeScript shapes no longer match the Rust definitions: make ClipOffsets fields
non-optional numbers (camera: number; mic: number; system_audio: number), ensure
ClipConfiguration.index is a numeric number (not a string/optional) and
ProjectConfiguration.clips is ClipConfiguration[] (not optional), then revert
any manual edits to this file and re-run the tauri-specta codegen to regenerate
the file so IPC/Rust↔TS contracts are in sync.

export type Crop = { position: XY<number>; size: XY<number> }
export type CurrentRecording = { target: CurrentRecordingTarget; mode: RecordingMode }
Expand Down Expand Up @@ -411,7 +413,7 @@ export type PostDeletionBehaviour = "doNothing" | "reopenRecordingWindow"
export type PostStudioRecordingBehaviour = "openEditor" | "showOverlay"
export type Preset = { name: string; config: ProjectConfiguration }
export type PresetsStore = { presets: Preset[]; default: number | null }
export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background: BackgroundConfiguration; camera: Camera; audio: AudioConfiguration; cursor: CursorConfiguration; hotkeys: HotkeysConfiguration; timeline?: TimelineConfiguration | null; captions?: CaptionsData | null }
export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background: BackgroundConfiguration; camera: Camera; audio: AudioConfiguration; cursor: CursorConfiguration; hotkeys: HotkeysConfiguration; timeline?: TimelineConfiguration | null; captions?: CaptionsData | null; clips?: ClipConfiguration[] }
export type ProjectRecordingsMeta = { segments: SegmentRecordings[] }
export type RecordingDeleted = { path: string }
export type RecordingEvent = { variant: "Countdown"; value: number } | { variant: "Started" } | { variant: "Stopped" } | { variant: "Failed"; error: string }
Expand Down
28 changes: 19 additions & 9 deletions crates/audio/src/renderer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,16 @@ pub enum StereoMode {
MonoR,
}

pub struct AudioRendererTrack<'a> {
pub data: &'a AudioData,
pub gain: f32,
pub stereo_mode: StereoMode,
pub offset: isize,
}

// Renders a combination of audio tracks into a single stereo buffer
pub fn render_audio(
tracks: &[(&AudioData, f32, StereoMode)],
tracks: &[AudioRendererTrack],
offset: usize,
samples: usize,
out_offset: usize,
Expand All @@ -17,7 +24,7 @@ pub fn render_audio(
let samples = samples.min(
tracks
.iter()
.flat_map(|t| (t.0.samples().len() / t.0.channels() as usize).checked_sub(offset))
.flat_map(|t| (t.data.samples().len() / t.data.channels() as usize).checked_sub(offset))
.max()
.unwrap_or(0),
);
Expand All @@ -27,27 +34,30 @@ pub fn render_audio(
let mut right = 0.0;

for track in tracks {
let gain = gain_for_db(track.1);
let i = i.wrapping_add_signed(track.offset);

let data = track.data;
let gain = gain_for_db(track.gain);

if gain == f32::NEG_INFINITY {
continue;
}

if track.0.channels() == 1 {
if let Some(sample) = track.0.samples().get(offset + i) {
if data.channels() == 1 {
if let Some(sample) = data.samples().get(offset + i) {
left += sample * 0.707 * gain;
right += sample * 0.707 * gain;
}
} else if track.0.channels() == 2 {
} else if data.channels() == 2 {
let base_idx = offset * 2 + i * 2;
let Some(l_sample) = track.0.samples().get(base_idx) else {
let Some(l_sample) = data.samples().get(base_idx) else {
continue;
};
let Some(r_sample) = track.0.samples().get(base_idx + 1) else {
let Some(r_sample) = data.samples().get(base_idx + 1) else {
continue;
};

match track.2 {
match track.stereo_mode {
StereoMode::Stereo => {
left += l_sample * gain;
right += r_sample * gain;
Expand Down
32 changes: 25 additions & 7 deletions crates/editor/src/audio.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use cap_audio::{AudioData, FromSampleBytes, StereoMode, cast_f32_slice_to_bytes};
use cap_audio::{
AudioData, AudioRendererTrack, FromSampleBytes, StereoMode, cast_f32_slice_to_bytes,
};
use cap_media::MediaError;
use cap_media_info::AudioInfo;
use cap_project::{AudioConfiguration, ProjectConfiguration, TimelineConfiguration};
use cap_project::{AudioConfiguration, ClipOffsets, ProjectConfiguration, TimelineConfiguration};
use ffmpeg::{ChannelLayout, format as avformat, frame::Audio as FFAudio, software::resampling};
use ringbuf::{
HeapRb,
Expand Down Expand Up @@ -29,23 +31,27 @@ pub struct AudioSegment {
pub tracks: Vec<AudioSegmentTrack>,
}

// yeah this is cursed oh well
#[derive(Clone)]
pub struct AudioSegmentTrack {
data: Arc<AudioData>,
get_gain: fn(&AudioConfiguration) -> f32,
get_stereo_mode: fn(&AudioConfiguration) -> StereoMode,
get_offset: fn(&ClipOffsets) -> f32,
}

impl AudioSegmentTrack {
pub fn new(
data: Arc<AudioData>,
get_gain: fn(&AudioConfiguration) -> f32,
get_stereo_mode: fn(&AudioConfiguration) -> StereoMode,
get_offset: fn(&ClipOffsets) -> f32,
) -> Self {
Self {
data,
get_gain,
get_stereo_mode,
get_offset,
}
}

Expand All @@ -60,6 +66,10 @@ impl AudioSegmentTrack {
pub fn stereo_mode(&self, config: &AudioConfiguration) -> StereoMode {
(self.get_stereo_mode)(config)
}

pub fn offset(&self, offsets: &ClipOffsets) -> f32 {
(self.get_offset)(offsets)
}
}

impl AudioRenderer {
Expand Down Expand Up @@ -184,16 +194,24 @@ impl AudioRenderer {
let track_datas = tracks
.iter()
.map(|t| {
(
t.data().as_ref(),
if project.audio.mute {
let offsets = project
.clips
.iter()
.find(|c| c.index == start.segment_index)
.map(|c| c.offsets)
.unwrap_or_default();

AudioRendererTrack {
data: t.data().as_ref(),
gain: if project.audio.mute {
f32::NEG_INFINITY
} else {
let g = t.gain(&project.audio);
if g < -30.0 { f32::NEG_INFINITY } else { g }
},
t.stereo_mode(&project.audio),
)
stereo_mode: t.stereo_mode(&project.audio),
offset: (t.offset(&offsets) * Self::SAMPLE_RATE as f32) as isize,
}
})
.collect::<Vec<_>>();

Expand Down
Loading
Loading