From 51c31e4aa9cdd25c88fc344538bc76274ce2a2b1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 01/24] feat(rendering): add create_bgra_texture to IOSurface cache --- crates/rendering/src/iosurface_texture.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/rendering/src/iosurface_texture.rs b/crates/rendering/src/iosurface_texture.rs index 4403146da54..f57d7e559d8 100644 --- a/crates/rendering/src/iosurface_texture.rs +++ b/crates/rendering/src/iosurface_texture.rs @@ -84,6 +84,26 @@ impl IOSurfaceTextureCache { .ok_or(IOSurfaceTextureError::TextureCreationFailed) } + pub fn create_bgra_texture( + &self, + io_surface: &io::Surf, + width: u32, + height: u32, + ) -> Result, IOSurfaceTextureError> { + let mut desc = mtl::TextureDesc::new_2d( + mtl::PixelFormat::Bgra8UNorm, + width as usize, + height as usize, + false, + ); + desc.set_storage_mode(mtl::StorageMode::Shared); + desc.set_usage(mtl::TextureUsage::SHADER_READ); + + self.metal_device + .new_texture_with_surf(&desc, io_surface, 0) + .ok_or(IOSurfaceTextureError::TextureCreationFailed) + } + pub fn create_rgba_texture( &self, io_surface: &io::Surf, From 44c29524c75001336a4dd065960f4ea11b1298bc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 02/24] refactor(camera-ffmpeg): extract sample_buf_as_ffmpeg conversion --- crates/camera-ffmpeg/src/macos.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index b761363130f..316e0864b38 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -60,9 +60,20 @@ static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false); impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { - let native = self.native(); + sample_buf_as_ffmpeg(self.native().sample_buf()) + } +} - let mut image_buf = native.image_buf().ok_or(AsFFmpegError::NoImageBuffer)?; +// Standalone so consumers holding a bare retained CMSampleBuffer (e.g. the +// native camera preview's CPU fallback) can reuse the same conversion. +pub fn sample_buf_as_ffmpeg( + sample_buf: &cm::SampleBuf, +) -> Result { + { + let mut image_buf = sample_buf + .image_buf() + .map(|b| b.retained()) + .ok_or(AsFFmpegError::NoImageBuffer)?; let width = image_buf.width(); let height = image_buf.height(); @@ -87,7 +98,9 @@ impl CapturedFrameExt for CapturedFrame { ), ]; - let format_desc = native.sample_buf().format_desc().unwrap(); + let format_desc = sample_buf + .format_desc() + .ok_or(AsFFmpegError::NoImageBuffer)?; let bytes_lock = ImageBufExt::base_addr_lock( image_buf.as_mut(), From 6535b57b59c90093389b415e545582c8216b2576 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 03/24] feat(camera-effects): add Linux ONNX Runtime dynamic loading --- crates/camera-effects/Cargo.toml | 5 ++++- crates/camera-effects/src/segmentation.rs | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/crates/camera-effects/Cargo.toml b/crates/camera-effects/Cargo.toml index c32c55a5404..b2d999d42da 100644 --- a/crates/camera-effects/Cargo.toml +++ b/crates/camera-effects/Cargo.toml @@ -13,7 +13,10 @@ tracing.workspace = true bytemuck = { version = "1.7", features = ["derive"] } ndarray = "0.16" -[target.'cfg(not(any(target_os = "macos", target_os = "windows")))'.dependencies] +[target.'cfg(target_os = "linux")'.dependencies] +ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "load-dynamic", "api-23"] } + +[target.'cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))'.dependencies] ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "download-binaries", "copy-dylibs", "tls-native"] } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/camera-effects/src/segmentation.rs b/crates/camera-effects/src/segmentation.rs index e2d56601f97..bf8a7ca564a 100644 --- a/crates/camera-effects/src/segmentation.rs +++ b/crates/camera-effects/src/segmentation.rs @@ -1,9 +1,14 @@ use anyhow::Context; use ort::session::Session; use ort::value::Value; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use std::path::PathBuf; +#[cfg(target_os = "macos")] +const ORT_LIBRARY_NAME: &str = "libonnxruntime.dylib"; +#[cfg(target_os = "linux")] +const ORT_LIBRARY_NAME: &str = "libonnxruntime.so"; + const MODEL_BYTES: &[u8] = include_bytes!("../assets/selfie_segmentation.onnx"); const MODEL_INPUT_SIZE: usize = 256; @@ -85,7 +90,7 @@ fn create_session() -> anyhow::Result { Ok(session) } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] fn init_runtime() -> anyhow::Result<()> { let path = std::env::var_os("ORT_DYLIB_PATH") .map(PathBuf::from) @@ -94,7 +99,7 @@ fn init_runtime() -> anyhow::Result<()> { .into_iter() .find(|path| path.exists()) }) - .context("Failed to find macOS ONNX Runtime dylib")?; + .context("Failed to find ONNX Runtime library")?; let _ = ort::init_from(&path) .with_context(|| format!("Failed to load ONNX Runtime from {}", path.display()))? @@ -103,19 +108,19 @@ fn init_runtime() -> anyhow::Result<()> { Ok(()) } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "linux")))] fn init_runtime() -> anyhow::Result<()> { Ok(()) } -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] fn onnx_runtime_candidates() -> Vec { let mut candidates = Vec::new(); if let Ok(exe) = std::env::current_exe() && let Some(exe_dir) = exe.parent() { - candidates.push(exe_dir.join("libonnxruntime.dylib")); + candidates.push(exe_dir.join(ORT_LIBRARY_NAME)); if let Some(contents_dir) = exe_dir.parent() { candidates.push( @@ -123,14 +128,15 @@ fn onnx_runtime_candidates() -> Vec { .join("Resources") .join("onnxruntime") .join("lib") - .join("libonnxruntime.dylib"), + .join(ORT_LIBRARY_NAME), ); } } candidates.push( PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../target/native-deps/onnxruntime/lib/libonnxruntime.dylib"), + .join("../../target/native-deps/onnxruntime/lib") + .join(ORT_LIBRARY_NAME), ); candidates From 5c89dc5988e4f7e0802239b83ac41bfc92274c23 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 04/24] perf(recording): optimize camera feed sender and ffmpeg path --- crates/recording/src/feeds/camera.rs | 60 ++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index d9ba332092f..2f9a7954837 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -45,6 +45,7 @@ pub struct CameraFeed { setup_generation: u64, state: State, senders: Vec>, + ffmpeg_sender_count: Arc, native_senders: Vec>, native_sender_count: Arc, on_ready: Vec>, @@ -189,6 +190,7 @@ impl Default for CameraFeed { attached: None, }), senders: Vec::new(), + ffmpeg_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), native_senders: Vec::new(), native_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), on_ready: Vec::new(), @@ -337,6 +339,7 @@ struct CameraSetupArgs { actor_ref: ActorRef, new_frame_recipient: Recipient, native_frame_recipient: Recipient, + ffmpeg_sender_count: Arc, native_sender_count: Arc, flow: CameraSetupFlow, } @@ -351,6 +354,7 @@ fn spawn_camera_setup( actor_ref, new_frame_recipient, native_frame_recipient, + ffmpeg_sender_count, native_sender_count, flow, } = args; @@ -401,6 +405,7 @@ fn spawn_camera_setup( settings, new_frame_recipient, native_frame_recipient, + ffmpeg_sender_count, native_sender_count, ) .await; @@ -517,6 +522,11 @@ fn release_camera_thread(handle: std::thread::JoinHandle<()>) { let _ = handle.join(); } else { warn!("Camera setup thread is still running after cancellation"); + let _ = std::thread::Builder::new() + .name("camera-setup-reaper".to_string()) + .spawn(move || { + let _ = handle.join(); + }); } } @@ -759,6 +769,7 @@ async fn setup_camera( settings: Option, recipient: Recipient, native_recipient: Recipient, + ffmpeg_sender_count: Arc, native_sender_count: Arc, ) -> Result { let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; @@ -786,6 +797,15 @@ async fn setup_camera( .try_send(); } + // Until the ready signal fires the first frame must still be + // converted to derive VideoInfo; afterwards skip the full-frame + // copy entirely when nothing consumes ffmpeg frames. + if ready_signal.is_none() + && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Relaxed) == 0 + { + return; + } + let Ok(mut ff_frame) = frame.as_ffmpeg() else { return; }; @@ -837,6 +857,7 @@ async fn setup_camera( settings: Option, recipient: Recipient, native_recipient: Recipient, + ffmpeg_sender_count: Arc, native_sender_count: Arc, ) -> Result { let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; @@ -891,6 +912,15 @@ async fn setup_camera( } } + // Until the ready signal fires the first frame must still be + // converted to derive VideoInfo; afterwards skip the full-frame + // copy entirely when nothing consumes ffmpeg frames. + if ready_signal.is_none() + && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Relaxed) == 0 + { + return; + } + let Ok(mut ff_frame) = frame.as_ffmpeg() else { return; }; @@ -1049,6 +1079,7 @@ impl Message for CameraFeed { actor_ref, new_frame_recipient, native_frame_recipient, + ffmpeg_sender_count: self.ffmpeg_sender_count.clone(), native_sender_count: self.native_sender_count.clone(), flow: CameraSetupFlow::Open, }); @@ -1086,6 +1117,7 @@ impl Message for CameraFeed { actor_ref, new_frame_recipient, native_frame_recipient, + ffmpeg_sender_count: self.ffmpeg_sender_count.clone(), native_sender_count: self.native_sender_count.clone(), flow: CameraSetupFlow::Locked, }); @@ -1122,6 +1154,8 @@ impl Message for CameraFeed { } self.senders.clear(); + self.ffmpeg_sender_count + .store(0, std::sync::atomic::Ordering::Release); self.native_senders.clear(); self.native_sender_count .store(0, std::sync::atomic::Ordering::Release); @@ -1152,6 +1186,8 @@ impl Message for CameraFeed { debug!("CameraFeed: Adding new sender"); self.senders.push(msg.0); + self.ffmpeg_sender_count + .store(self.senders.len(), std::sync::atomic::Ordering::Release); } } @@ -1189,6 +1225,8 @@ impl Message for CameraFeed { _: &mut Context, ) -> Self::Reply { self.senders.retain(|sender| !sender.same_channel(&msg.0)); + self.ffmpeg_sender_count + .store(self.senders.len(), std::sync::atomic::Ordering::Release); } } @@ -1251,6 +1289,19 @@ fn send_frame_to_camera_senders( frame_num: u64, sender_label: &str, ) -> bool { + // A disconnected sender whose queue is still full would otherwise never be + // try_send'd again, leaving it (and its queued frames) retained forever. + let len_before_retain = senders.len(); + senders.retain(|sender| !sender.is_disconnected()); + let removed_disconnected = senders.len() != len_before_retain; + if removed_disconnected { + debug!( + "Removed {} disconnected {} senders before fanout", + len_before_retain - senders.len(), + sender_label + ); + } + let mut last_ready_sender = None; for (i, sender) in senders.iter().enumerate() { @@ -1267,7 +1318,7 @@ fn send_frame_to_camera_senders( } let Some(last_ready_sender) = last_ready_sender else { - return false; + return removed_disconnected; }; let mut frame = Some(frame); @@ -1311,7 +1362,7 @@ fn send_frame_to_camera_senders( } if to_remove.is_empty() { - return false; + return removed_disconnected; } debug!( @@ -1331,7 +1382,10 @@ impl Message for CameraFeed { async fn handle(&mut self, msg: NewFrame, _: &mut Context) -> Self::Reply { let frame_num = CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - send_frame_to_camera_senders(&mut self.senders, msg.0, frame_num, "Camera"); + if send_frame_to_camera_senders(&mut self.senders, msg.0, frame_num, "Camera") { + self.ffmpeg_sender_count + .store(self.senders.len(), std::sync::atomic::Ordering::Release); + } } } From d9856b75bdc6db43d3e238ec5a49d3ab0f00ca39 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 05/24] feat(frame-ws): track websocket frame subscriber count --- apps/desktop/src-tauri/src/frame_ws.rs | 27 +++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src-tauri/src/frame_ws.rs b/apps/desktop/src-tauri/src/frame_ws.rs index 753b1e46300..0e6d7a83478 100644 --- a/apps/desktop/src-tauri/src/frame_ws.rs +++ b/apps/desktop/src-tauri/src/frame_ws.rs @@ -1,4 +1,5 @@ -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::time::Instant; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{broadcast, watch}; @@ -104,8 +105,17 @@ fn record_ws_frame_stats( MAX_CREATED_TO_SENT_NS.fetch_max(created_to_sent_ns, Ordering::Relaxed); } +struct SubscriberCountGuard(Arc); + +impl Drop for SubscriberCountGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + pub async fn create_watch_frame_ws( frame_rx: watch::Receiver>>, + subscribers: Arc, ) -> (u16, CancellationToken) { use axum::{ extract::{ @@ -116,23 +126,30 @@ pub async fn create_watch_frame_ws( routing::get, }; - type RouterState = watch::Receiver>>; + type RouterState = ( + watch::Receiver>>, + Arc, + ); #[axum::debug_handler] async fn ws_handler( ws: WebSocketUpgrade, - State(state): State, + State((state, subscribers)): State, ) -> impl IntoResponse { - ws.on_upgrade(move |socket| handle_socket(socket, state)) + ws.on_upgrade(move |socket| handle_socket(socket, state, subscribers)) } async fn handle_socket( mut socket: WebSocket, mut camera_rx: watch::Receiver>>, + subscribers: Arc, ) { tracing::info!("Socket connection established"); let now = std::time::Instant::now(); + subscribers.fetch_add(1, Ordering::AcqRel); + let _subscriber_guard = SubscriberCountGuard(subscribers); + { let packed = { let borrowed = camera_rx.borrow(); @@ -237,7 +254,7 @@ pub async fn create_watch_frame_ws( let router = axum::Router::new() .route("/", get(ws_handler)) - .with_state(frame_rx); + .with_state((frame_rx, subscribers)); let cancel_token = CancellationToken::new(); let cancel_token_child = cancel_token.child_token(); From 0c4309272c83d74bdf89d57326954c379e7624c0 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 06/24] refactor(frame-ws): pass subscriber count in editor websockets --- apps/desktop/src-tauri/src/editor_window.rs | 5 +++-- apps/desktop/src-tauri/src/screenshot_editor.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src-tauri/src/editor_window.rs b/apps/desktop/src-tauri/src/editor_window.rs index ad57c7d8cfe..63bea17cdcc 100644 --- a/apps/desktop/src-tauri/src/editor_window.rs +++ b/apps/desktop/src-tauri/src/editor_window.rs @@ -27,7 +27,7 @@ pub struct PendingEditorInstances(Arc>>) async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult { let (frame_tx, frame_rx) = watch::channel(None); - let (ws_port, ws_shutdown_token) = create_watch_frame_ws(frame_rx).await; + let (ws_port, ws_shutdown_token) = create_watch_frame_ws(frame_rx, Default::default()).await; let (inner, render_frame_event_id) = create_editor_instance_impl( &app, path, @@ -343,7 +343,8 @@ impl EditorInstances { let (frame_tx, frame_rx) = watch::channel(None); - let (ws_port, ws_shutdown_token) = create_watch_frame_ws(frame_rx).await; + let (ws_port, ws_shutdown_token) = + create_watch_frame_ws(frame_rx, Default::default()).await; let app_handle = window.app_handle().clone(); let (inner, render_frame_event_id) = create_editor_instance_impl( window.app_handle(), diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 4118f50a34e..0d71fde7b2d 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -107,7 +107,8 @@ impl ScreenshotEditorInstances { path: PathBuf, ) -> Result, String> { let (frame_tx, frame_rx) = watch::channel(None); - let (ws_port, ws_shutdown_token) = create_watch_frame_ws(frame_rx).await; + let (ws_port, ws_shutdown_token) = + create_watch_frame_ws(frame_rx, Default::default()).await; if ws_port == 0 { return Err("Failed to start screenshot editor frame websocket".to_string()); } From fe0fa4a4d49c97c62ec7f1467f6c1af3e58b4a87 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 07/24] feat(camera-native): add GPU-native IOSurface frame converter --- apps/desktop/src-tauri/src/camera_native.rs | 355 ++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 apps/desktop/src-tauri/src/camera_native.rs diff --git a/apps/desktop/src-tauri/src/camera_native.rs b/apps/desktop/src-tauri/src/camera_native.rs new file mode 100644 index 00000000000..44a1a7820c7 --- /dev/null +++ b/apps/desktop/src-tauri/src/camera_native.rs @@ -0,0 +1,355 @@ +use cap_recording::NativeCameraFrame; +use cap_rendering::iosurface_texture::{ + IOSurfaceTextureCache, IOSurfaceTextureError, import_metal_texture_to_wgpu, +}; + +// Converts IOSurface-backed camera frames into an RGBA destination texture +// entirely on the GPU: the CVPixelBuffer planes are imported as Metal textures +// (zero CPU copies) and a fullscreen pass does YUV->RGB plus downscaling in +// one step. BT.601 matches what the swscale CPU path produced for untagged +// webcam streams. +const CONVERT_SHADER: &str = r#" +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput { + var positions = array, 3>( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0), + ); + var uvs = array, 3>( + vec2(0.0, 1.0), + vec2(2.0, 1.0), + vec2(0.0, -1.0), + ); + var out: VertexOutput; + out.position = vec4(positions[vi], 0.0, 1.0); + out.uv = uvs[vi]; + return out; +} + +@group(0) @binding(0) var frame_sampler: sampler; +@group(0) @binding(1) var plane0_texture: texture_2d; +@group(0) @binding(2) var uv_texture: texture_2d; + +fn yuv_video_range(y: f32, u: f32, v: f32) -> vec3 { + let yy = y - 0.0625; + let uu = u - 0.5; + let vv = v - 0.5; + return clamp( + vec3( + 1.164 * yy + 1.596 * vv, + 1.164 * yy - 0.392 * uu - 0.813 * vv, + 1.164 * yy + 2.017 * uu, + ), + vec3(0.0), + vec3(1.0), + ); +} + +fn yuv_full_range(y: f32, u: f32, v: f32) -> vec3 { + let uu = u - 0.5; + let vv = v - 0.5; + return clamp( + vec3( + y + 1.402 * vv, + y - 0.344 * uu - 0.714 * vv, + y + 1.772 * uu, + ), + vec3(0.0), + vec3(1.0), + ); +} + +@fragment +fn fs_nv12_video(in: VertexOutput) -> @location(0) vec4 { + let y = textureSample(plane0_texture, frame_sampler, in.uv).r; + let uv = textureSample(uv_texture, frame_sampler, in.uv).rg; + return vec4(yuv_video_range(y, uv.r, uv.g), 1.0); +} + +@fragment +fn fs_nv12_full(in: VertexOutput) -> @location(0) vec4 { + let y = textureSample(plane0_texture, frame_sampler, in.uv).r; + let uv = textureSample(uv_texture, frame_sampler, in.uv).rg; + return vec4(yuv_full_range(y, uv.r, uv.g), 1.0); +} + +@fragment +fn fs_bgra(in: VertexOutput) -> @location(0) vec4 { + let color = textureSample(plane0_texture, frame_sampler, in.uv); + return vec4(color.rgb, 1.0); +} +"#; + +#[derive(Debug)] +pub enum NativeFrameError { + NoImageBuffer, + NoFormatDesc, + UnsupportedFormat(String), + Surface(IOSurfaceTextureError), +} + +impl std::fmt::Display for NativeFrameError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoImageBuffer => write!(f, "sample buffer has no image buffer"), + Self::NoFormatDesc => write!(f, "sample buffer has no format description"), + Self::UnsupportedFormat(fourcc) => { + write!(f, "unsupported pixel format for GPU path: {fourcc}") + } + Self::Surface(err) => write!(f, "{err}"), + } + } +} + +impl From for NativeFrameError { + fn from(err: IOSurfaceTextureError) -> Self { + Self::Surface(err) + } +} + +pub enum NativeFrameKind { + Nv12 { full_range: bool }, + Bgra, +} + +pub fn classify_frame(frame: &NativeCameraFrame) -> Result { + let format_desc = frame + .sample_buf + .format_desc() + .ok_or(NativeFrameError::NoFormatDesc)?; + match cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()) { + "420v" => Ok(NativeFrameKind::Nv12 { full_range: false }), + "420f" => Ok(NativeFrameKind::Nv12 { full_range: true }), + "BGRA" => Ok(NativeFrameKind::Bgra), + other => Err(NativeFrameError::UnsupportedFormat(other.to_string())), + } +} + +pub struct NativeFrameConverter { + cache: IOSurfaceTextureCache, + sampler: wgpu::Sampler, + nv12_layout: wgpu::BindGroupLayout, + single_layout: wgpu::BindGroupLayout, + nv12_video_pipeline: wgpu::RenderPipeline, + nv12_full_pipeline: wgpu::RenderPipeline, + bgra_pipeline: wgpu::RenderPipeline, +} + +impl NativeFrameConverter { + pub fn new(device: &wgpu::Device) -> Option { + let cache = IOSurfaceTextureCache::new()?; + + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Camera Native Convert Shader"), + source: wgpu::ShaderSource::Wgsl(std::borrow::Cow::Borrowed(CONVERT_SHADER)), + }); + + let sampler_entry = wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }; + let texture_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + + let nv12_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Camera NV12 Layout"), + entries: &[sampler_entry, texture_entry(1), texture_entry(2)], + }); + let single_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Camera BGRA Layout"), + entries: &[sampler_entry, texture_entry(1)], + }); + + let make_pipeline = |layout: &wgpu::BindGroupLayout, entry_point: &str| { + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: None, + bind_group_layouts: &[layout], + push_constant_ranges: &[], + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Camera Native Convert Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some(entry_point), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: Default::default(), + multiview: None, + cache: None, + }) + }; + + let nv12_video_pipeline = make_pipeline(&nv12_layout, "fs_nv12_video"); + let nv12_full_pipeline = make_pipeline(&nv12_layout, "fs_nv12_full"); + let bgra_pipeline = make_pipeline(&single_layout, "fs_bgra"); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Some(Self { + cache, + sampler, + nv12_layout, + single_layout, + nv12_video_pipeline, + nv12_full_pipeline, + bgra_pipeline, + }) + } + + pub fn render_frame( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + frame: &NativeCameraFrame, + kind: &NativeFrameKind, + dst_view: &wgpu::TextureView, + ) -> Result<(), NativeFrameError> { + let image_buf = frame + .sample_buf + .image_buf() + .ok_or(NativeFrameError::NoImageBuffer)?; + let io_surface = image_buf.io_surf().ok_or(NativeFrameError::Surface( + IOSurfaceTextureError::NoIOSurface, + ))?; + let width = image_buf.width() as u32; + let height = image_buf.height() as u32; + + let (pipeline, bind_group) = match kind { + NativeFrameKind::Nv12 { full_range } => { + let y_metal = self.cache.create_y_texture(io_surface, width, height)?; + let uv_metal = self.cache.create_uv_texture(io_surface, width, height)?; + let y_texture = import_metal_texture_to_wgpu( + device, + &y_metal, + wgpu::TextureFormat::R8Unorm, + width, + height, + Some("Camera Y Plane"), + )?; + let uv_texture = import_metal_texture_to_wgpu( + device, + &uv_metal, + wgpu::TextureFormat::Rg8Unorm, + width / 2, + height / 2, + Some("Camera UV Plane"), + )?; + let y_view = y_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let uv_view = uv_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Camera NV12 Bind Group"), + layout: &self.nv12_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&y_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&uv_view), + }, + ], + }); + let pipeline = if *full_range { + &self.nv12_full_pipeline + } else { + &self.nv12_video_pipeline + }; + (pipeline, bind_group) + } + NativeFrameKind::Bgra => { + let bgra_metal = self.cache.create_bgra_texture(io_surface, width, height)?; + let bgra_texture = import_metal_texture_to_wgpu( + device, + &bgra_metal, + wgpu::TextureFormat::Bgra8Unorm, + width, + height, + Some("Camera BGRA"), + )?; + let view = bgra_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Camera BGRA Bind Group"), + layout: &self.single_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&view), + }, + ], + }); + (&self.bgra_pipeline, bind_group) + } + }; + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Camera Native Convert"), + }); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Camera Native Convert Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: dst_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } + queue.submit(std::iter::once(encoder.finish())); + + Ok(()) + } +} From d7bc4bb581a2b6a6ca0b8c1d4bffd5af414e6944 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:37 +0100 Subject: [PATCH 08/24] build(desktop): add cap-camera-ffmpeg macOS dependency --- apps/desktop/src-tauri/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 5296356df6d..4c58eb00b6e 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -145,6 +145,7 @@ objc = "0.2.7" swift-rs = "1.0.6" tauri-nspanel = { git = "https://github.com/ahkohd/tauri-nspanel", branch = "v2" } cidre = { workspace = true } +cap-camera-ffmpeg = { path = "../../../crates/camera-ffmpeg" } [target.'cfg(not(all(target_os = "macos", target_arch = "x86_64")))'.dependencies] parakeet-rs = "0.3.4" From 7205dcd1a716448cb59475a6fe59262c5a4dc199 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:38 +0100 Subject: [PATCH 09/24] build(desktop): enable Win32 threading for camera preview QoS --- apps/desktop/src-tauri/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 4c58eb00b6e..cffef9dd615 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -161,6 +161,7 @@ windows = { workspace = true, features = [ "Win32_Foundation", "Win32_System", "Win32_System_Power", + "Win32_System_Threading", "Win32_System_WinRT", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", From 4d4129fd9a40b48c42ed06a81e12fc241bdb34fe Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:46 +0100 Subject: [PATCH 10/24] feat(general-settings): roll out native camera preview default on macOS --- .../desktop/src-tauri/src/general_settings.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 2cf259e2c8e..b07cdf50e69 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -221,7 +221,7 @@ pub struct GeneralSettingsStore { } fn default_enable_native_camera_preview() -> bool { - cfg!(all(debug_assertions, target_os = "macos")) + cfg!(target_os = "macos") } fn no(_: &bool) -> bool { @@ -328,6 +328,20 @@ pub enum AppTheme { } impl GeneralSettingsStore { + // The effective value: the native preview is macOS-only; it is not + // reliable on Windows, so the stored setting is ignored there and the + // websocket preview is always used. + pub fn native_camera_preview_enabled(app: &AppHandle) -> bool { + if cfg!(not(target_os = "macos")) { + return false; + } + Self::get(app) + .ok() + .flatten() + .map(|settings| settings.enable_native_camera_preview) + .unwrap_or_else(default_enable_native_camera_preview) + } + pub fn get(app: &AppHandle) -> Result, String> { match app.store("store").map(|s| s.get("general_settings")) { Ok(Some(store)) => { @@ -406,6 +420,22 @@ pub fn init(app: &AppHandle) { crate::posthog::set_telemetry_enabled(store.enable_telemetry); register_bundled_muxer_binary(app); + // One-time rollout of the native (GPU-surface) camera preview as the macOS + // default. The setting is always serialized, so existing users carry an + // explicit `false` from the old opt-in default; a raw marker key (outside + // the typed struct, so re-serialization never drops it) makes sure a user + // who explicitly turns it back off afterwards stays off. + #[cfg(target_os = "macos")] + { + const NATIVE_PREVIEW_MIGRATION_KEY: &str = "native_camera_preview_default_v1"; + if let Ok(raw_store) = app.store("store") + && raw_store.get(NATIVE_PREVIEW_MIGRATION_KEY).is_none() + { + store.enable_native_camera_preview = true; + raw_store.set(NATIVE_PREVIEW_MIGRATION_KEY, json!(true)); + } + } + if let Err(e) = store.save(app) { error!("Failed to save general settings: {}", e); } From b2d83fc1a4810dee1e4cb8deb8b775e6b396e989 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:46 +0100 Subject: [PATCH 11/24] feat(camera): add native preview renderer with IOSurface GPU path --- apps/desktop/src-tauri/src/camera.rs | 697 +++++++++++++++++++++------ 1 file changed, 549 insertions(+), 148 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 213accdd99a..ed3a441e0af 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -1,10 +1,14 @@ use anyhow::{Context, anyhow}; -use cap_recording::{ - FFmpegVideoFrame, - feeds::{self, camera::CameraFeed}, -}; +#[cfg(not(target_os = "macos"))] +use cap_recording::FFmpegVideoFrame; +#[cfg(target_os = "macos")] +use cap_recording::NativeCameraFrame; +use cap_recording::feeds::{self, camera::CameraFeed}; #[cfg(target_os = "macos")] use cap_utils::macos_qos::{MacOsQosClass, set_current_thread_qos}; + +#[cfg(target_os = "macos")] +use crate::camera_native::{NativeFrameConverter, classify_frame}; use ffmpeg::{ format::{self, Pixel}, frame, @@ -14,6 +18,7 @@ use kameo::actor::ActorRef; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ + mem::ManuallyDrop, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -32,9 +37,17 @@ use wgpu::{CompositeAlphaMode, SurfaceTexture}; static TOOLBAR_HEIGHT: f32 = 56.0; -static GPU_SURFACE_SCALE: u32 = 2; -const CAMERA_PREVIEW_TARGET_FRAME_INTERVAL: Duration = Duration::from_micros(33_333); -const CAMERA_PREVIEW_FRAME_INTERVAL_SLACK: Duration = Duration::from_millis(3); +const DEFAULT_SURFACE_SCALE: f64 = 2.0; +// Quantizing the target texture width stops drag-resizing from rebuilding the +// scaler, preview texture, and bind group on every 1px size change. +const CAMERA_PREVIEW_TEXTURE_WIDTH_BUCKET: u32 = 64; +const CAMERA_PREVIEW_MIN_TEXTURE_WIDTH: u32 = 320; +const CAMERA_PREVIEW_MAX_TEXTURE_WIDTH: u32 = 960; +const CAMERA_PREVIEW_MAX_TEXTURE_HEIGHT: u32 = 540; +const CAMERA_PREVIEW_BLUR_MAX_TEXTURE_WIDTH: u32 = 640; +const CAMERA_PREVIEW_BLUR_MAX_TEXTURE_HEIGHT: u32 = 360; +const CAMERA_PREVIEW_TARGET_FRAME_INTERVAL: Duration = Duration::from_micros(16_666); +const CAMERA_PREVIEW_FRAME_INTERVAL_SLACK: Duration = Duration::from_millis(1); const CAMERA_PREVIEW_BLUR_INFERENCE_INTERVAL: Duration = Duration::from_millis(150); pub const MIN_CAMERA_SIZE: f32 = 150.0; @@ -42,6 +55,65 @@ pub const MAX_CAMERA_SIZE: f32 = 600.0; pub const DEFAULT_CAMERA_SIZE: f32 = 230.0; pub const WIDE_CAMERA_ASPECT_RATIO: f32 = 16.0 / 9.0; +// On macOS the preview consumes retained CMSampleBuffers and renders them via +// a zero-copy IOSurface->Metal->wgpu import; everywhere else it consumes the +// CPU-converted ffmpeg frames. +#[cfg(target_os = "macos")] +type PreviewCameraFrame = NativeCameraFrame; +#[cfg(not(target_os = "macos"))] +type PreviewCameraFrame = FFmpegVideoFrame; + +#[derive(Clone)] +pub enum CameraPreviewSender { + #[cfg(not(target_os = "macos"))] + Ffmpeg(flume::Sender), + #[cfg(target_os = "macos")] + Native(flume::Sender), +} + +impl CameraPreviewSender { + fn from_tx(tx: flume::Sender) -> Self { + #[cfg(target_os = "macos")] + { + Self::Native(tx) + } + #[cfg(not(target_os = "macos"))] + { + Self::Ffmpeg(tx) + } + } + + pub async fn attach(&self, feed: &ActorRef) -> Result<(), String> { + match self { + #[cfg(not(target_os = "macos"))] + Self::Ffmpeg(tx) => feed + .ask(feeds::camera::AddSender(tx.clone())) + .await + .map_err(|err| err.to_string()), + #[cfg(target_os = "macos")] + Self::Native(tx) => feed + .ask(feeds::camera::AddNativeSender(tx.clone())) + .await + .map_err(|err| err.to_string()), + } + } + + pub async fn detach(&self, feed: &ActorRef) -> Result<(), String> { + match self { + #[cfg(not(target_os = "macos"))] + Self::Ffmpeg(tx) => feed + .ask(feeds::camera::RemoveSender(tx.clone())) + .await + .map_err(|err| err.to_string()), + #[cfg(target_os = "macos")] + Self::Native(tx) => feed + .ask(feeds::camera::RemoveNativeSender(tx.clone())) + .await + .map_err(|err| err.to_string()), + } + } +} + #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize, Type)] #[serde(rename_all = "lowercase")] pub enum CameraPreviewShape { @@ -96,8 +168,41 @@ fn camera_preview_frame_due(last_render_at: Option, now: Instant) -> bo }) } +fn camera_preview_texture_dimensions( + source_width: u32, + source_height: u32, + window_px: u32, + blur_enabled: bool, +) -> (u32, u32) { + let source_width = source_width.max(1); + let source_height = source_height.max(1); + let (max_width, max_height) = if blur_enabled { + ( + CAMERA_PREVIEW_BLUR_MAX_TEXTURE_WIDTH, + CAMERA_PREVIEW_BLUR_MAX_TEXTURE_HEIGHT, + ) + } else { + ( + CAMERA_PREVIEW_MAX_TEXTURE_WIDTH, + CAMERA_PREVIEW_MAX_TEXTURE_HEIGHT, + ) + }; + let requested_width = window_px + .max(CAMERA_PREVIEW_MIN_TEXTURE_WIDTH) + .div_ceil(CAMERA_PREVIEW_TEXTURE_WIDTH_BUCKET) + .saturating_mul(CAMERA_PREVIEW_TEXTURE_WIDTH_BUCKET) + .min(max_width); + let scale = (requested_width as f64 / source_width as f64) + .min(max_height as f64 / source_height as f64) + .min(1.0); + let target_width = ((source_width as f64 * scale).round() as u32).max(1); + let target_height = ((source_height as f64 * scale).round() as u32).max(1); + (target_width, target_height) +} + pub struct CameraPreviewManager { store: Result>, String>, + store_save_generation: Arc, preview: Option, preview_session_id: Arc, wgpu_instance: wgpu::Instance, @@ -109,6 +214,7 @@ impl CameraPreviewManager { store: tauri_plugin_store::StoreBuilder::new(app, "cameraPreview") .build() .map_err(|err| format!("Error initializing camera preview store: {err}")), + store_save_generation: Arc::new(AtomicU64::new(0)), preview: None, preview_session_id: Arc::new(AtomicU64::new(0)), wgpu_instance: wgpu::Instance::default(), @@ -137,7 +243,20 @@ impl CameraPreviewManager { let store = self.store.as_ref().map_err(|err| anyhow!("{err}"))?; store.set("state", serde_json::to_value(&state)?); - store.save()?; + + // set_state fires per mousemove during a size drag; debounce the disk + // write so only the post-drag state is persisted. + let generation = self.store_save_generation.fetch_add(1, Ordering::AcqRel) + 1; + let save_generation = self.store_save_generation.clone(); + let store = store.clone(); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + if save_generation.load(Ordering::Acquire) == generation + && let Err(err) = store.save() + { + error!("Error saving camera preview store: {err}"); + } + }); if let Some(preview) = &self.preview { preview @@ -160,8 +279,10 @@ impl CameraPreviewManager { .is_some_and(|preview| preview.is_paused) } - pub fn sender(&self) -> Option> { - self.preview.as_ref().map(|p| p.camera_tx.clone()) + pub fn sender(&self) -> Option { + self.preview + .as_ref() + .map(|p| CameraPreviewSender::from_tx(p.camera_tx.clone())) } pub fn notify_window_resized(&self, width: u32, height: u32) { @@ -210,10 +331,10 @@ impl CameraPreviewManager { actor: ActorRef, ) -> anyhow::Result<()> { if let Some(preview) = &mut self.preview { - actor - .ask(feeds::camera::AddSender(preview.camera_tx.clone())) + CameraPreviewSender::from_tx(preview.camera_tx.clone()) + .attach(&actor) .await - .context("Error re-attaching camera feed consumer")?; + .map_err(|err| anyhow!("Error re-attaching camera feed consumer: {err}"))?; if preview.is_paused { preview.is_paused = false; @@ -238,19 +359,22 @@ impl CameraPreviewManager { let session_id = self.preview_session_id.fetch_add(1, Ordering::AcqRel) + 1; - let (camera_tx, camera_rx) = flume::bounded(4); + let (camera_tx, camera_rx) = flume::bounded::(1); - actor - .ask(feeds::camera::AddSender(camera_tx.clone())) + CameraPreviewSender::from_tx(camera_tx.clone()) + .attach(&actor) .await - .context("Error attaching camera feed consumer")?; + .map_err(|err| anyhow!("Error attaching camera feed consumer: {err}"))?; let default_state = self .get_state() .map_err(|err| error!("Error getting camera preview state: {err}")) .unwrap_or_default(); - let (reconfigure, reconfigure_rx) = broadcast::channel(1); + // Capacity must absorb bursts of control events (State spam during a + // size drag interleaved with Pause/Resume); a lagged capacity-1 channel + // could drop a Resume and leave the preview stuck hidden. + let (reconfigure, reconfigure_rx) = broadcast::channel(8); let mut renderer = InitializedCameraPreview::init_wgpu( window.clone(), &default_state, @@ -291,9 +415,10 @@ impl CameraPreviewManager { ); let (drop_tx, drop_rx) = oneshot::channel(); + let renderer = ManuallyDrop::new(renderer); window .run_on_main_thread(move || { - drop(renderer); + drop(ManuallyDrop::into_inner(renderer)); let _ = drop_tx.send(()); }) .ok(); @@ -337,7 +462,7 @@ struct InitializedCameraPreview { reconfigure: broadcast::Sender, session_id: u64, shutdown_complete: oneshot::Receiver<()>, - camera_tx: flume::Sender, + camera_tx: flume::Sender, is_paused: bool, } @@ -353,9 +478,10 @@ impl InitializedCameraPreview { 1.0 }; - let size = resize_window(&window, default_state, aspect, false) - .await - .context("Error resizing Tauri window")?; + let (window_width, window_height, surface_scale) = + resize_window(&window, default_state, aspect, false) + .await + .context("Error resizing Tauri window")?; let (tx, rx) = oneshot::channel(); window @@ -376,7 +502,7 @@ impl InitializedCameraPreview { let adapter = instance .request_adapter(&wgpu::RequestAdapterOptions { - power_preference: wgpu::PowerPreference::default(), + power_preference: wgpu::PowerPreference::LowPower, force_fallback_adapter: false, compatible_surface: Some(&surface), }) @@ -549,12 +675,12 @@ impl InitializedCameraPreview { let surface_config = wgpu::SurfaceConfiguration { usage: wgpu::TextureUsages::RENDER_ATTACHMENT, format: swapchain_format, - width: size.0, - height: size.1, + width: window_width, + height: window_height, present_mode, alpha_mode, view_formats: vec![], - desired_maximum_frame_latency: 2, + desired_maximum_frame_latency: 1, }; surface.configure(&device, &surface_config); @@ -572,6 +698,7 @@ impl InitializedCameraPreview { let mut renderer = Renderer { surface: Some(surface), surface_config, + surface_scale, render_pipeline, device, queue, @@ -586,13 +713,17 @@ impl InitializedCameraPreview { blur_processor: None, blur_processor_init_attempted: false, blur_source_texture: None, + #[cfg(target_os = "macos")] + native_converter: None, + #[cfg(target_os = "macos")] + native_converter_init_attempted: false, }; renderer.update_state_uniforms(default_state); renderer .sync_ratio_uniform_and_resize_window_to_it(&window, default_state, aspect) .await; - renderer.reconfigure_gpu_surface(size.0, size.1); + renderer.reconfigure_gpu_surface(window_width, window_height); let initial_surface = renderer.acquire_surface_texture(); if let Some(surface) = initial_surface { @@ -623,6 +754,7 @@ impl InitializedCameraPreview { struct Renderer { surface: Option>, surface_config: wgpu::SurfaceConfiguration, + surface_scale: f64, render_pipeline: wgpu::RenderPipeline, device: wgpu::Device, queue: wgpu::Queue, @@ -637,6 +769,10 @@ struct Renderer { blur_processor: Option, blur_processor_init_attempted: bool, blur_source_texture: Option, + #[cfg(target_os = "macos")] + native_converter: Option, + #[cfg(target_os = "macos")] + native_converter_init_attempted: bool, } impl Renderer { @@ -645,7 +781,7 @@ impl Renderer { window: WebviewWindow, default_state: CameraPreviewState, mut reconfigure: broadcast::Receiver, - camera_rx: flume::Receiver, + camera_rx: flume::Receiver, ) { let mut resampler_frame = Cached::default(); let Ok(mut scaler) = scaling::Context::get( @@ -663,6 +799,16 @@ impl Renderer { let mut source_dimensions = Cached::default(); let mut last_render_at = None; + // The camera's pixel buffer pool only recycles an IOSurface once its + // CVPixelBuffer is released; holding the last couple of sample buffers + // keeps the camera from writing into a surface the GPU is still + // sampling. + #[cfg(target_os = "macos")] + let mut inflight_frames: std::collections::VecDeque = + std::collections::VecDeque::new(); + #[cfg(target_os = "macos")] + let mut native_fallback_logged = false; + let start_time = Instant::now(); let startup_timeout = Duration::from_secs(5); let mut received_first_frame = false; @@ -689,6 +835,8 @@ impl Renderer { Ok(ReconfigureEvent::Resume) => { is_paused = false; while camera_rx.try_recv().is_ok() {} + #[cfg(target_os = "macos")] + inflight_frames.clear(); if let Some(texture) = self.acquire_surface_texture() { let (buffer, stride) = render_solid_frame([0x11, 0x11, 0x11, 0xFF], 5, 5); @@ -719,9 +867,13 @@ impl Renderer { self.reconfigure_gpu_surface(width, height); } Ok(ReconfigureEvent::Pause) => {} - Err(_) => { + Err(broadcast::error::RecvError::Lagged(_)) => { continue; } + Err(broadcast::error::RecvError::Closed) => { + self.cleanup_for_shutdown(&window).await; + return; + } } } } @@ -729,6 +881,9 @@ impl Renderer { if needs_full_reconfigure { needs_full_reconfigure = false; self.update_state_uniforms(&state); + if state.background_blur == cap_project::BackgroundBlurMode::Off { + self.release_blur_resources(); + } source_dimensions = Cached::default(); last_render_at = None; let aspect_ratio = self.aspect_ratio.get_latest_key().copied().unwrap_or( @@ -739,12 +894,14 @@ impl Renderer { }, ); self.aspect_ratio = Cached::default(); - if let Ok((width, height)) = resize_window(&window, &state, aspect_ratio, false) - .await - .map_err(|err| { - error!("Error resizing camera preview window after resume: {err}") - }) + if let Ok((width, height, scale)) = + resize_window(&window, &state, aspect_ratio, false) + .await + .map_err(|err| { + error!("Error resizing camera preview window after resume: {err}") + }) { + self.surface_scale = scale; self.reconfigure_gpu_surface(width, height); } } @@ -776,10 +933,12 @@ impl Renderer { } }, result = reconfigure.recv() => { - if let Ok(result) = result { - break Err(result) - } else { - continue; + match result { + Ok(result) => break Err(result), + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => { + break Err(ReconfigureEvent::Shutdown); + } } }, _ = tokio::time::sleep(frame_timeout) => { @@ -806,8 +965,16 @@ impl Renderer { } last_render_at = Some(now); - let source_width = frame.inner.width(); - let source_height = frame.inner.height(); + #[cfg(target_os = "macos")] + let (source_width, source_height) = { + let Some(image_buf) = frame.sample_buf.image_buf() else { + continue 'main_loop; + }; + (image_buf.width() as u32, image_buf.height() as u32) + }; + #[cfg(not(target_os = "macos"))] + let (source_width, source_height) = (frame.inner.width(), frame.inner.height()); + let aspect_ratio = source_width as f32 / source_height as f32; if source_dimensions.update_key_and_should_init((source_width, source_height)) { self.sync_ratio_uniform_and_resize_window_to_it( @@ -820,100 +987,78 @@ impl Renderer { let surface_result = self.acquire_surface_texture(); if let Some(surface) = surface_result { - let window_px = (clamp_size(state.size) as u32) * GPU_SURFACE_SCALE; - let output_width = window_px.max(320).min(source_width); - let output_height = (output_width as f32 / aspect_ratio) as u32; - - let already_preview_rgba = frame.inner.format() == Pixel::RGBA - && output_width == source_width - && output_height == source_height; - let (frame_data, frame_stride) = if already_preview_rgba { - (frame.inner.data(0), frame.inner.stride(0) as u32) - } else { - let resampler_frame = resampler_frame - .get_or_init((output_width, output_height), frame::Video::empty); - - scaler.cached( - frame.inner.format(), - source_width, - source_height, - format::Pixel::RGBA, + let window_px = (clamp_size(state.size) as f64 + * self.surface_scale.max(1.0)) + .round() as u32; + let blur_mode = blur_mode_from_project(state.background_blur); + let (output_width, output_height) = camera_preview_texture_dimensions( + source_width, + source_height, + window_px, + blur_mode.is_some(), + ); + + #[cfg(target_os = "macos")] + { + match self.render_native_frame( + &frame, output_width, output_height, - ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR, - ); - - if let Err(err) = scaler.run(&frame.inner, resampler_frame) { - error!("Error rescaling frame with ffmpeg: {err:?}"); - continue 'main_loop; + blur_mode, + &surface, + ) { + Ok(()) => { + surface.present(); + inflight_frames.push_back(frame); + while inflight_frames.len() > 2 { + inflight_frames.pop_front(); + } + } + Err(err) => { + if !native_fallback_logged { + native_fallback_logged = true; + warn!( + "Camera GPU-native preview unavailable ({err}); using CPU conversion" + ); + } + match cap_camera_ffmpeg::sample_buf_as_ffmpeg(&frame.sample_buf) + { + Ok(inner) => { + if self.render_cpu_frame( + &inner, + output_width, + output_height, + blur_mode, + &surface, + &mut scaler, + &mut resampler_frame, + ) { + surface.present(); + } + } + Err(err) => { + error!( + "Camera preview CPU fallback conversion failed: {err}" + ); + } + } + } } - - (resampler_frame.data(0), resampler_frame.stride(0) as u32) - }; - - let blur_mode = blur_mode_from_project(state.background_blur); - let blurred = if let Some(mode) = blur_mode { - self.run_background_blur( - frame_data, - frame_stride, + } + #[cfg(not(target_os = "macos"))] + { + if self.render_cpu_frame( + &frame.inner, output_width, output_height, - mode, - ) - } else { - false - }; - - let prepared = - self.texture.get_or_init((output_width, output_height), || { - PreparedTexture::init( - self.device.clone(), - self.queue.clone(), - &self.sampler, - &self.bind_group_layout, - self.uniform_bind_group.clone(), - self.render_pipeline.clone(), - output_width, - output_height, - ) - }); - - if blurred { - let blur_output = self - .blur_processor - .as_mut() - .and_then(|p| p.process_returning_output()) - .expect("blurred flag guarantees output"); - let mut encoder = self.device.create_command_encoder( - &wgpu::CommandEncoderDescriptor { - label: Some("Blur Copy"), - }, - ); - encoder.copy_texture_to_texture( - wgpu::TexelCopyTextureInfo { - texture: blur_output, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyTextureInfo { - texture: &prepared.texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::Extent3d { - width: output_width, - height: output_height, - depth_or_array_layers: 1, - }, - ); - self.queue.submit(std::iter::once(encoder.finish())); - prepared.render_no_upload(&surface); - } else { - prepared.render(&surface, frame_data, frame_stride); + blur_mode, + &surface, + &mut scaler, + &mut resampler_frame, + ) { + surface.present(); + } } - surface.present(); } } Err(ReconfigureEvent::State(new_state)) => { @@ -932,10 +1077,15 @@ impl Renderer { self.sync_ratio_uniform_and_resize_window_to_it(&window, &state, aspect_ratio) .await; self.update_state_uniforms(&state); - if let Ok((width, height)) = resize_window(&window, &state, aspect_ratio, false) - .await - .map_err(|err| error!("Error resizing camera preview window: {err}")) + if state.background_blur == cap_project::BackgroundBlurMode::Off { + self.release_blur_resources(); + } + if let Ok((width, height, scale)) = + resize_window(&window, &state, aspect_ratio, false) + .await + .map_err(|err| error!("Error resizing camera preview window: {err}")) { + self.surface_scale = scale; self.reconfigure_gpu_surface(width, height); } } @@ -952,6 +1102,8 @@ impl Renderer { // instead of panel.order_front_regardless() - see the comment there // for why this is critical to avoid macOS crashes. is_paused = true; + #[cfg(target_os = "macos")] + inflight_frames.clear(); window .run_on_main_thread({ let window = window.clone(); @@ -963,6 +1115,8 @@ impl Renderer { } Err(ReconfigureEvent::Resume) => { while camera_rx.try_recv().is_ok() {} + #[cfg(target_os = "macos")] + inflight_frames.clear(); if let Some(texture) = self.acquire_surface_texture() { let (buffer, stride) = render_solid_frame([0x11, 0x11, 0x11, 0xFF], 5, 5); PreparedTexture::init( @@ -987,6 +1141,224 @@ impl Renderer { } } + #[allow(clippy::too_many_arguments)] + fn render_cpu_frame( + &mut self, + inner: &frame::Video, + output_width: u32, + output_height: u32, + blur_mode: Option, + surface: &SurfaceTexture, + scaler: &mut scaling::Context, + resampler_frame: &mut Cached<(u32, u32), frame::Video>, + ) -> bool { + let source_width = inner.width(); + let source_height = inner.height(); + + let already_preview_rgba = inner.format() == Pixel::RGBA + && output_width == source_width + && output_height == source_height; + let (frame_data, frame_stride) = if already_preview_rgba { + (inner.data(0), inner.stride(0) as u32) + } else { + let resampler_frame = + resampler_frame.get_or_init((output_width, output_height), frame::Video::empty); + + scaler.cached( + inner.format(), + source_width, + source_height, + format::Pixel::RGBA, + output_width, + output_height, + ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR, + ); + + if let Err(err) = scaler.run(inner, resampler_frame) { + error!("Error rescaling frame with ffmpeg: {err:?}"); + return false; + } + + (resampler_frame.data(0), resampler_frame.stride(0) as u32) + }; + + let blurred = if let Some(mode) = blur_mode { + self.run_background_blur(frame_data, frame_stride, output_width, output_height, mode) + } else { + false + }; + + let prepared = self.texture.get_or_init((output_width, output_height), || { + PreparedTexture::init( + self.device.clone(), + self.queue.clone(), + &self.sampler, + &self.bind_group_layout, + self.uniform_bind_group.clone(), + self.render_pipeline.clone(), + output_width, + output_height, + ) + }); + + if blurred { + let blur_output = self + .blur_processor + .as_mut() + .and_then(|p| p.process_returning_output()) + .expect("blurred flag guarantees output"); + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Blur Copy"), + }); + encoder.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + texture: blur_output, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyTextureInfo { + texture: &prepared.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + ); + self.queue.submit(std::iter::once(encoder.finish())); + prepared.render_no_upload(surface); + } else { + prepared.render(surface, frame_data, frame_stride); + } + true + } + + #[cfg(target_os = "macos")] + fn ensure_native_converter(&mut self) -> bool { + if self.native_converter.is_some() { + return true; + } + if self.native_converter_init_attempted { + return false; + } + self.native_converter_init_attempted = true; + self.native_converter = NativeFrameConverter::new(&self.device); + if self.native_converter.is_some() { + info!("Camera GPU-native frame converter initialized"); + true + } else { + warn!("Failed to initialize camera GPU-native frame converter"); + false + } + } + + #[cfg(target_os = "macos")] + fn render_native_frame( + &mut self, + frame: &NativeCameraFrame, + output_width: u32, + output_height: u32, + blur_mode: Option, + surface: &SurfaceTexture, + ) -> Result<(), String> { + let kind = classify_frame(frame).map_err(|err| err.to_string())?; + if !self.ensure_native_converter() { + return Err("GPU-native converter unavailable".to_string()); + } + + let use_blur = blur_mode.is_some() && self.ensure_blur_processor(); + + if use_blur { + self.ensure_blur_source_texture(output_width, output_height); + let src_tex = self.blur_source_texture.as_ref().expect("just ensured"); + let dst_view = src_tex.create_view(&wgpu::TextureViewDescriptor::default()); + self.native_converter + .as_ref() + .expect("ensured above") + .render_frame(&self.device, &self.queue, frame, &kind, &dst_view) + .map_err(|err| err.to_string())?; + + if let (Some(mode), Some(processor)) = (blur_mode, self.blur_processor.as_mut()) { + processor.process(&self.device, &self.queue, src_tex, mode); + } + + let prepared = self.texture.get_or_init((output_width, output_height), || { + PreparedTexture::init( + self.device.clone(), + self.queue.clone(), + &self.sampler, + &self.bind_group_layout, + self.uniform_bind_group.clone(), + self.render_pipeline.clone(), + output_width, + output_height, + ) + }); + + let blur_output = self + .blur_processor + .as_mut() + .and_then(|p| p.process_returning_output()) + .ok_or_else(|| "blur output missing".to_string())?; + let mut encoder = self + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("Blur Copy"), + }); + encoder.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + texture: blur_output, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyTextureInfo { + texture: &prepared.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::Extent3d { + width: output_width, + height: output_height, + depth_or_array_layers: 1, + }, + ); + self.queue.submit(std::iter::once(encoder.finish())); + prepared.render_no_upload(surface); + } else { + let prepared = self.texture.get_or_init((output_width, output_height), || { + PreparedTexture::init( + self.device.clone(), + self.queue.clone(), + &self.sampler, + &self.bind_group_layout, + self.uniform_bind_group.clone(), + self.render_pipeline.clone(), + output_width, + output_height, + ) + }); + let dst_view = prepared + .texture + .create_view(&wgpu::TextureViewDescriptor::default()); + self.native_converter + .as_ref() + .expect("ensured above") + .render_frame(&self.device, &self.queue, frame, &kind, &dst_view) + .map_err(|err| err.to_string())?; + prepared.render_no_upload(surface); + } + + Ok(()) + } + fn run_background_blur( &mut self, frame_data: &[u8], @@ -1032,6 +1404,18 @@ impl Renderer { true } + // The blur processor owns an ONNX session plus several GPU textures, so it + // is dropped as soon as blur is switched off instead of living for the + // window's lifetime; turning blur back on re-runs the lazy init. + fn release_blur_resources(&mut self) { + if self.blur_processor.is_some() || self.blur_source_texture.is_some() { + self.blur_processor = None; + self.blur_source_texture = None; + info!("Released camera background blur resources"); + } + self.blur_processor_init_attempted = false; + } + fn ensure_blur_processor(&mut self) -> bool { if self.blur_processor.is_some() { return true; @@ -1077,7 +1461,8 @@ impl Renderer { format: wgpu::TextureFormat::Rgba8Unorm, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST - | wgpu::TextureUsages::COPY_SRC, + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::RENDER_ATTACHMENT, view_formats: &[], })); } @@ -1092,11 +1477,11 @@ impl Renderer { drop(std::mem::take(&mut self.texture)); self.aspect_ratio = Cached::default(); - let surface = self.surface.take(); + let surface = ManuallyDrop::new(self.surface.take()); let (drop_tx, drop_rx) = oneshot::channel(); window .run_on_main_thread(move || { - drop(surface); + drop(ManuallyDrop::into_inner(surface)); let _ = drop_tx.send(()); }) .ok(); @@ -1136,25 +1521,32 @@ impl Renderer { } fn reconfigure_gpu_surface(&mut self, window_width: u32, window_height: u32) { - self.surface_config.width = if window_width > 0 { - window_width * GPU_SURFACE_SCALE + let scale = self.surface_scale.max(1.0); + let surface_width = if window_width > 0 { + ((window_width as f64 * scale).round() as u32).max(1) } else { 1 }; - self.surface_config.height = if window_height > 0 { - window_height * GPU_SURFACE_SCALE + let surface_height = if window_height > 0 { + ((window_height as f64 * scale).round() as u32).max(1) } else { 1 }; - if let Some(surface) = &self.surface { - surface.configure(&self.device, &self.surface_config); + + if self.surface_config.width != surface_width + || self.surface_config.height != surface_height + { + self.surface_config.width = surface_width; + self.surface_config.height = surface_height; + if let Some(surface) = &self.surface { + surface.configure(&self.device, &self.surface_config); + } } let window_uniforms = WindowUniforms { window_height: window_height as f32, window_width: window_width as f32, - toolbar_percentage: (TOOLBAR_HEIGHT * GPU_SURFACE_SCALE as f32) - / self.surface_config.height as f32, + toolbar_percentage: (TOOLBAR_HEIGHT * scale as f32) / self.surface_config.height as f32, _padding: 0.0, }; self.queue.write_buffer( @@ -1203,10 +1595,11 @@ impl Renderer { bytemuck::cast_slice(&[camera_uniforms]), ); - if let Ok((width, height)) = resize_window(window, state, aspect_ratio, false) + if let Ok((width, height, scale)) = resize_window(window, state, aspect_ratio, false) .await .map_err(|err| error!("Error resizing camera preview window: {err}")) { + self.surface_scale = scale; self.reconfigure_gpu_surface(width, height); } } @@ -1218,7 +1611,7 @@ async fn resize_window( state: &CameraPreviewState, aspect: f32, should_move: bool, -) -> anyhow::Result<(u32, u32)> { +) -> anyhow::Result<(u32, u32, f64)> { trace!("CameraPreview/resize_window"); let base = clamp_size(state.size); @@ -1238,7 +1631,7 @@ async fn resize_window( .run_on_main_thread({ let window = window.clone(); move || { - let result: tauri::Result<(u32, u32)> = (|| { + let result: tauri::Result<(u32, u32, f64)> = (|| { if should_move { let (monitor_size, monitor_offset, monitor_scale_factor): ( PhysicalSize, @@ -1301,7 +1694,13 @@ async fn resize_window( window.set_size(LogicalSize::new(window_width, window_height))?; - Ok((window_width, window_height)) + let scale_factor = window + .scale_factor() + .ok() + .filter(|scale| *scale > 0.0) + .unwrap_or(DEFAULT_SURFACE_SCALE); + + Ok((window_width, window_height, scale_factor)) })(); tx.send(result).ok(); @@ -1430,7 +1829,9 @@ impl PreparedTexture { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::RENDER_ATTACHMENT, view_formats: &[], }); From 89c3de7d81abee8f6625ed7fa33ca1e9b7490600 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:46 +0100 Subject: [PATCH 12/24] perf(camera-legacy): optimize websocket camera preview pipeline --- apps/desktop/src-tauri/src/camera_legacy.rs | 304 ++++++++++++++------ 1 file changed, 213 insertions(+), 91 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera_legacy.rs b/apps/desktop/src-tauri/src/camera_legacy.rs index f860b6044ae..7b4dd4f390b 100644 --- a/apps/desktop/src-tauri/src/camera_legacy.rs +++ b/apps/desktop/src-tauri/src/camera_legacy.rs @@ -1,13 +1,16 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; use std::time::{Duration, Instant}; use cap_recording::FFmpegVideoFrame; +#[cfg(target_os = "macos")] +use cap_utils::macos_qos::{MacOsQosClass, set_current_thread_qos}; use flume::Sender; use tokio::sync::watch; use tokio_util::sync::CancellationToken; -use crate::frame_ws::{WSFrame, create_frame_ws}; +use crate::camera::{CameraPreviewState, MAX_CAMERA_SIZE, MIN_CAMERA_SIZE}; +use crate::frame_ws::{WSFrame, create_watch_frame_ws}; const WS_READBACK_PENDING: u8 = 0; const WS_READBACK_READY_OK: u8 = 1; @@ -23,10 +26,17 @@ struct WsReadback { state: WsReadbackState, } -const WS_PREVIEW_MAX_WIDTH: u32 = 640; -const WS_PREVIEW_MAX_HEIGHT: u32 = 360; -const WS_PREVIEW_TARGET_FRAME_INTERVAL: Duration = Duration::from_micros(33_333); -const WS_PREVIEW_FRAME_INTERVAL_SLACK: Duration = Duration::from_millis(3); +const WS_PREVIEW_SURFACE_SCALE: u32 = 2; +// Same drag-resize hysteresis as the native preview: only rebuild the scaler +// and reallocate buffers when the target width crosses a 64px bucket. +const WS_PREVIEW_WIDTH_BUCKET: u32 = 64; +const WS_PREVIEW_MIN_WIDTH: u32 = 320; +const WS_PREVIEW_MAX_WIDTH: u32 = 960; +const WS_PREVIEW_MAX_HEIGHT: u32 = 540; +const WS_PREVIEW_BLUR_MAX_WIDTH: u32 = 640; +const WS_PREVIEW_BLUR_MAX_HEIGHT: u32 = 360; +const WS_PREVIEW_TARGET_FRAME_INTERVAL: Duration = Duration::from_micros(16_666); +const WS_PREVIEW_FRAME_INTERVAL_SLACK: Duration = Duration::from_millis(1); const WS_BLUR_INFERENCE_INTERVAL: Duration = Duration::from_millis(150); fn preview_frame_due(last_preview_at: Option, now: Instant) -> bool { @@ -36,9 +46,67 @@ fn preview_frame_due(last_preview_at: Option, now: Instant) -> bool { }) } -fn scaled_preview_dimensions(width: u32, height: u32) -> (u32, u32) { - let width_scale = WS_PREVIEW_MAX_WIDTH as f64 / width.max(1) as f64; - let height_scale = WS_PREVIEW_MAX_HEIGHT as f64 / height.max(1) as f64; +const FRAME_POOL_MAX: usize = 4; + +// Reuses a previously-sent frame buffer once every WSFrame referencing it has +// been dropped (watch cell replaced + socket sends finished), eliminating the +// ~2MB allocation + page-fault churn per frame at steady state. +fn with_pooled_buffer( + pool: &mut Vec>>, + fill: impl FnOnce(&mut Vec), +) -> Arc> { + for buf in pool.iter_mut() { + if Arc::strong_count(buf) == 1 + && let Some(vec) = Arc::get_mut(buf) + { + vec.clear(); + fill(vec); + return buf.clone(); + } + } + + let mut vec = Vec::new(); + fill(&mut vec); + let buf = Arc::new(vec); + if pool.len() < FRAME_POOL_MAX { + pool.push(buf.clone()); + } + buf +} + +// Copies rows without ffmpeg's stride padding so the payload is packed +// (stride == width * 4); this lets the frontend skip stride correction. +fn pack_rows(dst: &mut Vec, src: &[u8], width: u32, height: u32, stride: u32) { + let row_bytes = (width as usize) * 4; + let stride = stride as usize; + let height = height as usize; + dst.reserve(row_bytes * height); + if stride == row_bytes { + dst.extend_from_slice(&src[..row_bytes * height]); + } else { + for row in 0..height { + let start = row * stride; + dst.extend_from_slice(&src[start..start + row_bytes]); + } + } +} + +fn scaled_preview_dimensions(width: u32, height: u32, state: &CameraPreviewState) -> (u32, u32) { + let blur_enabled = state.background_blur != cap_project::BackgroundBlurMode::Off; + let (max_width, max_height) = if blur_enabled { + (WS_PREVIEW_BLUR_MAX_WIDTH, WS_PREVIEW_BLUR_MAX_HEIGHT) + } else { + (WS_PREVIEW_MAX_WIDTH, WS_PREVIEW_MAX_HEIGHT) + }; + let visible_width = (state.size.clamp(MIN_CAMERA_SIZE, MAX_CAMERA_SIZE) as u32) + .saturating_mul(WS_PREVIEW_SURFACE_SCALE); + let requested_width = visible_width + .max(WS_PREVIEW_MIN_WIDTH) + .div_ceil(WS_PREVIEW_WIDTH_BUCKET) + .saturating_mul(WS_PREVIEW_WIDTH_BUCKET) + .min(max_width); + let width_scale = requested_width as f64 / width.max(1) as f64; + let height_scale = max_height as f64 / height.max(1) as f64; let scale = width_scale.min(height_scale).min(1.0); let target_width = ((width as f64 * scale).round() as u32).max(1); let target_height = ((height as f64 * scale).round() as u32).max(1); @@ -46,20 +114,44 @@ fn scaled_preview_dimensions(width: u32, height: u32) -> (u32, u32) { } pub async fn create_camera_preview_ws( - blur_rx: watch::Receiver, + state_rx: watch::Receiver, ) -> (Sender, u16, CancellationToken) { - let (camera_tx, camera_rx) = flume::bounded::(4); - let (frame_tx, _) = tokio::sync::broadcast::channel::(4); + let (camera_tx, camera_rx) = flume::bounded::(1); + let (frame_tx, frame_rx) = watch::channel::>>(None); + let subscriber_count = Arc::new(AtomicUsize::new(0)); let frame_tx_clone = frame_tx.clone(); + let thread_subscriber_count = subscriber_count.clone(); std::thread::spawn(move || { use ffmpeg::format::Pixel; + #[cfg(target_os = "macos")] + { + let result = set_current_thread_qos(MacOsQosClass::UserInteractive); + if result != 0 { + tracing::warn!(result, "pthread_set_qos_class_self_np failed"); + } + } + #[cfg(windows)] + { + use windows::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_ABOVE_NORMAL, + }; + if let Err(err) = + unsafe { SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL) } + { + tracing::warn!("SetThreadPriority failed: {err}"); + } + } + let mut converter: Option<(Pixel, ffmpeg::software::scaling::Context)> = None; let mut reusable_frame: Option = None; - let mut blur_rx = blur_rx; + let mut state_rx = state_rx; let mut blur_state = WsBlurState::new(); let mut last_preview_at = None; + let mut frame_pool: Vec>> = Vec::new(); + let mut frame_counter: u32 = 0; + let mut idle = true; while let Ok(raw_frame) = camera_rx.recv() { let mut frame = raw_frame.inner; @@ -68,13 +160,31 @@ pub async fn create_camera_preview_ws( frame = newer.inner; } + // With no connected ws clients, skip all conversion work and + // release retained resources; the cleared watch cell also stops + // stale frames from being replayed to the next connection. + if thread_subscriber_count.load(Ordering::Acquire) == 0 { + if !idle { + idle = true; + converter = None; + reusable_frame = None; + frame_pool.clear(); + blur_state.release(); + last_preview_at = None; + let _previous_frame = frame_tx_clone.send_replace(None); + } + continue; + } + idle = false; + let now = Instant::now(); if !preview_frame_due(last_preview_at, now) { continue; } last_preview_at = Some(now); - let blur_mode = *blur_rx.borrow_and_update(); + let state = state_rx.borrow_and_update().clone(); + let blur_mode = state.background_blur; let blur_enabled = blur_mode != cap_project::BackgroundBlurMode::Off; let effects_mode = match blur_mode { cap_project::BackgroundBlurMode::Off | cap_project::BackgroundBlurMode::Light => { @@ -84,17 +194,23 @@ pub async fn create_camera_preview_ws( }; let (target_width, target_height) = - scaled_preview_dimensions(frame.width(), frame.height()); + scaled_preview_dimensions(frame.width(), frame.height(), &state); let needs_convert = frame.format() != Pixel::RGBA || frame.width() != target_width || frame.height() != target_height; + if !blur_enabled { + blur_state.release(); + } + if needs_convert { let ctx = match &mut converter { Some((format, ctx)) if *format == frame.format() && ctx.input().width == frame.width() - && ctx.input().height == frame.height() => + && ctx.input().height == frame.height() + && ctx.output().width == target_width + && ctx.output().height == target_height => { ctx } @@ -128,85 +244,75 @@ pub async fn create_camera_preview_ws( continue; } - let (data, width, height, stride) = if blur_enabled { - match blur_state.process( + let width = out_frame.width(); + let height = out_frame.height(); + let src_stride = out_frame.stride(0) as u32; + let data = if blur_enabled { + blur_state.process( out_frame.data(0), - out_frame.width(), - out_frame.height(), - out_frame.stride(0) as u32, - effects_mode, - ) { - Some(blurred) => blurred, - None => ( - std::sync::Arc::new(out_frame.data(0).to_vec()), - out_frame.width(), - out_frame.height(), - out_frame.stride(0) as u32, - ), - } - } else { - ( - std::sync::Arc::new(out_frame.data(0).to_vec()), - out_frame.width(), - out_frame.height(), - out_frame.stride(0) as u32, - ) - }; - - frame_tx_clone - .send(WSFrame { - data, width, height, - stride, - frame_number: 0, - target_time_ns: 0, - format: crate::frame_ws::WSFrameFormat::Rgba, - created_at: Instant::now(), - }) - .ok(); - } else { - let (data, width, height, stride) = if blur_enabled { - match blur_state.process( - frame.data(0), - frame.width(), - frame.height(), - frame.stride(0) as u32, + src_stride, effects_mode, - ) { - Some(blurred) => blurred, - None => ( - std::sync::Arc::new(frame.data(0).to_vec()), - frame.width(), - frame.height(), - frame.stride(0) as u32, - ), - } - } else { - ( - std::sync::Arc::new(frame.data(0).to_vec()), - frame.width(), - frame.height(), - frame.stride(0) as u32, + &mut frame_pool, ) - }; + } else { + None + } + .unwrap_or_else(|| { + with_pooled_buffer(&mut frame_pool, |vec| { + pack_rows(vec, out_frame.data(0), width, height, src_stride) + }) + }); - frame_tx_clone - .send(WSFrame { - data, + frame_counter = frame_counter.wrapping_add(1); + let _previous_frame = frame_tx_clone.send_replace(Some(Arc::new(WSFrame { + data, + width, + height, + stride: width * 4, + frame_number: frame_counter, + target_time_ns: 0, + format: crate::frame_ws::WSFrameFormat::Rgba, + created_at: Instant::now(), + }))); + } else { + let width = frame.width(); + let height = frame.height(); + let src_stride = frame.stride(0) as u32; + let data = if blur_enabled { + blur_state.process( + frame.data(0), width, height, - stride, - frame_number: 0, - target_time_ns: 0, - format: crate::frame_ws::WSFrameFormat::Rgba, - created_at: Instant::now(), + src_stride, + effects_mode, + &mut frame_pool, + ) + } else { + None + } + .unwrap_or_else(|| { + with_pooled_buffer(&mut frame_pool, |vec| { + pack_rows(vec, frame.data(0), width, height, src_stride) }) - .ok(); + }); + + frame_counter = frame_counter.wrapping_add(1); + let _previous_frame = frame_tx_clone.send_replace(Some(Arc::new(WSFrame { + data, + width, + height, + stride: width * 4, + frame_number: frame_counter, + target_time_ns: 0, + format: crate::frame_ws::WSFrameFormat::Rgba, + created_at: Instant::now(), + }))); } } }); - let (camera_ws_port, _shutdown) = create_frame_ws(frame_tx).await; + let (camera_ws_port, _shutdown) = create_watch_frame_ws(frame_rx, subscriber_count).await; (camera_tx, camera_ws_port, _shutdown) } @@ -233,6 +339,16 @@ impl WsBlurState { } } + // Drops the dedicated wgpu device, ONNX session, and readback buffers as + // soon as blur is off; re-enabling re-runs the lazy init. + fn release(&mut self) { + if self.processor.is_some() { + self.processor = None; + tracing::info!("Released WebSocket camera blur resources"); + } + self.init_attempted = false; + } + fn process( &mut self, rgba_data: &[u8], @@ -240,7 +356,8 @@ impl WsBlurState { height: u32, stride: u32, mode: cap_camera_effects::BlurMode, - ) -> Option<(Arc>, u32, u32, u32)> { + pool: &mut Vec>>, + ) -> Option>> { if !self.init_attempted { self.init_attempted = true; self.processor = init_headless_blur(); @@ -332,12 +449,14 @@ impl WsBlurState { width, height, bytes_per_row_aligned, + pool, ); let curr_data = try_drain_readback( &mut res.readbacks.as_mut().unwrap().2[current_idx], width, height, bytes_per_row_aligned, + pool, ); let blurred_out = prev_data.or(curr_data); @@ -406,7 +525,7 @@ impl WsBlurState { res.current_idx = 1 - idx; } - blurred_out.map(|out| (Arc::new(out), width, height, width * 4)) + blurred_out } } @@ -415,7 +534,8 @@ fn try_drain_readback( width: u32, height: u32, bytes_per_row_aligned: u32, -) -> Option> { + pool: &mut Vec>>, +) -> Option>> { let WsReadbackState::InFlight(status) = &readback.state else { return None; }; @@ -424,11 +544,13 @@ fn try_drain_readback( let slice = readback.buffer.slice(..); let data = slice.get_mapped_range(); let row_bytes = (width * 4) as usize; - let mut out = Vec::with_capacity(row_bytes * height as usize); - for row in 0..height as usize { - let start = row * bytes_per_row_aligned as usize; - out.extend_from_slice(&data[start..start + row_bytes]); - } + let out = with_pooled_buffer(pool, |vec| { + vec.reserve(row_bytes * height as usize); + for row in 0..height as usize { + let start = row * bytes_per_row_aligned as usize; + vec.extend_from_slice(&data[start..start + row_bytes]); + } + }); drop(data); readback.buffer.unmap(); readback.state = WsReadbackState::Idle; From 55707bf0ea83b8b99ea2674d6487dceb513f7bf3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:46 +0100 Subject: [PATCH 13/24] refactor(lib): wire native camera preview sender and state channel --- apps/desktop/src-tauri/src/lib.rs | 54 ++++++++++++++++++------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 78914f8ce2c..52a39b3becf 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -6,6 +6,8 @@ mod audio_meter; mod auth; mod camera; mod camera_legacy; +#[cfg(target_os = "macos")] +mod camera_native; mod captions; mod cli; mod crash_sentinel; @@ -45,7 +47,7 @@ mod windows; use audio::AppSounds; use auth::{AuthStore, Plan}; -use camera::{CameraPreviewManager, CameraPreviewState}; +use camera::{CameraPreviewManager, CameraPreviewSender, CameraPreviewState}; use cap_editor::{EditorInstance, EditorState}; use cap_project::{ InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SharingMeta, @@ -540,7 +542,7 @@ pub struct App { #[deprecated = "can be removed when native camera preview is ready"] camera_ws_sender: flume::Sender, camera_preview: CameraPreviewManager, - camera_blur_tx: tokio::sync::watch::Sender, + camera_preview_state_tx: tokio::sync::watch::Sender, handle: AppHandle, recording_state: RecordingState, recording_logging_handle: LoggingHandle, @@ -656,17 +658,21 @@ async fn remove_camera_preview_sender( async fn sync_camera_preview_sender( camera_feed: &ActorRef, camera_ws_sender: flume::Sender, - camera_preview_sender: Option>, + camera_preview_sender: Option, use_ws_preview: bool, ) { if use_ws_preview { - if let Some(sender) = camera_preview_sender { - remove_camera_preview_sender(camera_feed, sender, "native preview").await; + if let Some(sender) = camera_preview_sender + && let Err(err) = sender.detach(camera_feed).await + { + warn!(error = %err, "Failed to remove native preview camera sender"); } add_camera_preview_ws_sender(camera_feed, camera_ws_sender).await; } else if let Some(sender) = camera_preview_sender { - add_camera_preview_sender(camera_feed, sender, "native preview").await; + if let Err(err) = sender.attach(camera_feed).await { + warn!(error = %err, "Failed to add native preview camera sender"); + } remove_camera_preview_sender(camera_feed, camera_ws_sender, "WebSocket").await; } else { add_camera_preview_ws_sender(camera_feed, camera_ws_sender).await; @@ -1255,6 +1261,10 @@ async fn set_native_camera_preview_enabled( state: MutableState<'_, App>, enabled: bool, ) -> Result<(), String> { + if enabled && cfg!(not(target_os = "macos")) { + return Err("Native camera preview is only available on macOS".to_string()); + } + let operation_lock = app_handle.state::(); let _operation_guard = operation_lock.lock().await; @@ -3974,13 +3984,13 @@ async fn set_camera_preview_state( state: CameraPreviewState, ) -> Result<(), String> { let app_guard = app.read().await; - let blur_mode = state.background_blur; + let state_for_ws = state.clone(); app_guard .camera_preview .set_state(state) .map_err(|err| format!("Error saving camera window state: {err}"))?; - app_guard.camera_blur_tx.send(blur_mode).ok(); + let _ = app_guard.camera_preview_state_tx.send(state_for_ws); drop(app_guard); Ok(()) @@ -4032,8 +4042,8 @@ async fn refresh_camera_feed(state: MutableState<'_, App>) -> Result<(), String> if use_ws_preview { if let Some(sender) = camera_preview_sender { - camera_feed - .ask(feeds::camera::RemoveSender(sender)) + sender + .detach(&camera_feed) .await .map_err(|err| format!("error removing native preview sender: {err}"))?; } @@ -4050,8 +4060,8 @@ async fn refresh_camera_feed(state: MutableState<'_, App>) -> Result<(), String> .await .map_err(|err| format!("error removing camera ws sender: {err}"))?; - camera_feed - .ask(feeds::camera::AddSender(sender)) + sender + .attach(&camera_feed) .await .map_err(|err| format!("error re-adding camera preview sender: {err}"))?; } else { @@ -4334,17 +4344,17 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { .typ::(); #[cfg(debug_assertions)] - specta_builder - .export( - specta_typescript::Typescript::default(), - "../src/utils/tauri.ts", - ) - .expect("Failed to export typescript bindings"); + if let Err(err) = specta_builder.export( + specta_typescript::Typescript::default(), + "../src/utils/tauri.ts", + ) { + warn!(error = %err, "Failed to export TypeScript bindings"); + } - let (camera_blur_tx, camera_blur_rx) = - tokio::sync::watch::channel(cap_project::BackgroundBlurMode::Off); + let (camera_preview_state_tx, camera_preview_state_rx) = + tokio::sync::watch::channel(CameraPreviewState::default()); let (camera_tx, camera_ws_port, _shutdown) = - camera_legacy::create_camera_preview_ws(camera_blur_rx).await; + camera_legacy::create_camera_preview_ws(camera_preview_state_rx).await; let camera_ws_sender = camera_tx.clone(); let (mic_samples_tx, mic_samples_rx) = flume::bounded(8); @@ -4579,7 +4589,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { camera_ws_sender, handle: app.clone(), camera_preview, - camera_blur_tx, + camera_preview_state_tx, recording_state: RecordingState::None, recording_logging_handle, mic_feed, From 684962437b4eef8964d493941f69613f2dfdabfa Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 14/24] refactor(windows): use native camera preview settings helper --- apps/desktop/src-tauri/src/windows.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index cf6c9438b6c..f890ee6b23a 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -348,7 +348,9 @@ pub(crate) async fn restore_main_window_inputs(app: &AppHandle) { let _ = camera_feed .ask(feeds::camera::RemoveSender(camera_ws_sender)) .await; - let _ = camera_feed.ask(feeds::camera::AddSender(sender)).await; + if let Err(err) = sender.attach(&camera_feed).await { + warn!(error = %err, "Failed to add native preview camera sender"); + } } else { #[allow(deprecated)] let _ = camera_feed @@ -1065,10 +1067,8 @@ impl ShowCapWindow { }; let mut app_state = state.write().await; - let enable_native_camera_preview = GeneralSettingsStore::get(app) - .ok() - .and_then(|v| v.map(|v| v.enable_native_camera_preview)) - .unwrap_or_default(); + let enable_native_camera_preview = + GeneralSettingsStore::native_camera_preview_enabled(app); let shutdown_preview = if !enable_native_camera_preview { app_state.camera_preview.begin_shutdown() @@ -1146,10 +1146,8 @@ impl ShowCapWindow { }; let mut app_state = state.write().await; - let enable_native_camera_preview = GeneralSettingsStore::get(app) - .ok() - .and_then(|v| v.map(|v| v.enable_native_camera_preview)) - .unwrap_or_default(); + let enable_native_camera_preview = + GeneralSettingsStore::native_camera_preview_enabled(app); let shutdown_preview = if !enable_native_camera_preview { app_state.camera_preview.begin_shutdown() @@ -1922,10 +1920,8 @@ impl ShowCapWindow { return Err(tauri::Error::WindowNotFound); }; - let enable_native_camera_preview = GeneralSettingsStore::get(app) - .ok() - .and_then(|v| v.map(|v| v.enable_native_camera_preview)) - .unwrap_or_default(); + let enable_native_camera_preview = + GeneralSettingsStore::native_camera_preview_enabled(app); { let Some(state) = app.try_state::>() else { From c59f5c73a86b46bca9f84eba193c0b8712d6155b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 15/24] feat(recording): prefer 60fps 16:9 camera formats up to 720p --- apps/desktop/src-tauri/src/recording.rs | 69 ++++++++++++++++++++----- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 246da557a84..564e8f82ec5 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -683,22 +683,63 @@ pub struct CameraWithFormats { } fn get_best_format(formats: &[CameraFormatInfo]) -> Option { - formats + let preferred_rate = 59.0..=60.0; + let supported_rate = 24.0..=60.0; + + let mut ideal_formats = formats .iter() - .filter(|f| f.frame_rate >= 24.0 && f.frame_rate <= 60.0) - .max_by(|a, b| { - let res_a = a.width * a.height; - let res_b = b.width * b.height; - res_a.cmp(&res_b) - }) - .or_else(|| { - formats.iter().max_by(|a, b| { - let res_a = a.width * a.height; - let res_b = b.width * b.height; - res_a.cmp(&res_b) + .filter(|f| preferred_rate.contains(&f.frame_rate) && f.width <= 1280 && f.height <= 720) + .collect::>(); + + if ideal_formats.is_empty() { + ideal_formats = formats + .iter() + .filter(|f| preferred_rate.contains(&f.frame_rate) && f.width < 2000 && f.height < 2000) + .collect(); + } + + if ideal_formats.is_empty() { + ideal_formats = formats + .iter() + .filter(|f| { + supported_rate.contains(&f.frame_rate) && f.width <= 1280 && f.height <= 720 }) - }) - .cloned() + .collect(); + } + + if ideal_formats.is_empty() { + ideal_formats = formats + .iter() + .filter(|f| supported_rate.contains(&f.frame_rate) && f.width < 2000 && f.height < 2000) + .collect(); + } + + if ideal_formats.is_empty() { + ideal_formats = formats.iter().collect(); + } + + ideal_formats.sort_by(|a, b| { + let target_aspect_ratio = 16.0 / 9.0; + let aspect_ratio_a = a.width as f32 / a.height as f32; + let aspect_ratio_b = b.width as f32 / b.height as f32; + let aspect_cmp_a = (aspect_ratio_a - target_aspect_ratio).abs(); + let aspect_cmp_b = (aspect_ratio_b - target_aspect_ratio).abs(); + let resolution_cmp = (a.width * a.height).cmp(&(b.width * b.height)); + let fr_cmp_a = (a.frame_rate - 60.0).abs(); + let fr_cmp_b = (b.frame_rate - 60.0).abs(); + + aspect_cmp_a + .partial_cmp(&aspect_cmp_b) + .unwrap_or(std::cmp::Ordering::Equal) + .then(resolution_cmp.reverse()) + .then( + fr_cmp_a + .partial_cmp(&fr_cmp_b) + .unwrap_or(std::cmp::Ordering::Equal), + ) + }); + + ideal_formats.into_iter().next().cloned() } #[tauri::command(async)] From fd685421efd920e587063a859b6d227fbba273d3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 16/24] feat(webgpu-renderer): support configurable adapter power preference --- apps/desktop/src/utils/webgpu-renderer.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/desktop/src/utils/webgpu-renderer.ts b/apps/desktop/src/utils/webgpu-renderer.ts index a25625ac98d..0d48775a253 100644 --- a/apps/desktop/src/utils/webgpu-renderer.ts +++ b/apps/desktop/src/utils/webgpu-renderer.ts @@ -94,22 +94,26 @@ function createEmptyTiming(start: number): WebGPURenderTiming { }; } -async function requestWebGPUAdapter(): Promise { - let highPerformanceAdapter: GPUAdapter | null = null; +async function requestWebGPUAdapter( + powerPreference: GPUPowerPreference = "high-performance", +): Promise { + let preferredAdapter: GPUAdapter | null = null; try { - highPerformanceAdapter = await navigator.gpu.requestAdapter({ - powerPreference: "high-performance", + preferredAdapter = await navigator.gpu.requestAdapter({ + powerPreference, }); } catch {} - return highPerformanceAdapter ?? navigator.gpu.requestAdapter(); + return preferredAdapter ?? navigator.gpu.requestAdapter(); } -export async function isWebGPUSupported(): Promise { +export async function isWebGPUSupported( + powerPreference: GPUPowerPreference = "high-performance", +): Promise { if (typeof navigator === "undefined" || !navigator.gpu) { return false; } try { - const adapter = await requestWebGPUAdapter(); + const adapter = await requestWebGPUAdapter(powerPreference); return adapter !== null; } catch { return false; @@ -118,8 +122,9 @@ export async function isWebGPUSupported(): Promise { export async function initWebGPU( canvas: OffscreenCanvas, + powerPreference: GPUPowerPreference = "high-performance", ): Promise { - const adapter = await requestWebGPUAdapter(); + const adapter = await requestWebGPUAdapter(powerPreference); if (!adapter) { throw new Error("No WebGPU adapter available"); } From 44d19bdda45a99a15a50f5d299e726cf17658264 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 17/24] perf(socket): optimize camera preview websocket frame rendering --- apps/desktop/src/utils/socket.ts | 518 +++++++++++++++++++++---------- 1 file changed, 349 insertions(+), 169 deletions(-) diff --git a/apps/desktop/src/utils/socket.ts b/apps/desktop/src/utils/socket.ts index 1c7babc8ed6..37b8d251689 100644 --- a/apps/desktop/src/utils/socket.ts +++ b/apps/desktop/src/utils/socket.ts @@ -13,7 +13,6 @@ import StrideCorrectionWorker from "./stride-correction-worker?worker"; import { disposeWebGPU, initWebGPU, - isWebGPUSupported, renderFrameWebGPU, renderNv12FrameWebGPU, type WebGPURenderer, @@ -21,9 +20,11 @@ import { } from "./webgpu-renderer"; const SAB_SUPPORTED = isSharedArrayBufferSupported(); +// Preview frames are capped at 960x540 RGBA (~2.1MB) by the Rust sender, so +// 4MB slots leave 2x headroom; oversized frames fall back to postMessage. const FRAME_BUFFER_CONFIG: SharedFrameBufferConfig = { - slotCount: 6, - slotSize: 16 * 1024 * 1024, + slotCount: 4, + slotSize: 4 * 1024 * 1024, }; let mainThreadNv12Buffer: Uint8ClampedArray | null = null; @@ -147,6 +148,10 @@ export type CanvasControls = { dispose: () => void; }; +export type ImageDataWSOptions = { + powerPreference?: GPUPowerPreference; +}; + interface ReadyMessage { type: "ready"; } @@ -191,6 +196,7 @@ export function createImageDataWS( url: string, onmessage: (data: FrameData) => void, onRequestFrame?: () => void, + options: ImageDataWSOptions = {}, ): [ Omit, () => boolean, @@ -201,28 +207,39 @@ export function createImageDataWS( const [isWorkerReady, setIsWorkerReady] = createSignal(false); const ws = createWS(url); - const worker = new FrameWorker(); + // The frame worker (and its SharedArrayBuffer) only exists for the + // OffscreenCanvas path; the direct-canvas consumers never need it, so it + // is created lazily to avoid a worker + SAB reservation per window. + let worker: Worker | null = null; + let workerCanvasMode = false; let pendingFrame: ArrayBuffer | null = null; let isProcessing = false; let isProcessingSharedFrame = false; let nextFrame: ArrayBuffer | null = null; let producer: Producer | null = null; - if (SAB_SUPPORTED) { - try { - const init = createSharedFrameBuffer(FRAME_BUFFER_CONFIG); - producer = createProducer(init); - worker.postMessage({ - type: "init-shared-buffer", - buffer: init.buffer, - }); - } catch (e) { - console.error( - "[socket] SharedArrayBuffer allocation failed, falling back to non-SAB mode:", - e instanceof Error ? e.message : e, - ); - producer = null; + + function ensureWorker(): Worker { + if (worker) return worker; + worker = new FrameWorker(); + worker.onmessage = handleWorkerMessage; + if (SAB_SUPPORTED) { + try { + const init = createSharedFrameBuffer(FRAME_BUFFER_CONFIG); + producer = createProducer(init); + worker.postMessage({ + type: "init-shared-buffer", + buffer: init.buffer, + }); + } catch (e) { + console.error( + "[socket] SharedArrayBuffer allocation failed, falling back to non-SAB mode:", + e instanceof Error ? e.message : e, + ); + producer = null; + } } + return worker; } const [hasRenderedFrame, setHasRenderedFrame] = createSignal(false); @@ -248,6 +265,12 @@ export function createImageDataWS( null; let pendingNv12RafId: number | null = null; let pendingRgbaRafId: number | null = null; + let pendingCanvas2DNv12RafId: number | null = null; + let pendingCanvas2DRgbaFrame: { + buffer: ArrayBuffer; + receivedAt: number; + } | null = null; + let pendingCanvas2DRgbaRafId: number | null = null; let lastRenderedFrameData: { data: Uint8ClampedArray; @@ -287,8 +310,11 @@ export function createImageDataWS( producer = null; } - worker.onmessage = null; - worker.terminate(); + if (worker) { + worker.onmessage = null; + worker.terminate(); + worker = null; + } if (strideWorker) { strideWorker.onmessage = null; @@ -309,6 +335,7 @@ export function createImageDataWS( mainThreadWebGPUInitializing = false; pendingNv12Frame = null; pendingRgbaFrame = null; + pendingCanvas2DRgbaFrame = null; if (pendingNv12RafId !== null) { cancelAnimationFrame(pendingNv12RafId); pendingNv12RafId = null; @@ -317,6 +344,14 @@ export function createImageDataWS( cancelAnimationFrame(pendingRgbaRafId); pendingRgbaRafId = null; } + if (pendingCanvas2DNv12RafId !== null) { + cancelAnimationFrame(pendingCanvas2DNv12RafId); + pendingCanvas2DNv12RafId = null; + } + if (pendingCanvas2DRgbaRafId !== null) { + cancelAnimationFrame(pendingCanvas2DRgbaRafId); + pendingCanvas2DRgbaRafId = null; + } mainThreadNv12Buffer = null; mainThreadNv12BufferSize = 0; cachedDirectImageData = null; @@ -460,6 +495,7 @@ export function createImageDataWS( width: number, height: number, yStride: number, + receivedAt?: number, ) { if (!directCanvas || !directCtx) return; @@ -485,14 +521,19 @@ export function createImageDataWS( directCtx.putImageData(cachedDirectImageData, 0, 0); storeRenderedFrame(frameData, width, height, yStride, true); - recordRender(performance.now() - renderStart, "canvas2d"); + recordRender( + performance.now() - renderStart, + "canvas2d", + undefined, + receivedAt, + ); onmessage({ width, height }); } function renderPendingFrameCanvas2D() { if (!pendingNv12Frame || !directCanvas || !directCtx) return; - const { buffer } = pendingNv12Frame; + const { buffer, receivedAt } = pendingNv12Frame; pendingNv12Frame = null; const NV12_MAGIC = 0x4e563132; @@ -513,19 +554,180 @@ export function createImageDataWS( const totalSize = ySize + uvSize; const frameData = new Uint8ClampedArray(buffer, 0, totalSize); - renderNv12FrameCanvas2D(frameData, width, height, yStride); + renderNv12FrameCanvas2D(frameData, width, height, yStride, receivedAt); + } + } + + function schedulePendingNv12FrameCanvas2D( + buffer: ArrayBuffer, + receivedAt: number, + ) { + pendingNv12Frame = { buffer, receivedAt }; + if (pendingCanvas2DNv12RafId !== null) return; + + pendingCanvas2DNv12RafId = requestAnimationFrame(() => { + pendingCanvas2DNv12RafId = null; + renderPendingFrameCanvas2D(); + }); + } + + function ensureStrideWorker(): Worker { + if (strideWorker) return strideWorker; + strideWorker = new StrideCorrectionWorker(); + strideWorker.onmessage = (e: MessageEvent) => { + if (e.data.type !== "corrected" || !directCanvas || !directCtx) return; + + const { buffer, width, height } = e.data; + const renderStart = performance.now(); + if (directCanvas.width !== width || directCanvas.height !== height) { + directCanvas.width = width; + directCanvas.height = height; + } + + const frameData = new Uint8ClampedArray(buffer); + if ( + !cachedStrideImageData || + cachedStrideWidth !== width || + cachedStrideHeight !== height + ) { + cachedStrideImageData = new ImageData(width, height); + cachedStrideWidth = width; + cachedStrideHeight = height; + } + cachedStrideImageData.data.set(frameData); + directCtx.putImageData(cachedStrideImageData, 0, 0); + + storeRenderedFrame( + cachedStrideImageData.data, + width, + height, + width * 4, + false, + ); + recordRender(performance.now() - renderStart, "canvas2d"); + onmessage({ width, height }); + }; + return strideWorker; + } + + function renderRgbaFrameCanvas2D(buffer: ArrayBuffer, receivedAt: number) { + if (!directCanvas || !directCtx) return; + if (buffer.byteLength < 24) return; + + const metadataOffset = buffer.byteLength - 24; + const meta = new DataView(buffer, metadataOffset, 24); + const strideBytes = meta.getUint32(0, true); + const height = meta.getUint32(4, true); + const width = meta.getUint32(8, true); + + if (width <= 0 || height <= 0) return; + + const expectedRowBytes = width * 4; + const needsStrideCorrection = strideBytes !== expectedRowBytes; + const availableLength = strideBytes * height; + + if ( + strideBytes === 0 || + strideBytes < expectedRowBytes || + buffer.byteLength - 24 < availableLength + ) { + return; + } + + if (!needsStrideCorrection) { + const frameData = new Uint8ClampedArray( + buffer, + 0, + expectedRowBytes * height, + ); + const renderStart = performance.now(); + + if (directCanvas.width !== width || directCanvas.height !== height) { + directCanvas.width = width; + directCanvas.height = height; + } + + if ( + !cachedDirectImageData || + cachedDirectWidth !== width || + cachedDirectHeight !== height + ) { + cachedDirectImageData = new ImageData(width, height); + cachedDirectWidth = width; + cachedDirectHeight = height; + } + cachedDirectImageData.data.set(frameData); + directCtx.putImageData(cachedDirectImageData, 0, 0); + + storeRenderedFrame( + cachedDirectImageData.data, + width, + height, + width * 4, + false, + ); + recordRender( + performance.now() - renderStart, + "canvas2d", + undefined, + receivedAt, + ); + onmessage({ width, height }); + return; + } + + ensureStrideWorker().postMessage( + { + type: "correct-stride", + buffer, + strideBytes, + width, + height, + }, + [buffer], + ); + } + + function renderPendingRgbaFrameCanvas2D() { + const pending = pendingCanvas2DRgbaFrame ?? pendingRgbaFrame; + if (!pending) return; + + if (pendingCanvas2DRgbaFrame) { + pendingCanvas2DRgbaFrame = null; + } else { + pendingRgbaFrame = null; } + + renderRgbaFrameCanvas2D(pending.buffer, pending.receivedAt); + } + + function schedulePendingRgbaFrameCanvas2D( + buffer: ArrayBuffer, + receivedAt: number, + ) { + pendingCanvas2DRgbaFrame = { buffer, receivedAt }; + if (pendingCanvas2DRgbaRafId !== null) return; + + pendingCanvas2DRgbaRafId = requestAnimationFrame(() => { + pendingCanvas2DRgbaRafId = null; + renderPendingRgbaFrameCanvas2D(); + }); } const canvasControls: CanvasControls = { initCanvas: (canvas: OffscreenCanvas) => { - worker.postMessage({ type: "init-canvas", canvas }, [canvas]); + if (isCleanedUp) return; + workerCanvasMode = true; + ensureWorker().postMessage({ type: "init-canvas", canvas }, [canvas]); }, resizeCanvas: (width: number, height: number) => { - worker.postMessage({ type: "resize", width, height }); + if (isCleanedUp) return; + worker?.postMessage({ type: "resize", width, height }); }, hasRenderedFrame, initDirectCanvas: (canvas: HTMLCanvasElement) => { + if (isCleanedUp) return; + const isNewCanvas = directCanvas !== canvas; if (isNewCanvas && directCanvas) { @@ -546,86 +748,71 @@ export function createImageDataWS( if (!mainThreadWebGPUInitializing && !mainThreadWebGPU) { mainThreadWebGPUInitializing = true; - isWebGPUSupported().then((supported) => { - if (supported && directCanvas) { - initWebGPU(directCanvas as unknown as OffscreenCanvas) - .then((renderer) => { - mainThreadWebGPU = renderer; + // initWebGPU's catch covers the no-adapter case, so a separate + // isWebGPUSupported probe would just request the adapter twice. + const maybeSupported = + typeof navigator !== "undefined" && !!navigator.gpu; + if (maybeSupported && directCanvas) { + initWebGPU( + directCanvas as unknown as OffscreenCanvas, + options.powerPreference, + ) + .then((renderer) => { + if (isCleanedUp || !directCanvas) { + disposeWebGPU(renderer); mainThreadWebGPUInitializing = false; - if (pendingNv12Frame && directCanvas) { - renderPendingNv12Frame(); - } - if (pendingRgbaFrame && directCanvas) { - renderPendingRgbaFrame(); - } - onRequestFrame?.(); - }) - .catch((e) => { + return; + } + + mainThreadWebGPU = renderer; + mainThreadWebGPUInitializing = false; + if (pendingNv12Frame && directCanvas) { + renderPendingNv12Frame(); + } + if (pendingRgbaFrame && directCanvas) { + renderPendingRgbaFrame(); + } + onRequestFrame?.(); + }) + .catch((e) => { + if (isCleanedUp) { mainThreadWebGPUInitializing = false; - console.error("[Socket] Main thread WebGPU init failed:", e); - directCtx = - directCanvas?.getContext("2d", { alpha: false }) ?? null; - if (pendingNv12Frame && directCanvas && directCtx) { - renderPendingFrameCanvas2D(); - } - onRequestFrame?.(); - }); - } else { - mainThreadWebGPUInitializing = false; - directCtx = - directCanvas?.getContext("2d", { alpha: false }) ?? null; - if (pendingNv12Frame && directCanvas && directCtx) { - renderPendingFrameCanvas2D(); - } - onRequestFrame?.(); - } - }); - } - - if (!strideWorker) { - strideWorker = new StrideCorrectionWorker(); - strideWorker.onmessage = ( - e: MessageEvent, - ) => { - if (e.data.type !== "corrected" || !directCanvas || !directCtx) - return; - - const { buffer, width, height } = e.data; - const renderStart = performance.now(); - if (directCanvas.width !== width || directCanvas.height !== height) { - directCanvas.width = width; - directCanvas.height = height; + return; + } + + mainThreadWebGPUInitializing = false; + console.error("[Socket] Main thread WebGPU init failed:", e); + directCtx = + directCanvas?.getContext("2d", { alpha: false }) ?? null; + if (pendingNv12Frame && directCanvas && directCtx) { + renderPendingFrameCanvas2D(); + } + if (pendingRgbaFrame && directCanvas && directCtx) { + renderPendingRgbaFrameCanvas2D(); + } + onRequestFrame?.(); + }); + } else { + mainThreadWebGPUInitializing = false; + directCtx = directCanvas?.getContext("2d", { alpha: false }) ?? null; + if (pendingNv12Frame && directCanvas && directCtx) { + renderPendingFrameCanvas2D(); } - - const frameData = new Uint8ClampedArray(buffer); - if ( - !cachedStrideImageData || - cachedStrideWidth !== width || - cachedStrideHeight !== height - ) { - cachedStrideImageData = new ImageData(width, height); - cachedStrideWidth = width; - cachedStrideHeight = height; + if (pendingRgbaFrame && directCanvas && directCtx) { + renderPendingRgbaFrameCanvas2D(); } - cachedStrideImageData.data.set(frameData); - directCtx.putImageData(cachedStrideImageData, 0, 0); - - storeRenderedFrame( - cachedStrideImageData.data, - width, - height, - width * 4, - false, - ); - recordRender(performance.now() - renderStart, "canvas2d"); - onmessage({ width, height }); - }; + onRequestFrame?.(); + } } }, resetFrameState: () => { - worker.postMessage({ type: "reset-frame-state" }); + if (isCleanedUp) return; + worker?.postMessage({ type: "reset-frame-state" }); }, captureFrame: async () => { + if (isCleanedUp) { + return null; + } if (!lastRenderedFrameData) { return null; } @@ -674,7 +861,7 @@ export function createImageDataWS( }, }; - worker.onmessage = (e: MessageEvent) => { + function handleWorkerMessage(e: MessageEvent) { if (e.data.type === "ready") { setIsWorkerReady(true); return; @@ -699,11 +886,11 @@ export function createImageDataWS( if (e.data.type === "frame-rendered") { const { width, height } = e.data; - onmessage({ width, height }); - recordRender(0, "worker"); if (!hasRenderedFrame()) { setHasRenderedFrame(true); } + onmessage({ width, height }); + recordRender(0, "worker"); if (isProcessingSharedFrame) { isProcessingSharedFrame = false; isProcessing = false; @@ -724,7 +911,7 @@ export function createImageDataWS( isProcessing = false; processNextFrame(); } - }; + } function processNextFrame() { if (isProcessing) return; @@ -740,20 +927,21 @@ export function createImageDataWS( isProcessing = true; + const frameWorker = ensureWorker(); if (producer) { const written = producer.write(buffer); if (!written) { sharedBufferFallbacks++; isProcessingSharedFrame = false; - worker.postMessage({ type: "frame", buffer }, [buffer]); + frameWorker.postMessage({ type: "frame", buffer }, [buffer]); } else { sharedBufferWrites++; isProcessingSharedFrame = true; - worker.postMessage({ type: "wake" }); + frameWorker.postMessage({ type: "wake" }); } } else { isProcessingSharedFrame = false; - worker.postMessage({ type: "frame", buffer }, [buffer]); + frameWorker.postMessage({ type: "frame", buffer }, [buffer]); } } @@ -960,17 +1148,30 @@ export function createImageDataWS( const uvSize = yStride * (height / 2); const totalSize = ySize + uvSize; - const nv12Data = new Uint8ClampedArray(buffer, 0, totalSize); - renderNv12FrameCanvas2D(nv12Data, width, height, yStride); + if (totalSize > 0) { + schedulePendingNv12FrameCanvas2D(buffer, now); + } } return; } - if (isProcessing) { - nextFrame = buffer; - } else { - pendingFrame = buffer; - processNextFrame(); + if (workerCanvasMode) { + if (isProcessing) { + nextFrame = buffer; + } else { + pendingFrame = buffer; + processNextFrame(); + } + return; + } + + pendingNv12Frame = { buffer, receivedAt: now }; + const metadataOffset = buffer.byteLength - 28; + const meta = new DataView(buffer, metadataOffset, 28); + const height = meta.getUint32(4, true); + const width = meta.getUint32(8, true); + if (width > 0 && height > 0) { + onmessage({ width, height }); } return; } @@ -992,77 +1193,56 @@ export function createImageDataWS( return; } - if (directCanvas && directCtx && strideWorker) { + if ( + mainThreadWebGPUInitializing && + directCanvas && + buffer.byteLength >= 24 + ) { + const metadataOffset = buffer.byteLength - 24; + const meta = new DataView(buffer, metadataOffset, 24); + const height = meta.getUint32(4, true); + const width = meta.getUint32(8, true); + + if (width > 0 && height > 0) { + pendingRgbaFrame = { buffer, receivedAt: now }; + onmessage({ width, height }); + } + return; + } + + if (directCanvas && directCtx) { if (buffer.byteLength >= 24) { const metadataOffset = buffer.byteLength - 24; const meta = new DataView(buffer, metadataOffset, 24); - const strideBytes = meta.getUint32(0, true); const height = meta.getUint32(4, true); const width = meta.getUint32(8, true); if (width > 0 && height > 0) { - const expectedRowBytes = width * 4; - const needsStrideCorrection = strideBytes !== expectedRowBytes; - - if (!needsStrideCorrection) { - const frameData = new Uint8ClampedArray( - buffer, - 0, - expectedRowBytes * height, - ); - const renderStart = performance.now(); - - if ( - directCanvas.width !== width || - directCanvas.height !== height - ) { - directCanvas.width = width; - directCanvas.height = height; - } - - if ( - !cachedDirectImageData || - cachedDirectWidth !== width || - cachedDirectHeight !== height - ) { - cachedDirectImageData = new ImageData(width, height); - cachedDirectWidth = width; - cachedDirectHeight = height; - } - cachedDirectImageData.data.set(frameData); - directCtx.putImageData(cachedDirectImageData, 0, 0); - - storeRenderedFrame( - cachedDirectImageData.data, - width, - height, - width * 4, - false, - ); - recordRender(performance.now() - renderStart, "canvas2d"); - onmessage({ width, height }); - } else { - strideWorker.postMessage( - { - type: "correct-stride", - buffer, - strideBytes, - width, - height, - }, - [buffer], - ); - } + schedulePendingRgbaFrameCanvas2D(buffer, now); } } return; } - if (isProcessing) { - nextFrame = buffer; - } else { - pendingFrame = buffer; - processNextFrame(); + if (workerCanvasMode) { + if (isProcessing) { + nextFrame = buffer; + } else { + pendingFrame = buffer; + processNextFrame(); + } + return; + } + + if (buffer.byteLength >= 24) { + const metadataOffset = buffer.byteLength - 24; + const meta = new DataView(buffer, metadataOffset, 24); + const height = meta.getUint32(4, true); + const width = meta.getUint32(8, true); + if (width > 0 && height > 0) { + pendingRgbaFrame = { buffer, receivedAt: now }; + onmessage({ width, height }); + } } }; From a5bf8b9076e140c1c9567a2668657ad4ec4f3496 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 18/24] refactor(camera): use shared websocket renderer in legacy preview --- apps/desktop/src/routes/camera.tsx | 281 +++++++++-------------------- 1 file changed, 86 insertions(+), 195 deletions(-) diff --git a/apps/desktop/src/routes/camera.tsx b/apps/desktop/src/routes/camera.tsx index ad2af5e1fe1..08cc7f02ca1 100644 --- a/apps/desktop/src/routes/camera.tsx +++ b/apps/desktop/src/routes/camera.tsx @@ -40,7 +40,11 @@ import { import { generalSettingsStore } from "~/store"; import { createTauriEventListener } from "~/utils/createEventListener"; import { createCameraMutation } from "~/utils/queries"; -import { createLazySignal } from "~/utils/socket"; +import { + type CanvasControls, + createImageDataWS, + type FrameData, +} from "~/utils/socket"; import { commands, events } from "~/utils/tauri"; import { RecordingOptionsProvider } from "./(window-chrome)/OptionsContext"; @@ -406,14 +410,7 @@ function LegacyCameraPreviewPage(props: { const [hasPositioned, setHasPositioned] = createSignal(isCameraOnlyMode()); - const [latestFrame, setLatestFrame] = createLazySignal<{ - width: number; - data: ImageData; - } | null>(); - let reusableFrameData: ImageData | null = null; - let reusableFrameWidth = 0; - let reusableFrameHeight = 0; - + const [hasFrame, setHasFrame] = createSignal(false); const [frameDimensions, setFrameDimensions] = createSignal<{ width: number; height: number; @@ -449,67 +446,6 @@ function LegacyCameraPreviewPage(props: { onCleanup(() => resizeObserver.disconnect()); }); - function getReusableFrameData(width: number, height: number) { - if ( - !reusableFrameData || - reusableFrameWidth !== width || - reusableFrameHeight !== height - ) { - reusableFrameData = new ImageData(width, height); - reusableFrameWidth = width; - reusableFrameHeight = height; - } - - return reusableFrameData; - } - - let pendingRender = false; - let rafId: number | null = null; - let cachedCtx: CanvasRenderingContext2D | null = null; - - function scheduleRender() { - if (rafId !== null) return; - rafId = requestAnimationFrame(() => { - rafId = null; - if (!pendingRender) return; - pendingRender = false; - - if (!cachedCtx && cameraCanvasRef) { - cachedCtx = cameraCanvasRef.getContext("2d"); - } - if (cachedCtx && reusableFrameData) { - cachedCtx.putImageData(reusableFrameData, 0, 0); - } - }); - } - - function imageDataHandler(imageData: { width: number; data: ImageData }) { - const currentFrame = latestFrame(); - if ( - !currentFrame || - currentFrame.data !== imageData.data || - currentFrame.data.width !== imageData.data.width || - currentFrame.data.height !== imageData.data.height - ) { - setLatestFrame(imageData); - } - - const currentDimensions = frameDimensions(); - if ( - !currentDimensions || - currentDimensions.width !== imageData.data.width || - currentDimensions.height !== imageData.data.height - ) { - setFrameDimensions({ - width: imageData.data.width, - height: imageData.data.height, - }); - } - - pendingRender = true; - scheduleRender(); - } - const STALL_TIMEOUT_MS = 2000; const WS_INITIAL_BACKOFF_MS = 1000; const WS_MAX_BACKOFF_MS = 30000; @@ -517,13 +453,51 @@ function LegacyCameraPreviewPage(props: { const { cameraWsPort } = window.__CAP__; const [isWindowVisible, setIsWindowVisible] = createSignal(!document.hidden); - const [_isConnected, setIsConnected] = createSignal(false); - let ws: WebSocket | undefined; + let ws: Omit | undefined; + let canvasControls: CanvasControls | undefined; let reconnectTimeout: ReturnType | undefined; let stallCheckInterval: ReturnType | undefined; let retryCount = 0; let isCleanedUp = false; let lastFrameTime = 0; + let cameraCanvasRef: HTMLCanvasElement | undefined; + + const closeSocket = () => { + const socket = ws; + const controls = canvasControls; + ws = undefined; + canvasControls = undefined; + controls?.dispose(); + if ( + socket && + socket.readyState !== WebSocket.CLOSING && + socket.readyState !== WebSocket.CLOSED + ) { + socket.close(); + } + }; + + const initCanvasControls = () => { + if (!canvasControls || !cameraCanvasRef) return; + canvasControls.initDirectCanvas(cameraCanvasRef); + }; + + const updateFrameState = (frame: FrameData) => { + retryCount = 0; + lastFrameTime = Date.now(); + + const currentDimensions = frameDimensions(); + if ( + !currentDimensions || + currentDimensions.width !== frame.width || + currentDimensions.height !== frame.height + ) { + setFrameDimensions({ width: frame.width, height: frame.height }); + } + if (canvasControls?.hasRenderedFrame()) { + setHasFrame(true); + } + }; onMount(() => { const handleVisibilityChange = () => { @@ -540,107 +514,36 @@ function LegacyCameraPreviewPage(props: { }); const createSocket = () => { - const socket = new WebSocket(`ws://localhost:${cameraWsPort}`); - socket.binaryType = "arraybuffer"; + const [socket, _isConnected, _isWorkerReady, controls] = createImageDataWS( + `ws://localhost:${cameraWsPort}`, + updateFrameState, + () => commands.refreshCameraFeed().catch(() => {}), + { powerPreference: "low-power" }, + ); + canvasControls = controls; + initCanvasControls(); - socket.onopen = () => { - setIsConnected(true); + socket.addEventListener("open", () => { lastFrameTime = Date.now(); - reusableFrameData = null; - reusableFrameWidth = 0; - reusableFrameHeight = 0; - if (cachedCtx && cameraCanvasRef) { - cachedCtx.clearRect( - 0, - 0, - cameraCanvasRef.width, - cameraCanvasRef.height, - ); - } - }; + setHasFrame(false); + setFrameDimensions(null); + }); - socket.onclose = () => { - setIsConnected(false); - cleanupSocket(socket); + socket.addEventListener("close", () => { + if (canvasControls === controls) { + canvasControls = undefined; + } if (ws === socket) ws = undefined; scheduleReconnect(); - }; - - socket.onerror = () => { - setIsConnected(false); - socket.close(); - }; - - socket.onmessage = (event) => { - if (!isWindowVisible()) return; - - retryCount = 0; - lastFrameTime = Date.now(); - if (pendingRender) return; - - const buffer = event.data as ArrayBuffer; - const clamped = new Uint8ClampedArray(buffer); - if (clamped.length < 24) { - console.error("Received frame too small to contain metadata"); - return; - } - - const metadataOffset = clamped.length - 24; - const meta = new DataView(buffer, metadataOffset, 24); - const strideBytes = meta.getUint32(0, true); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); - - if (!width || !height) { - console.error("Received invalid frame dimensions", { width, height }); - return; - } - - const source = clamped.subarray(0, metadataOffset); - const expectedRowBytes = width * 4; - const expectedLength = expectedRowBytes * height; - const availableLength = strideBytes * height; - - if ( - strideBytes === 0 || - strideBytes < expectedRowBytes || - source.length < availableLength - ) { - console.error("Received invalid frame stride", { - strideBytes, - expectedRowBytes, - height, - sourceLength: source.length, - }); - return; - } + }); - const imageData = getReusableFrameData(width, height); - if (strideBytes === expectedRowBytes) { - imageData.data.set(source.subarray(0, expectedLength)); - } else { - for (let row = 0; row < height; row += 1) { - const srcStart = row * strideBytes; - const destStart = row * expectedRowBytes; - imageData.data.set( - source.subarray(srcStart, srcStart + expectedRowBytes), - destStart, - ); - } - } - imageDataHandler({ width, data: imageData }); - }; + socket.addEventListener("error", () => { + controls.dispose(); + }); return socket; }; - const cleanupSocket = (socket: WebSocket) => { - socket.onopen = null; - socket.onclose = null; - socket.onerror = null; - socket.onmessage = null; - }; - const scheduleReconnect = () => { if ( isCleanedUp || @@ -677,13 +580,7 @@ function LegacyCameraPreviewPage(props: { stallCheckInterval = undefined; } - if (ws) { - cleanupSocket(ws); - ws.close(); - ws = undefined; - } - - setIsConnected(false); + closeSocket(); }; const startSocket = () => { @@ -717,14 +614,6 @@ function LegacyCameraPreviewPage(props: { onCleanup(() => { isCleanedUp = true; - if (rafId !== null) { - cancelAnimationFrame(rafId); - rafId = null; - } - cachedCtx = null; - reusableFrameData = null; - reusableFrameWidth = 0; - reusableFrameHeight = 0; stopSocket(); }); @@ -849,8 +738,6 @@ function LegacyCameraPreviewPage(props: { }, ); - let cameraCanvasRef: HTMLCanvasElement | undefined; - onMount(() => getCurrentWindow().show()); return ( @@ -888,13 +775,17 @@ function LegacyCameraPreviewPage(props: { data-tauri-drag-region > }> - - + { + cameraCanvasRef = canvas; + initCanvasControls(); + }} + containerSize={externalContainerSize() ?? undefined} + /> + + @@ -906,16 +797,18 @@ function LegacyCameraPreviewPage(props: { } function Canvas(props: { - latestFrame: Accessor<{ width: number; data: ImageData } | null | undefined>; + frameDimensions: Accessor< + { width: number; height: number } | null | undefined + >; state: CameraWindowState; - ref: HTMLCanvasElement | undefined; + onCanvas: (canvas: HTMLCanvasElement) => void; containerSize?: { width: number; height: number }; }) { const style = () => { - const frame = props.latestFrame(); - if (!frame) return {}; + const dimensions = props.frameDimensions(); + if (!dimensions) return {}; - const aspectRatio = frame.data.width / frame.data.height; + const aspectRatio = dimensions.width / dimensions.height; const targetSize = props.containerSize ?? @@ -949,9 +842,7 @@ function Canvas(props: { data-tauri-drag-region class={cx("absolute")} style={style()} - width={props.latestFrame()?.data.width} - height={props.latestFrame()?.data.height} - ref={props.ref} + ref={(canvas) => props.onCanvas(canvas)} /> ); } From bf549b277abffb5732c6afee803e148582d27a03 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:33:59 +0100 Subject: [PATCH 19/24] refactor(overlay): use shared websocket renderer in inline preview --- .../src/routes/target-select-overlay.tsx | 283 ++++++------------ 1 file changed, 88 insertions(+), 195 deletions(-) diff --git a/apps/desktop/src/routes/target-select-overlay.tsx b/apps/desktop/src/routes/target-select-overlay.tsx index 7893479aca7..21b58e77a64 100644 --- a/apps/desktop/src/routes/target-select-overlay.tsx +++ b/apps/desktop/src/routes/target-select-overlay.tsx @@ -65,6 +65,11 @@ import { createOptionsQuery, createOrganizationsQuery, } from "~/utils/queries"; +import { + type CanvasControls, + createImageDataWS, + type FrameData, +} from "~/utils/socket"; import { type CameraInfo, commands, @@ -1186,7 +1191,11 @@ function CameraPreviewInline() { createStore(getDefaultCameraWindowState()), { name: CAMERA_WINDOW_STATE_STORAGE_KEY }, ); - const [frame, setFrame] = createSignal(null); + const [hasFrame, setHasFrame] = createSignal(false); + const [frameDimensions, setFrameDimensions] = createSignal<{ + width: number; + height: number; + } | null>(null); const [connectionFailed, setConnectionFailed] = createSignal(false); const [chromeVisible, setChromeVisible] = createSignal(false); const [viewportSize, setViewportSize] = createSignal({ @@ -1194,19 +1203,54 @@ function CameraPreviewInline() { height: window.innerHeight, }); let canvasRef: HTMLCanvasElement | undefined; - let ws: WebSocket | undefined; + let ws: Omit | undefined; + let canvasControls: CanvasControls | undefined; let retryCount = 0; let reconnectTimeoutId: ReturnType | undefined; let stallCheckInterval: ReturnType | undefined; let isCleanedUp = false; - let reusableFrame: ImageData | null = null; - let reusableFrameWidth = 0; - let reusableFrameHeight = 0; let lastFrameTime = 0; const cameraWsPort = window.__CAP__?.cameraWsPort; const hasCameraSelected = () => rawOptions.cameraID !== null; + const closeSocket = () => { + const socket = ws; + const controls = canvasControls; + ws = undefined; + canvasControls = undefined; + controls?.dispose(); + if ( + socket && + socket.readyState !== WebSocket.CLOSING && + socket.readyState !== WebSocket.CLOSED + ) { + socket.close(); + } + }; + + const initCanvasControls = () => { + if (!canvasControls || !canvasRef) return; + canvasControls.initDirectCanvas(canvasRef); + }; + + const updateFrameState = (frame: FrameData) => { + resetBackoff(); + lastFrameTime = Date.now(); + + const dimensions = frameDimensions(); + if ( + !dimensions || + dimensions.width !== frame.width || + dimensions.height !== frame.height + ) { + setFrameDimensions({ width: frame.width, height: frame.height }); + } + if (canvasControls?.hasRenderedFrame()) { + setHasFrame(true); + } + }; + createEventListener(window, "resize", () => { setViewportSize({ width: window.innerWidth, @@ -1237,55 +1281,6 @@ function CameraPreviewInline() { }); }); - const getReusableFrame = (width: number, height: number) => { - if ( - !reusableFrame || - reusableFrameWidth !== width || - reusableFrameHeight !== height - ) { - reusableFrame = new ImageData(width, height); - reusableFrameWidth = width; - reusableFrameHeight = height; - } - - return reusableFrame; - }; - - let pendingRender = false; - let rafId: number | null = null; - let cachedCtx: CanvasRenderingContext2D | null = null; - let latestImageData: ImageData | null = null; - - const scheduleRender = () => { - if (rafId !== null) return; - rafId = requestAnimationFrame(() => { - rafId = null; - if (!pendingRender || !latestImageData) return; - pendingRender = false; - - const canvas = canvasRef; - if (!canvas) return; - if ( - canvas.width !== latestImageData.width || - canvas.height !== latestImageData.height - ) { - canvas.width = latestImageData.width; - canvas.height = latestImageData.height; - cachedCtx = null; - } - if (!cachedCtx) { - cachedCtx = canvas.getContext("2d"); - } - cachedCtx?.putImageData(latestImageData, 0, 0); - }); - }; - - const drawFrame = (image: ImageData) => { - latestImageData = image; - pendingRender = true; - scheduleRender(); - }; - const scheduleReconnect = () => { if (isCleanedUp || reconnectTimeoutId !== undefined || ws) return; @@ -1318,106 +1313,36 @@ function CameraPreviewInline() { } }; - const cleanupSocket = (socket: WebSocket) => { - socket.onopen = null; - socket.onclose = null; - socket.onerror = null; - socket.onmessage = null; - }; - const createSocket = () => { if (!cameraWsPort) return undefined; - const socket = new WebSocket(`ws://localhost:${cameraWsPort}`); - socket.binaryType = "arraybuffer"; + const [socket, _isConnected, _isWorkerReady, controls] = createImageDataWS( + `ws://localhost:${cameraWsPort}`, + updateFrameState, + () => commands.refreshCameraFeed().catch(() => {}), + { powerPreference: "low-power" }, + ); + canvasControls = controls; + initCanvasControls(); - socket.onopen = () => { + socket.addEventListener("open", () => { setConnectionFailed(false); lastFrameTime = Date.now(); - }; + setHasFrame(false); + setFrameDimensions(null); + }); - socket.onclose = () => { - cleanupSocket(socket); + socket.addEventListener("close", () => { + if (canvasControls === controls) { + canvasControls = undefined; + } if (ws === socket) ws = undefined; if (!isCleanedUp && hasCameraSelected()) scheduleReconnect(); - }; - - socket.onerror = () => { - socket.close(); - }; - - socket.onmessage = (event) => { - resetBackoff(); - lastFrameTime = Date.now(); - if (pendingRender) return; - - const buffer = event.data as ArrayBuffer; - const clamped = new Uint8ClampedArray(buffer); - if (clamped.length < 24) return; - - const MAX_FRAME_DIMENSION = 8192; - const MAX_STRIDE_BYTES = MAX_FRAME_DIMENSION * 4 * 2; - const MAX_FRAME_SIZE = MAX_FRAME_DIMENSION * MAX_FRAME_DIMENSION * 4; - - const metadataOffset = clamped.length - 24; - const meta = new DataView(buffer, metadataOffset, 24); - const strideBytes = meta.getUint32(0, true); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); - - if (!width || !height || strideBytes === 0) return; - - if ( - width > MAX_FRAME_DIMENSION || - height > MAX_FRAME_DIMENSION || - strideBytes > MAX_STRIDE_BYTES - ) - return; - - const source = clamped.subarray(0, metadataOffset); - const expectedRowBytes = width * 4; - const availableLength = strideBytes * height; - - if ( - expectedRowBytes > MAX_STRIDE_BYTES || - availableLength > MAX_FRAME_SIZE - ) - return; - - if (strideBytes < expectedRowBytes || source.length < availableLength) - return; - - const expectedLength = expectedRowBytes * height; - - if (expectedLength > MAX_FRAME_SIZE) return; - - const imageData = getReusableFrame(width, height); - - if (strideBytes === expectedRowBytes) { - imageData.data.set(source.subarray(0, expectedLength)); - } else { - for (let row = 0; row < height; row += 1) { - const srcStart = row * strideBytes; - const destStart = row * expectedRowBytes; - imageData.data.set( - source.subarray(srcStart, srcStart + expectedRowBytes), - destStart, - ); - } - } - - drawFrame(imageData); + }); - const currentFrame = frame(); - if ( - !currentFrame || - currentFrame !== imageData || - currentFrame.width !== imageData.width || - currentFrame.height !== imageData.height - ) { - setFrame(imageData); - } - }; + socket.addEventListener("error", () => { + controls.dispose(); + }); return socket; }; @@ -1456,31 +1381,15 @@ function CameraPreviewInline() { } }, WS_STALL_TIMEOUT_MS); } else { - if ( - ws && - ws.readyState !== WebSocket.CLOSING && - ws.readyState !== WebSocket.CLOSED - ) { - cleanupSocket(ws); - ws.close(); - } - ws = undefined; - setFrame(null); - reusableFrame = null; - reusableFrameWidth = 0; - reusableFrameHeight = 0; + setHasFrame(false); + setFrameDimensions(null); + closeSocket(); setConnectionFailed(false); } }); onCleanup(() => { isCleanedUp = true; - if (rafId !== null) { - cancelAnimationFrame(rafId); - rafId = null; - } - cachedCtx = null; - latestImageData = null; if (reconnectTimeoutId !== undefined) { clearTimeout(reconnectTimeoutId); reconnectTimeoutId = undefined; @@ -1489,22 +1398,15 @@ function CameraPreviewInline() { clearInterval(stallCheckInterval); stallCheckInterval = undefined; } - reusableFrame = null; - reusableFrameWidth = 0; - reusableFrameHeight = 0; - if (ws) { - cleanupSocket(ws); - ws.close(); - ws = undefined; - } + closeSocket(); }); const previewDimensions = () => { - const f = frame(); + const dimensions = frameDimensions(); const { width, height } = cameraPreviewDimensions( state.size, state.shape, - f ? f.width / f.height : undefined, + dimensions ? dimensions.width / dimensions.height : undefined, ); const viewport = viewportSize(); const maxWidth = Math.max(160, viewport.width - 48); @@ -1530,27 +1432,15 @@ function CameraPreviewInline() { return { height: "100%", "object-fit": "cover" as const, + opacity: hasFrame() ? "1" : "0", transform: state.mirrored ? "scaleX(-1)" : "scaleX(1)", width: "100%", }; }; - createEffect(() => { - const image = frame(); - const canvas = canvasRef; - if (!image || !canvas) return; - if (canvas.width !== image.width || canvas.height !== image.height) { - canvas.width = image.width; - canvas.height = image.height; - cachedCtx = null; - } - }); - const handleRetryConnection = () => { resetBackoff(); - if (ws) { - ws.close(); - } + closeSocket(); ws = createSocket(); }; @@ -1571,7 +1461,7 @@ function CameraPreviewInline() {
} > - Loading camera...
- } - > - + { + canvasRef = canvas; + initCanvasControls(); + }} + class="absolute inset-0" + style={canvasStyle()} + /> + +
Loading camera...
 From c91f82c7a6e8e0483f47d31857fc15c15259e238 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:34:00 +0100 Subject: [PATCH 20/24] docs(settings): update native camera preview experimental description --- .../src/routes/(window-chrome)/settings/experimental.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx index 3a52fdcfeb3..a7319f80423 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx @@ -70,7 +70,7 @@ function Inner(props: { handleChange("enableNativeCameraPreview", value) From e6e7caa5e23bf743f5fad3ed80b39d48708d16dc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:34:54 +0100 Subject: [PATCH 21/24] Update Cargo.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index cacc29748d3..fe96b2f588c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1310,6 +1310,7 @@ dependencies = [ "cap-audio", "cap-camera", "cap-camera-effects", + "cap-camera-ffmpeg", "cap-cli-install", "cap-editor", "cap-enc-ffmpeg", From ef3a82aba6ad8ab5d18107ada5ce090bcb6632c3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:53:41 +0100 Subject: [PATCH 22/24] fix: address camera preview PR feedback --- apps/desktop/src-tauri/src/camera.rs | 32 ++++++++++----------- apps/desktop/src-tauri/src/camera_native.rs | 3 +- apps/desktop/src/utils/socket.ts | 2 -- crates/recording/src/feeds/camera.rs | 11 ++++--- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index ed3a441e0af..221e189de97 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -18,7 +18,6 @@ use kameo::actor::ActorRef; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ - mem::ManuallyDrop, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -415,13 +414,11 @@ impl CameraPreviewManager { ); let (drop_tx, drop_rx) = oneshot::channel(); - let renderer = ManuallyDrop::new(renderer); - window - .run_on_main_thread(move || { - drop(ManuallyDrop::into_inner(renderer)); - let _ = drop_tx.send(()); - }) - .ok(); + let renderer = Box::new(renderer); + let _ = window.run_on_main_thread(move || { + drop(renderer); + let _ = drop_tx.send(()); + }); wait_for_shutdown_signal(&rt, drop_rx, Duration::from_millis(250)); @@ -1202,11 +1199,14 @@ impl Renderer { }); if blurred { - let blur_output = self + let Some(blur_output) = self .blur_processor .as_mut() .and_then(|p| p.process_returning_output()) - .expect("blurred flag guarantees output"); + else { + prepared.render(surface, frame_data, frame_stride); + return true; + }; let mut encoder = self .device .create_command_encoder(&wgpu::CommandEncoderDescriptor { @@ -1477,14 +1477,12 @@ impl Renderer { drop(std::mem::take(&mut self.texture)); self.aspect_ratio = Cached::default(); - let surface = ManuallyDrop::new(self.surface.take()); + let surface = self.surface.take(); let (drop_tx, drop_rx) = oneshot::channel(); - window - .run_on_main_thread(move || { - drop(ManuallyDrop::into_inner(surface)); - let _ = drop_tx.send(()); - }) - .ok(); + let _ = window.run_on_main_thread(move || { + drop(surface); + let _ = drop_tx.send(()); + }); let _ = tokio::time::timeout(Duration::from_millis(250), drop_rx).await; self.device.destroy(); diff --git a/apps/desktop/src-tauri/src/camera_native.rs b/apps/desktop/src-tauri/src/camera_native.rs index 44a1a7820c7..9d398845128 100644 --- a/apps/desktop/src-tauri/src/camera_native.rs +++ b/apps/desktop/src-tauri/src/camera_native.rs @@ -123,7 +123,8 @@ pub fn classify_frame(frame: &NativeCameraFrame) -> Result Ok(NativeFrameKind::Nv12 { full_range: false }), "420f" => Ok(NativeFrameKind::Nv12 { full_range: true }), "BGRA" => Ok(NativeFrameKind::Bgra), diff --git a/apps/desktop/src/utils/socket.ts b/apps/desktop/src/utils/socket.ts index 37b8d251689..46af09bbea7 100644 --- a/apps/desktop/src/utils/socket.ts +++ b/apps/desktop/src/utils/socket.ts @@ -748,8 +748,6 @@ export function createImageDataWS( if (!mainThreadWebGPUInitializing && !mainThreadWebGPU) { mainThreadWebGPUInitializing = true; - // initWebGPU's catch covers the no-adapter case, so a separate - // isWebGPUSupported probe would just request the adapter twice. const maybeSupported = typeof navigator !== "undefined" && !!navigator.gpu; if (maybeSupported && directCanvas) { diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 2f9a7954837..67528cd2dc4 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -522,11 +522,14 @@ fn release_camera_thread(handle: std::thread::JoinHandle<()>) { let _ = handle.join(); } else { warn!("Camera setup thread is still running after cancellation"); - let _ = std::thread::Builder::new() + if let Err(err) = std::thread::Builder::new() .name("camera-setup-reaper".to_string()) .spawn(move || { let _ = handle.join(); - }); + }) + { + warn!(?err, "Failed to spawn camera-setup-reaper thread"); + } } } @@ -801,7 +804,7 @@ async fn setup_camera( // converted to derive VideoInfo; afterwards skip the full-frame // copy entirely when nothing consumes ffmpeg frames. if ready_signal.is_none() - && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Relaxed) == 0 + && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Acquire) == 0 { return; } @@ -916,7 +919,7 @@ async fn setup_camera( // converted to derive VideoInfo; afterwards skip the full-frame // copy entirely when nothing consumes ffmpeg frames. if ready_signal.is_none() - && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Relaxed) == 0 + && ffmpeg_sender_count.load(std::sync::atomic::Ordering::Acquire) == 0 { return; } From 2e6868e2238c0021f3c9f72d31fabeaf9549c801 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:31:24 +0100 Subject: [PATCH 23/24] fix: address remaining camera feedback --- .../desktop-display-transport-benchmark.rs | 7 +- apps/desktop/src-tauri/src/editor_window.rs | 6 +- apps/desktop/src-tauri/src/fake_window.rs | 34 ++-- apps/desktop/src-tauri/src/frame_ws.rs | 144 +++++++++------- apps/desktop/src/utils/frame-worker.ts | 53 ++++-- apps/desktop/src/utils/socket.ts | 161 +++++++++++------- apps/desktop/src/utils/webgpu-renderer.ts | 46 ++++- crates/recording/src/feeds/camera.rs | 1 + 8 files changed, 297 insertions(+), 155 deletions(-) diff --git a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs index 587de53b43d..78527f481d8 100644 --- a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs +++ b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs @@ -220,7 +220,8 @@ async fn main() { }; let (frame_watch_tx, frame_watch_rx) = watch::channel(None); - let (ws_port, ws_shutdown_token) = create_watch_frame_ws(frame_watch_rx).await; + let (ws_port, ws_shutdown_token) = + create_watch_frame_ws(frame_watch_rx, Default::default()).await; println!("DISPLAY_WS_URL=ws://127.0.0.1:{ws_port}"); println!( @@ -248,7 +249,7 @@ async fn main() { let ws_frame = match output { EditorFrameOutput::Nv12(frame) => { let ws_format = match frame.format { - GpuOutputFormat::Nv12 => WSFrameFormat::Nv12, + GpuOutputFormat::Nv12 => WSFrameFormat::Nv12 { full_range: false }, GpuOutputFormat::Rgba => WSFrameFormat::Rgba, }; let metadata_bytes = match frame.format { @@ -292,8 +293,6 @@ async fn main() { let renderer = match Renderer::spawn_with_telemetry( render_constants.clone(), frame_cb, - &recording_meta, - meta.as_ref(), layers_rx, Some(telemetry.clone()), ) { diff --git a/apps/desktop/src-tauri/src/editor_window.rs b/apps/desktop/src-tauri/src/editor_window.rs index 63bea17cdcc..7e2620c1ff4 100644 --- a/apps/desktop/src-tauri/src/editor_window.rs +++ b/apps/desktop/src-tauri/src/editor_window.rs @@ -35,7 +35,7 @@ async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult { let ws_frame = match output { cap_editor::EditorFrameOutput::Nv12(frame) => { let ws_format = match frame.format { - GpuOutputFormat::Nv12 => WSFrameFormat::Nv12, + GpuOutputFormat::Nv12 => WSFrameFormat::Nv12 { full_range: false }, GpuOutputFormat::Rgba => WSFrameFormat::Rgba, }; WSFrame { @@ -353,7 +353,9 @@ impl EditorInstances { let ws_frame = match output { cap_editor::EditorFrameOutput::Nv12(frame) => { let ws_format = match frame.format { - GpuOutputFormat::Nv12 => WSFrameFormat::Nv12, + GpuOutputFormat::Nv12 => { + WSFrameFormat::Nv12 { full_range: false } + } GpuOutputFormat::Rgba => WSFrameFormat::Rgba, }; WSFrame { diff --git a/apps/desktop/src-tauri/src/fake_window.rs b/apps/desktop/src-tauri/src/fake_window.rs index 70634dfe6c7..773d882fe82 100644 --- a/apps/desktop/src-tauri/src/fake_window.rs +++ b/apps/desktop/src-tauri/src/fake_window.rs @@ -469,6 +469,23 @@ pub fn spawn_fake_window_listener(app: AppHandle, window: WebviewWindow) { }); } +pub fn cancel_fake_window_listener(app: &AppHandle, label: &str) { + if let Some(listeners) = app.try_state::() { + listeners.cancel(label); + } +} + +pub fn cancel_all_fake_window_listeners(app: &AppHandle) { + if let Some(listeners) = app.try_state::() { + listeners.cancel_all(); + } +} + +pub fn init(app: &AppHandle) { + app.manage(FakeWindowBounds(Default::default())); + app.manage(FakeWindowListeners::default()); +} + #[cfg(test)] mod tests { use super::*; @@ -586,20 +603,3 @@ mod tests { )); } } - -pub fn cancel_fake_window_listener(app: &AppHandle, label: &str) { - if let Some(listeners) = app.try_state::() { - listeners.cancel(label); - } -} - -pub fn cancel_all_fake_window_listeners(app: &AppHandle) { - if let Some(listeners) = app.try_state::() { - listeners.cancel_all(); - } -} - -pub fn init(app: &AppHandle) { - app.manage(FakeWindowBounds(Default::default())); - app.manage(FakeWindowListeners::default()); -} diff --git a/apps/desktop/src-tauri/src/frame_ws.rs b/apps/desktop/src-tauri/src/frame_ws.rs index 0e6d7a83478..8542707ff68 100644 --- a/apps/desktop/src-tauri/src/frame_ws.rs +++ b/apps/desktop/src-tauri/src/frame_ws.rs @@ -1,21 +1,12 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::{broadcast, watch}; use tokio_util::sync::CancellationToken; -static TOTAL_BYTES_SENT: AtomicU64 = AtomicU64::new(0); -static TOTAL_FRAMES_SENT: AtomicU32 = AtomicU32::new(0); -static LAST_LOG_TIME: AtomicU64 = AtomicU64::new(0); -static TOTAL_PACK_NS: AtomicU64 = AtomicU64::new(0); -static MAX_PACK_NS: AtomicU64 = AtomicU64::new(0); -static TOTAL_SEND_NS: AtomicU64 = AtomicU64::new(0); -static MAX_SEND_NS: AtomicU64 = AtomicU64::new(0); -static TOTAL_CREATED_TO_SENT_NS: AtomicU64 = AtomicU64::new(0); -static MAX_CREATED_TO_SENT_NS: AtomicU64 = AtomicU64::new(0); - -const NV12_FORMAT_MAGIC: u32 = 0x4e563132; +const NV12_VIDEO_FORMAT_MAGIC: u32 = 0x4e563132; +const NV12_FULL_FORMAT_MAGIC: u32 = 0x4e563146; fn pack_frame_data( mut data: Vec, @@ -37,7 +28,7 @@ fn pack_frame_data( #[derive(Clone, Copy, PartialEq, Eq)] pub enum WSFrameFormat { Rgba, - Nv12, + Nv12 { full_range: bool }, } #[derive(Clone)] @@ -55,20 +46,25 @@ pub struct WSFrame { fn pack_ws_frame(frame: &WSFrame) -> Vec { let metadata_size = match frame.format { - WSFrameFormat::Nv12 => 28usize, + WSFrameFormat::Nv12 { .. } => 28usize, WSFrameFormat::Rgba => 24, }; let mut buf = Vec::with_capacity(frame.data.len() + metadata_size); buf.extend_from_slice(&frame.data); match frame.format { - WSFrameFormat::Nv12 => { + WSFrameFormat::Nv12 { full_range } => { buf.extend_from_slice(&frame.stride.to_le_bytes()); buf.extend_from_slice(&frame.height.to_le_bytes()); buf.extend_from_slice(&frame.width.to_le_bytes()); buf.extend_from_slice(&frame.frame_number.to_le_bytes()); buf.extend_from_slice(&frame.target_time_ns.to_le_bytes()); - buf.extend_from_slice(&NV12_FORMAT_MAGIC.to_le_bytes()); + let magic = if full_range { + NV12_FULL_FORMAT_MAGIC + } else { + NV12_VIDEO_FORMAT_MAGIC + }; + buf.extend_from_slice(&magic.to_le_bytes()); } WSFrameFormat::Rgba => { buf.extend_from_slice(&frame.stride.to_le_bytes()); @@ -86,23 +82,64 @@ fn duration_ns(duration: std::time::Duration) -> u64 { duration.as_nanos().min(u128::from(u64::MAX)) as u64 } -fn record_ws_frame_stats( - packed_len: usize, - pack_duration: std::time::Duration, - send_duration: std::time::Duration, - created_to_sent: std::time::Duration, -) { - TOTAL_BYTES_SENT.fetch_add(packed_len as u64, Ordering::Relaxed); - TOTAL_FRAMES_SENT.fetch_add(1, Ordering::Relaxed); - let pack_ns = duration_ns(pack_duration); - let send_ns = duration_ns(send_duration); - let created_to_sent_ns = duration_ns(created_to_sent); - TOTAL_PACK_NS.fetch_add(pack_ns, Ordering::Relaxed); - MAX_PACK_NS.fetch_max(pack_ns, Ordering::Relaxed); - TOTAL_SEND_NS.fetch_add(send_ns, Ordering::Relaxed); - MAX_SEND_NS.fetch_max(send_ns, Ordering::Relaxed); - TOTAL_CREATED_TO_SENT_NS.fetch_add(created_to_sent_ns, Ordering::Relaxed); - MAX_CREATED_TO_SENT_NS.fetch_max(created_to_sent_ns, Ordering::Relaxed); +#[derive(Default)] +struct WsFrameStats { + total_bytes_sent: u64, + total_frames_sent: u32, + last_log_time_ms: u64, + total_pack_ns: u64, + max_pack_ns: u64, + total_send_ns: u64, + max_send_ns: u64, + total_created_to_sent_ns: u64, + max_created_to_sent_ns: u64, +} + +impl WsFrameStats { + fn record( + &mut self, + packed_len: usize, + pack_duration: std::time::Duration, + send_duration: std::time::Duration, + created_to_sent: std::time::Duration, + ) { + self.total_bytes_sent += packed_len as u64; + self.total_frames_sent += 1; + let pack_ns = duration_ns(pack_duration); + let send_ns = duration_ns(send_duration); + let created_to_sent_ns = duration_ns(created_to_sent); + self.total_pack_ns += pack_ns; + self.max_pack_ns = self.max_pack_ns.max(pack_ns); + self.total_send_ns += send_ns; + self.max_send_ns = self.max_send_ns.max(send_ns); + self.total_created_to_sent_ns += created_to_sent_ns; + self.max_created_to_sent_ns = self.max_created_to_sent_ns.max(created_to_sent_ns); + } + + fn reset_window(&mut self, now_ms: u64) -> WsFrameStatsWindow { + self.last_log_time_ms = now_ms; + WsFrameStatsWindow { + total_bytes_sent: std::mem::take(&mut self.total_bytes_sent), + total_frames_sent: std::mem::take(&mut self.total_frames_sent), + total_pack_ns: std::mem::take(&mut self.total_pack_ns), + max_pack_ns: std::mem::take(&mut self.max_pack_ns), + total_send_ns: std::mem::take(&mut self.total_send_ns), + max_send_ns: std::mem::take(&mut self.max_send_ns), + total_created_to_sent_ns: std::mem::take(&mut self.total_created_to_sent_ns), + max_created_to_sent_ns: std::mem::take(&mut self.max_created_to_sent_ns), + } + } +} + +struct WsFrameStatsWindow { + total_bytes_sent: u64, + total_frames_sent: u32, + total_pack_ns: u64, + max_pack_ns: u64, + total_send_ns: u64, + max_send_ns: u64, + total_created_to_sent_ns: u64, + max_created_to_sent_ns: u64, } struct SubscriberCountGuard(Arc); @@ -146,6 +183,7 @@ pub async fn create_watch_frame_ws( ) { tracing::info!("Socket connection established"); let now = std::time::Instant::now(); + let mut stats = WsFrameStats::default(); subscribers.fetch_add(1, Ordering::AcqRel); let _subscriber_guard = SubscriberCountGuard(subscribers); @@ -184,7 +222,8 @@ pub async fn create_watch_frame_ws( let width = frame.width; let height = frame.height; let format_label = match frame.format { - WSFrameFormat::Nv12 => "NV12", + WSFrameFormat::Nv12 { full_range: false } => "NV12", + WSFrameFormat::Nv12 { full_range: true } => "NV12-full", WSFrameFormat::Rgba => "RGBA", }; @@ -197,7 +236,7 @@ pub async fn create_watch_frame_ws( match socket.send(Message::Binary(packed)).await { Ok(()) => { let send_duration = send_start.elapsed(); - record_ws_frame_stats( + stats.record( packed_len, pack_duration, send_duration, @@ -207,31 +246,20 @@ pub async fn create_watch_frame_ws( .duration_since(UNIX_EPOCH) .map(|duration| duration.as_millis() as u64) .unwrap_or_default(); - let last_log = LAST_LOG_TIME.load(Ordering::Relaxed); - if now_ms - last_log > 2000 { - LAST_LOG_TIME.store(now_ms, Ordering::Relaxed); - let total_bytes = TOTAL_BYTES_SENT.swap(0, Ordering::Relaxed); - let total_frames = TOTAL_FRAMES_SENT.swap(0, Ordering::Relaxed); - let total_pack_ns = TOTAL_PACK_NS.swap(0, Ordering::Relaxed); - let max_pack_ns = MAX_PACK_NS.swap(0, Ordering::Relaxed); - let total_send_ns = TOTAL_SEND_NS.swap(0, Ordering::Relaxed); - let max_send_ns = MAX_SEND_NS.swap(0, Ordering::Relaxed); - let total_created_to_sent_ns = - TOTAL_CREATED_TO_SENT_NS.swap(0, Ordering::Relaxed); - let max_created_to_sent_ns = - MAX_CREATED_TO_SENT_NS.swap(0, Ordering::Relaxed); - let frames = total_frames.max(1) as f64; - let mb_per_sec = total_bytes as f64 / 1_000_000.0 / 2.0; + if now_ms.saturating_sub(stats.last_log_time_ms) > 2000 { + let window = stats.reset_window(now_ms); + let frames = window.total_frames_sent.max(1) as f64; + let mb_per_sec = window.total_bytes_sent as f64 / 1_000_000.0 / 2.0; tracing::info!( - fps = total_frames / 2, + fps = window.total_frames_sent / 2, mb_per_sec = format!("{:.1}", mb_per_sec), - avg_kb = format!("{:.1}", (total_bytes as f64 / total_frames.max(1) as f64) / 1024.0), - pack_avg_ms = format!("{:.3}", total_pack_ns as f64 / frames / 1_000_000.0), - pack_max_ms = format!("{:.3}", max_pack_ns as f64 / 1_000_000.0), - send_avg_ms = format!("{:.3}", total_send_ns as f64 / frames / 1_000_000.0), - send_max_ms = format!("{:.3}", max_send_ns as f64 / 1_000_000.0), - created_to_sent_avg_ms = format!("{:.3}", total_created_to_sent_ns as f64 / frames / 1_000_000.0), - created_to_sent_max_ms = format!("{:.3}", max_created_to_sent_ns as f64 / 1_000_000.0), + avg_kb = format!("{:.1}", (window.total_bytes_sent as f64 / window.total_frames_sent.max(1) as f64) / 1024.0), + pack_avg_ms = format!("{:.3}", window.total_pack_ns as f64 / frames / 1_000_000.0), + pack_max_ms = format!("{:.3}", window.max_pack_ns as f64 / 1_000_000.0), + send_avg_ms = format!("{:.3}", window.total_send_ns as f64 / frames / 1_000_000.0), + send_max_ms = format!("{:.3}", window.max_send_ns as f64 / 1_000_000.0), + created_to_sent_avg_ms = format!("{:.3}", window.total_created_to_sent_ns as f64 / frames / 1_000_000.0), + created_to_sent_max_ms = format!("{:.3}", window.max_created_to_sent_ns as f64 / 1_000_000.0), dims = format!("{}x{}", width, height), format = format_label, "WS frame stats" diff --git a/apps/desktop/src/utils/frame-worker.ts b/apps/desktop/src/utils/frame-worker.ts index 2be2376cbe1..3e3cd042d74 100644 --- a/apps/desktop/src/utils/frame-worker.ts +++ b/apps/desktop/src/utils/frame-worker.ts @@ -128,6 +128,7 @@ interface PendingFrameWebGPUNv12 { width: number; height: number; yStride: number; + fullRange: boolean; timing: FrameTiming; releaseCallback?: () => void; } @@ -205,6 +206,7 @@ interface FrameMetadataNv12 { width: number; height: number; yStride: number; + fullRange: boolean; frameNumber: number; targetTimeNs: bigint; ySize: number; @@ -214,7 +216,8 @@ interface FrameMetadataNv12 { type FrameMetadata = FrameMetadataRgba | FrameMetadataNv12; -const NV12_MAGIC = 0x4e563132; +const NV12_VIDEO_MAGIC = 0x4e563132; +const NV12_FULL_MAGIC = 0x4e563146; function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null { if (bytes.byteLength < 24) return null; @@ -224,7 +227,7 @@ function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null { const formatView = new DataView(bytes.buffer, formatOffset, 4); const formatFlag = formatView.getUint32(0, true); - if (formatFlag === NV12_MAGIC) { + if (formatFlag === NV12_VIDEO_MAGIC || formatFlag === NV12_FULL_MAGIC) { const metadataOffset = bytes.byteOffset + bytes.byteLength - 28; const meta = new DataView(bytes.buffer, metadataOffset, 28); const yStride = meta.getUint32(0, true); @@ -248,6 +251,7 @@ function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null { width, height, yStride, + fullRange: formatFlag === NV12_FULL_MAGIC, frameNumber, targetTimeNs, ySize, @@ -297,6 +301,7 @@ function convertNv12ToRgba( width: number, height: number, yStride: number, + fullRange = false, ): Uint8ClampedArray { const rgbaSize = width * height * 4; if (!nv12ConversionBuffer || nv12ConversionBufferSize < rgbaSize) { @@ -322,11 +327,20 @@ function convertNv12ToRgba( const d = u; const e = v; - const y0 = yPlane[yRowOffset + col] - 16; - const c0 = 298 * y0; - let r = (c0 + 409 * e + 128) >> 8; - let g = (c0 - 100 * d - 208 * e + 128) >> 8; - let b = (c0 + 516 * d + 128) >> 8; + const y0 = yPlane[yRowOffset + col]; + let r: number; + let g: number; + let b: number; + if (fullRange) { + r = y0 + ((359 * e + 128) >> 8); + g = y0 - ((88 * d + 183 * e + 128) >> 8); + b = y0 + ((454 * d + 128) >> 8); + } else { + const c0 = 298 * (y0 - 16); + r = (c0 + 409 * e + 128) >> 8; + g = (c0 - 100 * d - 208 * e + 128) >> 8; + b = (c0 + 516 * d + 128) >> 8; + } r = r < 0 ? 0 : r > 255 ? 255 : r; g = g < 0 ? 0 : g > 255 ? 255 : g; @@ -340,11 +354,20 @@ function convertNv12ToRgba( const nextCol = col + 1; if (nextCol < width) { - const y1 = yPlane[yRowOffset + nextCol] - 16; - const c1 = 298 * y1; - let nextR = (c1 + 409 * e + 128) >> 8; - let nextG = (c1 - 100 * d - 208 * e + 128) >> 8; - let nextB = (c1 + 516 * d + 128) >> 8; + const y1 = yPlane[yRowOffset + nextCol]; + let nextR: number; + let nextG: number; + let nextB: number; + if (fullRange) { + nextR = y1 + ((359 * e + 128) >> 8); + nextG = y1 - ((88 * d + 183 * e + 128) >> 8); + nextB = y1 + ((454 * d + 128) >> 8); + } else { + const c1 = 298 * (y1 - 16); + nextR = (c1 + 409 * e + 128) >> 8; + nextG = (c1 - 100 * d - 208 * e + 128) >> 8; + nextB = (c1 + 516 * d + 128) >> 8; + } nextR = nextR < 0 ? 0 : nextR > 255 ? 255 : nextR; nextG = nextG < 0 ? 0 : nextG > 255 ? 255 : nextG; @@ -407,6 +430,7 @@ function renderBorrowedWebGPU(bytes: Uint8Array, release: () => void): boolean { width, height, meta.yStride, + meta.fullRange, ); release(); } else { @@ -489,6 +513,7 @@ function queueFrameFromBytes( width, height, yStride: meta.yStride, + fullRange: meta.fullRange, timing, releaseCallback, }); @@ -664,6 +689,7 @@ function renderLoop() { frame.width, frame.height, frame.yStride, + frame.fullRange, ); } else { const expectedRowBytes = frame.width * 4; @@ -735,6 +761,7 @@ function renderLoop() { frame.width, frame.height, frame.yStride, + frame.fullRange, ); } else { renderFrameWebGPU( @@ -996,6 +1023,7 @@ function processFrameBytesSync( width, height, yStride: meta.yStride, + fullRange: meta.fullRange, timing, releaseCallback, }); @@ -1035,6 +1063,7 @@ function processFrameBytesSync( width, height, meta.yStride, + meta.fullRange, ); } else { const frameData = new Uint8ClampedArray( diff --git a/apps/desktop/src/utils/socket.ts b/apps/desktop/src/utils/socket.ts index 46af09bbea7..11448ad13c2 100644 --- a/apps/desktop/src/utils/socket.ts +++ b/apps/desktop/src/utils/socket.ts @@ -30,6 +30,34 @@ const FRAME_BUFFER_CONFIG: SharedFrameBufferConfig = { let mainThreadNv12Buffer: Uint8ClampedArray | null = null; let mainThreadNv12BufferSize = 0; +const NV12_VIDEO_MAGIC = 0x4e563132; +const NV12_FULL_MAGIC = 0x4e563146; +const NV12_METADATA_SIZE = 28; + +type Nv12Metadata = { + yStride: number; + height: number; + width: number; + fullRange: boolean; +}; + +function readNv12Metadata(buffer: ArrayBuffer): Nv12Metadata | null { + if (buffer.byteLength < NV12_METADATA_SIZE) return null; + + const formatCheck = new DataView(buffer, buffer.byteLength - 4, 4); + const magic = formatCheck.getUint32(0, true); + if (magic !== NV12_VIDEO_MAGIC && magic !== NV12_FULL_MAGIC) return null; + + const metadataOffset = buffer.byteLength - NV12_METADATA_SIZE; + const meta = new DataView(buffer, metadataOffset, NV12_METADATA_SIZE); + return { + yStride: meta.getUint32(0, true), + height: meta.getUint32(4, true), + width: meta.getUint32(8, true), + fullRange: magic === NV12_FULL_MAGIC, + }; +} + export type FpsStats = { fps: number; renderFps: number; @@ -67,6 +95,7 @@ function convertNv12ToRgbaMainThread( width: number, height: number, yStride: number, + fullRange = false, ): Uint8ClampedArray { const rgbaSize = width * height * 4; if (!mainThreadNv12Buffer || mainThreadNv12BufferSize < rgbaSize) { @@ -92,11 +121,20 @@ function convertNv12ToRgbaMainThread( const d = u; const e = v; - const y0 = yPlane[yRowOffset + col] - 16; - const c0 = 298 * y0; - let r = (c0 + 409 * e + 128) >> 8; - let g = (c0 - 100 * d - 208 * e + 128) >> 8; - let b = (c0 + 516 * d + 128) >> 8; + const y0 = yPlane[yRowOffset + col]; + let r: number; + let g: number; + let b: number; + if (fullRange) { + r = y0 + ((359 * e + 128) >> 8); + g = y0 - ((88 * d + 183 * e + 128) >> 8); + b = y0 + ((454 * d + 128) >> 8); + } else { + const c0 = 298 * (y0 - 16); + r = (c0 + 409 * e + 128) >> 8; + g = (c0 - 100 * d - 208 * e + 128) >> 8; + b = (c0 + 516 * d + 128) >> 8; + } r = r < 0 ? 0 : r > 255 ? 255 : r; g = g < 0 ? 0 : g > 255 ? 255 : g; @@ -110,11 +148,20 @@ function convertNv12ToRgbaMainThread( const nextCol = col + 1; if (nextCol < width) { - const y1 = yPlane[yRowOffset + nextCol] - 16; - const c1 = 298 * y1; - let nextR = (c1 + 409 * e + 128) >> 8; - let nextG = (c1 - 100 * d - 208 * e + 128) >> 8; - let nextB = (c1 + 516 * d + 128) >> 8; + const y1 = yPlane[yRowOffset + nextCol]; + let nextR: number; + let nextG: number; + let nextB: number; + if (fullRange) { + nextR = y1 + ((359 * e + 128) >> 8); + nextG = y1 - ((88 * d + 183 * e + 128) >> 8); + nextB = y1 + ((454 * d + 128) >> 8); + } else { + const c1 = 298 * (y1 - 16); + nextR = (c1 + 409 * e + 128) >> 8; + nextG = (c1 - 100 * d - 208 * e + 128) >> 8; + nextB = (c1 + 516 * d + 128) >> 8; + } nextR = nextR < 0 ? 0 : nextR > 255 ? 255 : nextR; nextG = nextG < 0 ? 0 : nextG > 255 ? 255 : nextG; @@ -278,6 +325,7 @@ export function createImageDataWS( height: number; yStride: number; isNv12: boolean; + nv12FullRange: boolean; } | null = null; function storeRenderedFrame( @@ -286,6 +334,7 @@ export function createImageDataWS( height: number, yStride: number, isNv12: boolean, + nv12FullRange = false, ) { lastRenderedFrameData = { data: frameData, @@ -293,6 +342,7 @@ export function createImageDataWS( height, yStride, isNv12, + nv12FullRange, }; if (!hasRenderedFrame()) { setHasRenderedFrame(true); @@ -381,17 +431,10 @@ export function createImageDataWS( const { buffer, receivedAt } = pendingNv12Frame; pendingNv12Frame = null; - const NV12_MAGIC = 0x4e563132; - if (buffer.byteLength < 28) return; + const metadata = readNv12Metadata(buffer); + if (!metadata) return; - const formatCheck = new DataView(buffer, buffer.byteLength - 4, 4); - if (formatCheck.getUint32(0, true) !== NV12_MAGIC) return; - - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const yStride = meta.getUint32(0, true); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { yStride, height, width, fullRange } = metadata; if (width > 0 && height > 0) { const ySize = yStride * height; @@ -412,9 +455,10 @@ export function createImageDataWS( width, height, yStride, + fullRange, ); - storeRenderedFrame(frameData, width, height, yStride, true); + storeRenderedFrame(frameData, width, height, yStride, true, fullRange); recordRender( performance.now() - renderStart, "webgpu", @@ -495,6 +539,7 @@ export function createImageDataWS( width: number, height: number, yStride: number, + fullRange: boolean, receivedAt?: number, ) { if (!directCanvas || !directCtx) return; @@ -506,7 +551,13 @@ export function createImageDataWS( directCanvas.height = height; } - const rgba = convertNv12ToRgbaMainThread(frameData, width, height, yStride); + const rgba = convertNv12ToRgbaMainThread( + frameData, + width, + height, + yStride, + fullRange, + ); if ( !cachedDirectImageData || @@ -520,7 +571,7 @@ export function createImageDataWS( cachedDirectImageData.data.set(rgba); directCtx.putImageData(cachedDirectImageData, 0, 0); - storeRenderedFrame(frameData, width, height, yStride, true); + storeRenderedFrame(frameData, width, height, yStride, true, fullRange); recordRender( performance.now() - renderStart, "canvas2d", @@ -536,17 +587,10 @@ export function createImageDataWS( const { buffer, receivedAt } = pendingNv12Frame; pendingNv12Frame = null; - const NV12_MAGIC = 0x4e563132; - if (buffer.byteLength < 28) return; + const metadata = readNv12Metadata(buffer); + if (!metadata) return; - const formatCheck = new DataView(buffer, buffer.byteLength - 4, 4); - if (formatCheck.getUint32(0, true) !== NV12_MAGIC) return; - - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const yStride = meta.getUint32(0, true); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { yStride, height, width, fullRange } = metadata; if (width > 0 && height > 0) { const ySize = yStride * height; @@ -554,7 +598,14 @@ export function createImageDataWS( const totalSize = ySize + uvSize; const frameData = new Uint8ClampedArray(buffer, 0, totalSize); - renderNv12FrameCanvas2D(frameData, width, height, yStride, receivedAt); + renderNv12FrameCanvas2D( + frameData, + width, + height, + yStride, + fullRange, + receivedAt, + ); } } @@ -814,10 +865,17 @@ export function createImageDataWS( if (!lastRenderedFrameData) { return null; } - const { data, width, height, yStride, isNv12 } = lastRenderedFrameData; + const { data, width, height, yStride, isNv12, nv12FullRange } = + lastRenderedFrameData; let imageData: ImageData; if (isNv12) { - const rgba = convertNv12ToRgbaMainThread(data, width, height, yStride); + const rgba = convertNv12ToRgbaMainThread( + data, + width, + height, + yStride, + nv12FullRange, + ); imageData = new ImageData(new Uint8ClampedArray(rgba), width, height); } else { const expectedRowBytes = width * 4; @@ -1060,8 +1118,6 @@ export function createImageDataWS( globalFpsStatsGetter = getLocalFpsStats; (globalThis as Record).__capFpsStats = getLocalFpsStats; - const NV12_MAGIC = 0x4e563132; - ws.binaryType = "arraybuffer"; ws.onmessage = (event) => { const buffer = event.data as ArrayBuffer; @@ -1071,11 +1127,7 @@ export function createImageDataWS( } totalBytesReceived += buffer.byteLength; - let isNv12Format = false; - if (buffer.byteLength >= 28) { - const formatCheck = new DataView(buffer, buffer.byteLength - 4, 4); - isNv12Format = formatCheck.getUint32(0, true) === NV12_MAGIC; - } + const nv12Metadata = readNv12Metadata(buffer); if (lastFrameTime > 0) { const delta = now - lastFrameTime; @@ -1086,12 +1138,9 @@ export function createImageDataWS( } lastFrameTime = now; - if (isNv12Format) { + if (nv12Metadata) { if (mainThreadWebGPU && directCanvas) { - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { height, width } = nv12Metadata; if (width > 0 && height > 0) { if (directCanvas.width !== width || directCanvas.height !== height) { @@ -1106,10 +1155,7 @@ export function createImageDataWS( if (mainThreadWebGPUInitializing || !directCanvas) { pendingNv12Frame = { buffer, receivedAt: now }; - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { height, width } = nv12Metadata; if (width > 0 && height > 0) { onmessage({ width, height }); } @@ -1135,11 +1181,7 @@ export function createImageDataWS( } } - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const yStride = meta.getUint32(0, true); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { yStride, height, width } = nv12Metadata; if (width > 0 && height > 0) { const ySize = yStride * height; @@ -1164,10 +1206,7 @@ export function createImageDataWS( } pendingNv12Frame = { buffer, receivedAt: now }; - const metadataOffset = buffer.byteLength - 28; - const meta = new DataView(buffer, metadataOffset, 28); - const height = meta.getUint32(4, true); - const width = meta.getUint32(8, true); + const { height, width } = nv12Metadata; if (width > 0 && height > 0) { onmessage({ width, height }); } diff --git a/apps/desktop/src/utils/webgpu-renderer.ts b/apps/desktop/src/utils/webgpu-renderer.ts index 0d48775a253..3167ddd2e1d 100644 --- a/apps/desktop/src/utils/webgpu-renderer.ts +++ b/apps/desktop/src/utils/webgpu-renderer.ts @@ -56,11 +56,33 @@ fn fs(@location(0) texCoord: vec2f) -> @location(0) vec4f { } `; +const NV12_FULL_FRAGMENT_SHADER = ` +@group(0) @binding(0) var frameSampler: sampler; +@group(0) @binding(1) var yTexture: texture_2d; +@group(0) @binding(2) var uvTexture: texture_2d; + +@fragment +fn fs(@location(0) texCoord: vec2f) -> @location(0) vec4f { + let y = textureSample(yTexture, frameSampler, texCoord).r; + let uv = textureSample(uvTexture, frameSampler, texCoord).rg; + + let u = uv.r - 0.5; + let v = uv.g - 0.5; + + let r = clamp(y + 1.402 * v, 0.0, 1.0); + let g = clamp(y - 0.344136 * u - 0.714136 * v, 0.0, 1.0); + let b = clamp(y + 1.772 * u, 0.0, 1.0); + + return vec4f(r, g, b, 1.0); +} +`; + export interface WebGPURenderer { device: GPUDevice; context: GPUCanvasContext; pipeline: GPURenderPipeline; nv12Pipeline: GPURenderPipeline; + nv12FullPipeline: GPURenderPipeline; sampler: GPUSampler; frameTexture: GPUTexture | null; bindGroup: GPUBindGroup | null; @@ -200,6 +222,9 @@ export async function initWebGPU( const nv12FragmentModule = device.createShaderModule({ code: NV12_FRAGMENT_SHADER, }); + const nv12FullFragmentModule = device.createShaderModule({ + code: NV12_FULL_FRAGMENT_SHADER, + }); const pipeline = device.createRenderPipeline({ layout: pipelineLayout, @@ -233,6 +258,22 @@ export async function initWebGPU( }, }); + const nv12FullPipeline = device.createRenderPipeline({ + layout: nv12PipelineLayout, + vertex: { + module: vertexModule, + entryPoint: "vs", + }, + fragment: { + module: nv12FullFragmentModule, + entryPoint: "fs", + targets: [{ format }], + }, + primitive: { + topology: "triangle-list", + }, + }); + const sampler = device.createSampler({ magFilter: "linear", minFilter: "linear", @@ -245,6 +286,7 @@ export async function initWebGPU( context, pipeline, nv12Pipeline, + nv12FullPipeline, sampler, frameTexture: null, bindGroup: null, @@ -370,6 +412,7 @@ export function renderNv12FrameWebGPU( width: number, height: number, yStride: number, + fullRange = false, ): WebGPURenderTiming { const totalStart = performance.now(); let resizeMs = 0; @@ -380,6 +423,7 @@ export function renderNv12FrameWebGPU( device, context, nv12Pipeline, + nv12FullPipeline, sampler, nv12BindGroupLayout, canvas, @@ -482,7 +526,7 @@ export function renderNv12FrameWebGPU( ], }); - pass.setPipeline(nv12Pipeline); + pass.setPipeline(fullRange ? nv12FullPipeline : nv12Pipeline); pass.setBindGroup(0, renderer.nv12BindGroup); pass.draw(3); pass.end(); diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 67528cd2dc4..858f4820d15 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -975,6 +975,7 @@ async fn setup_camera( settings: Option, recipient: Recipient, _native_recipient: Recipient, + _ffmpeg_sender_count: Arc, _native_sender_count: Arc, ) -> Result { let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?; From 02a6aa942575f739c71f322e473ab19bd39c1207 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 10 Jun 2026 20:40:05 +0100 Subject: [PATCH 24/24] fix: validate nv12 frame metadata --- apps/desktop/src/utils/frame-worker.ts | 4 ++-- apps/desktop/src/utils/socket.ts | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/utils/frame-worker.ts b/apps/desktop/src/utils/frame-worker.ts index 3e3cd042d74..89b77300629 100644 --- a/apps/desktop/src/utils/frame-worker.ts +++ b/apps/desktop/src/utils/frame-worker.ts @@ -236,10 +236,10 @@ function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null { const frameNumber = meta.getUint32(12, true); const targetTimeNs = meta.getBigUint64(16, true); - if (!width || !height) return null; + if (!width || !height || !yStride) return null; const ySize = yStride * height; - const uvSize = yStride * (height / 2); + const uvSize = yStride * Math.floor(height / 2); const totalSize = ySize + uvSize; if (bytes.byteLength - 28 < totalSize) { diff --git a/apps/desktop/src/utils/socket.ts b/apps/desktop/src/utils/socket.ts index 11448ad13c2..0b191512976 100644 --- a/apps/desktop/src/utils/socket.ts +++ b/apps/desktop/src/utils/socket.ts @@ -50,10 +50,26 @@ function readNv12Metadata(buffer: ArrayBuffer): Nv12Metadata | null { const metadataOffset = buffer.byteLength - NV12_METADATA_SIZE; const meta = new DataView(buffer, metadataOffset, NV12_METADATA_SIZE); + const yStride = meta.getUint32(0, true); + const height = meta.getUint32(4, true); + const width = meta.getUint32(8, true); + + const ySize = yStride * height; + const uvSize = yStride * Math.floor(height / 2); + const totalSize = ySize + uvSize; + if ( + yStride === 0 || + height === 0 || + width === 0 || + buffer.byteLength - NV12_METADATA_SIZE < totalSize + ) { + return null; + } + return { - yStride: meta.getUint32(0, true), - height: meta.getUint32(4, true), - width: meta.getUint32(8, true), + yStride, + height, + width, fullRange: magic === NV12_FULL_MAGIC, }; }