Desktop: Use dirty rects for UI texture uploads (#4305)

desktop/src/cef.rs

@@ -21,9 +21,8 @@ use std::sync::{Arc, Mutex};
 use std::time::Instant;
 
 use crate::event::{AppEvent, AppEventScheduler};
-use crate::render::FrameBufferRef;
 use crate::window::Cursor;
-use crate::wrapper::{WgpuContext, deserialize_editor_message};
+use crate::wrapper::deserialize_editor_message;
 
 mod consts;
 mod context;
@@ -32,17 +31,14 @@ mod internal;
 mod ipc;
 mod platform;
 mod utility;
-
+mod view;
-#[cfg(feature = "accelerated_paint")]
-use cef::osr_texture_import::SharedTextureHandle;
 
 pub(crate) use context::{CefContext, CefContextBuilder, InitError};
+pub(crate) use view::View;
 
 pub(crate) trait CefEventHandler: Send + Sync + 'static {
 	fn view_info(&self) -> ViewInfo;
-	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>);
+	fn draw(&self, view: &View);
-	#[cfg(feature = "accelerated_paint")]
-	fn draw_gpu(&self, shared_texture: SharedTextureHandle);
 	fn load_resource(&self, path: PathBuf) -> Option<Resource>;
 	fn cursor_change(&self, cursor: Cursor);
 	/// Schedule the main event loop to run the CEF event loop after the timeout.
@@ -120,15 +116,13 @@ impl Read for ResourceReader {
 }
 
 pub(crate) struct CefHandler {
-	wgpu_context: WgpuContext,
 	app_event_scheduler: AppEventScheduler,
 	view_info_receiver: Arc<Mutex<ViewInfoReceiver>>,
 }
 
 impl CefHandler {
-	pub(crate) fn new(wgpu_context: WgpuContext, app_event_scheduler: AppEventScheduler, view_info_receiver: Receiver<ViewInfoUpdate>) -> Self {
+	pub(crate) fn new(app_event_scheduler: AppEventScheduler, view_info_receiver: Receiver<ViewInfoUpdate>) -> Self {
 		Self {
-			wgpu_context,
 			app_event_scheduler,
 			view_info_receiver: Arc::new(Mutex::new(ViewInfoReceiver::new(view_info_receiver))),
 		}
@@ -147,55 +141,10 @@ impl CefEventHandler for CefHandler {
 		}
 		*view_info
 	}
-	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>) {
-		let width = frame_buffer.width() as u32;
-		let height = frame_buffer.height() as u32;
-		let texture = self.wgpu_context.device.create_texture(&wgpu::TextureDescriptor {
-			label: Some("CEF Texture"),
-			size: wgpu::Extent3d {
-				width,
-				height,
-				depth_or_array_layers: 1,
-			},
-			mip_level_count: 1,
-			sample_count: 1,
-			dimension: wgpu::TextureDimension::D2,
-			format: wgpu::TextureFormat::Bgra8Unorm,
-			usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
-			view_formats: &[],
-		});
-		self.wgpu_context.queue.write_texture(
-			wgpu::TexelCopyTextureInfo {
-				texture: &texture,
-				mip_level: 0,
-				origin: wgpu::Origin3d::ZERO,
-				aspect: wgpu::TextureAspect::All,
-			},
-			frame_buffer.buffer(),
-			wgpu::TexelCopyBufferLayout {
-				offset: 0,
-				bytes_per_row: Some(4 * width),
-				rows_per_image: Some(height),
-			},
-			wgpu::Extent3d {
-				width,
-				height,
-				depth_or_array_layers: 1,
-			},
-		);
 
-		self.app_event_scheduler.schedule(AppEvent::UiUpdate(texture));
+	fn draw(&self, view: &View) {
-	}
+		if let Some(texture) = view.texture() {
-
+			self.app_event_scheduler.schedule(AppEvent::UiUpdate(texture));
-	#[cfg(feature = "accelerated_paint")]
-	fn draw_gpu(&self, shared_texture: SharedTextureHandle) {
-		match shared_texture.import_texture(&self.wgpu_context.device) {
-			Ok(texture) => {
-				self.app_event_scheduler.schedule(AppEvent::UiUpdate(texture));
-			}
-			Err(e) => {
-				tracing::error!("Failed to import shared texture: {}", e);
-			}
 		}
 	}
 
@@ -279,7 +228,6 @@ impl CefEventHandler for CefHandler {
 		Self: Sized,
 	{
 		Self {
-			wgpu_context: self.wgpu_context.clone(),
 			app_event_scheduler: self.app_event_scheduler.clone(),
 			view_info_receiver: self.view_info_receiver.clone(),
 		}

desktop/src/cef/context/builder.rs

@@ -13,6 +13,7 @@ use crate::cef::consts::{RESOURCE_DOMAIN, RESOURCE_SCHEME};
 use crate::cef::input::InputState;
 use crate::cef::internal::{BrowserProcessAppImpl, BrowserProcessClientImpl, RenderProcessAppImpl, SchemeHandlerFactoryImpl};
 use crate::dirs::TempDir;
+use crate::wrapper::WgpuContext;
 
 pub(crate) struct CefContextBuilder<H: CefEventHandler> {
 	pub(crate) args: Args,
@@ -67,19 +68,19 @@ impl<H: CefEventHandler> CefContextBuilder<H> {
 	}
 
 	#[cfg(target_os = "macos")]
-	pub(crate) fn create(self, event_handler: H, disable_gpu_acceleration: bool) -> Result<impl CefContext, InitError> {
+	pub(crate) fn create(self, event_handler: H, wgpu_context: WgpuContext, disable_gpu_acceleration: bool) -> Result<impl CefContext, InitError> {
 		let instance_dir = TempDir::new().expect("Failed to create temporary directory for CEF instance");
 		let accelerated_paint = accelerated_paint(disable_gpu_acceleration);
 		self.build_inner(&event_handler, instance_dir.as_ref(), accelerated_paint)?;
-		create_browser(event_handler, instance_dir, accelerated_paint)
+		create_browser(event_handler, wgpu_context, instance_dir, accelerated_paint)
 	}
 
 	#[cfg(not(target_os = "macos"))]
-	pub(crate) fn create(self, event_handler: H, disable_gpu_acceleration: bool) -> Result<impl CefContext, InitError> {
+	pub(crate) fn create(self, event_handler: H, wgpu_context: WgpuContext, disable_gpu_acceleration: bool) -> Result<impl CefContext, InitError> {
 		let instance_dir = TempDir::new().expect("Failed to create temporary directory for CEF instance");
 		let accelerated_paint = accelerated_paint(disable_gpu_acceleration);
 		self.build_inner(&event_handler, instance_dir.as_ref(), accelerated_paint)?;
-		super::multithreaded::run_on_ui_thread(move || match create_browser(event_handler, instance_dir, accelerated_paint) {
+		super::multithreaded::run_on_ui_thread(move || match create_browser(event_handler, wgpu_context, instance_dir, accelerated_paint) {
 			Ok(context) => super::multithreaded::CONTEXT.with(|b| *b.borrow_mut() = Some(context)),
 			Err(e) => panic!("Failed to initialize CEF context: {:?}", e),
 		});
@@ -149,8 +150,8 @@ fn platform_settings(instance_dir: &Path) -> Settings {
 	}
 }
 
-fn create_browser<H: CefEventHandler>(event_handler: H, instance_dir: TempDir, accelerated_paint: bool) -> Result<SingleThreadedCefContext, InitError> {
+fn create_browser<H: CefEventHandler>(event_handler: H, wgpu_context: WgpuContext, instance_dir: TempDir, accelerated_paint: bool) -> Result<SingleThreadedCefContext, InitError> {
-	let mut client = Client::new(BrowserProcessClientImpl::new(&event_handler));
+	let mut client = Client::new(BrowserProcessClientImpl::new(&event_handler, wgpu_context));
 
 	let window_info = WindowInfo {
 		windowless_rendering_enabled: 1,

desktop/src/cef/internal/browser_process_client.rs

@@ -4,6 +4,7 @@ use cef::{ContextMenuHandler, DisplayHandler, ImplClient, LifeSpanHandler, LoadH
 
 use crate::cef::CefEventHandler;
 use crate::cef::ipc::{MessageType, UnpackMessage, UnpackedMessage};
+use crate::wrapper::WgpuContext;
 
 use super::context_menu_handler::ContextMenuHandlerImpl;
 use super::display_handler::DisplayHandlerImpl;
@@ -21,12 +22,12 @@ pub(crate) struct BrowserProcessClientImpl<H: CefEventHandler> {
 	request_handler: RequestHandler,
 }
 impl<H: CefEventHandler> BrowserProcessClientImpl<H> {
-	pub(crate) fn new(event_handler: &H) -> Self {
+	pub(crate) fn new(event_handler: &H, wgpu_context: WgpuContext) -> Self {
 		Self {
 			object: std::ptr::null_mut(),
 			event_handler: event_handler.duplicate(),
 			load_handler: LoadHandler::new(LoadHandlerImpl::new(event_handler.duplicate())),
-			render_handler: RenderHandler::new(RenderHandlerImpl::new(event_handler.duplicate())),
+			render_handler: RenderHandler::new(RenderHandlerImpl::new(event_handler.duplicate(), wgpu_context)),
 			display_handler: DisplayHandler::new(DisplayHandlerImpl::new(event_handler.duplicate())),
 			request_handler: RequestHandler::new(RequestHandlerImpl::new()),
 		}

desktop/src/cef/internal/render_handler.rs

@@ -2,18 +2,20 @@ use cef::rc::{Rc, RcImpl};
 use cef::sys::{_cef_render_handler_t, cef_base_ref_counted_t};
 use cef::{Browser, ImplRenderHandler, PaintElementType, Rect, WrapRenderHandler};
 
-use crate::cef::CefEventHandler;
+use crate::cef::{CefEventHandler, View};
-use crate::render::FrameBufferRef;
+use crate::wrapper::WgpuContext;
 
 pub(crate) struct RenderHandlerImpl<H: CefEventHandler> {
 	object: *mut RcImpl<_cef_render_handler_t, Self>,
 	event_handler: H,
+	view: View,
 }
 impl<H: CefEventHandler> RenderHandlerImpl<H> {
-	pub(crate) fn new(event_handler: H) -> Self {
+	pub(crate) fn new(event_handler: H, wgpu_context: WgpuContext) -> Self {
 		Self {
 			object: std::ptr::null_mut(),
 			event_handler,
+			view: View::new(wgpu_context),
 		}
 	}
 }
@@ -31,29 +33,26 @@ impl<H: CefEventHandler> ImplRenderHandler for RenderHandlerImpl<H> {
 		}
 	}
 
-	fn on_paint(&self, _browser: Option<&mut Browser>, _type_: PaintElementType, _dirty_rects: Option<&[Rect]>, buffer: *const u8, width: std::ffi::c_int, height: std::ffi::c_int) {
+	fn on_paint(&self, _browser: Option<&mut Browser>, type_: PaintElementType, dirty_rects: Option<&[Rect]>, buffer: *const u8, width: std::ffi::c_int, height: std::ffi::c_int) {
-		let buffer_size = (width * height * 4) as usize;
-		let buffer_slice = unsafe { std::slice::from_raw_parts(buffer, buffer_size) };
-		let frame_buffer = FrameBufferRef::new(buffer_slice, width as usize, height as usize).expect("Failed to create frame buffer");
-
-		self.event_handler.draw(frame_buffer)
-	}
-
-	#[cfg(feature = "accelerated_paint")]
-	fn on_accelerated_paint(&self, _browser: Option<&mut Browser>, type_: PaintElementType, _dirty_rects: Option<&[Rect]>, info: Option<&cef::AcceleratedPaintInfo>) {
-		use cef::osr_texture_import::SharedTextureHandle;
-
 		if type_ != PaintElementType::default() {
 			return;
 		}
 
-		let shared_handle = SharedTextureHandle::new(info.unwrap());
+		let buffer_size = (width * height * 4) as usize;
-		if let SharedTextureHandle::Unsupported = shared_handle {
+		let buffer_slice = unsafe { std::slice::from_raw_parts(buffer, buffer_size) };
-			tracing::error!("Platform does not support accelerated painting");
+
+		self.view.upload_frame_buffer(buffer_slice, width as u32, height as u32, dirty_rects.unwrap_or(&[]));
+		self.event_handler.draw(&self.view)
+	}
+
+	#[cfg(feature = "accelerated_paint")]
+	fn on_accelerated_paint(&self, _browser: Option<&mut Browser>, type_: PaintElementType, _dirty_rects: Option<&[Rect]>, info: Option<&cef::AcceleratedPaintInfo>) {
+		if type_ != PaintElementType::default() {
 			return;
 		}
 
-		self.event_handler.draw_gpu(shared_handle);
+		self.view.import_shared_texture(info.unwrap());
+		self.event_handler.draw(&self.view)
 	}
 
 	fn get_raw(&self) -> *mut _cef_render_handler_t {
@@ -70,6 +69,7 @@ impl<H: CefEventHandler> Clone for RenderHandlerImpl<H> {
 		Self {
 			object: self.object,
 			event_handler: self.event_handler.duplicate(),
+			view: self.view.clone(),
 		}
 	}
 }

desktop/src/cef/view.rs

@@ -0,0 +1,108 @@
+use cef::Rect;
+use std::sync::{Arc, Mutex};
+
+use crate::wrapper::WgpuContext;
+
+#[derive(Clone)]
+pub(crate) struct View {
+	context: WgpuContext,
+	texture: Arc<Mutex<Option<wgpu::Texture>>>,
+}
+
+impl View {
+	pub(crate) fn new(context: WgpuContext) -> Self {
+		Self {
+			context,
+			texture: Arc::new(Mutex::new(None)),
+		}
+	}
+
+	pub(crate) fn texture(&self) -> Option<wgpu::Texture> {
+		let Ok(texture) = self.texture.lock() else {
+			tracing::error!("Failed to lock view texture");
+			return None;
+		};
+		texture.clone()
+	}
+
+	pub(super) fn upload_frame_buffer(&self, buffer: &[u8], width: u32, height: u32, dirty_rects: &[Rect]) {
+		debug_assert_eq!(buffer.len(), width as usize * height as usize * 4);
+
+		let Ok(mut slot) = self.texture.lock() else {
+			tracing::error!("Failed to lock view texture");
+			return;
+		};
+
+		let needs_new_texture = slot.as_ref().is_none_or(|texture| texture.width() != width || texture.height() != height);
+		if needs_new_texture {
+			*slot = Some(self.context.device.create_texture(&wgpu::TextureDescriptor {
+				label: Some("CEF Texture"),
+				size: wgpu::Extent3d {
+					width,
+					height,
+					depth_or_array_layers: 1,
+				},
+				mip_level_count: 1,
+				sample_count: 1,
+				dimension: wgpu::TextureDimension::D2,
+				format: wgpu::TextureFormat::Bgra8Unorm,
+				usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+				view_formats: &[],
+			}));
+		}
+		let texture = slot.as_ref().expect("Texture was just created");
+
+		let full_frame = [Rect {
+			x: 0,
+			y: 0,
+			width: width as i32,
+			height: height as i32,
+		}];
+		let rects = if needs_new_texture || dirty_rects.is_empty() { &full_frame } else { dirty_rects };
+
+		for rect in rects {
+			let x = (rect.x.max(0) as u32).min(width);
+			let y = (rect.y.max(0) as u32).min(height);
+			let rect_width = (rect.width.max(0) as u32).min(width - x);
+			let rect_height = (rect.height.max(0) as u32).min(height - y);
+			if rect_width == 0 || rect_height == 0 {
+				continue;
+			}
+			self.context.queue.write_texture(
+				wgpu::TexelCopyTextureInfo {
+					texture,
+					mip_level: 0,
+					origin: wgpu::Origin3d { x, y, z: 0 },
+					aspect: wgpu::TextureAspect::All,
+				},
+				buffer,
+				wgpu::TexelCopyBufferLayout {
+					offset: 4 * (y as u64 * width as u64 + x as u64),
+					bytes_per_row: Some(4 * width),
+					rows_per_image: None,
+				},
+				wgpu::Extent3d {
+					width: rect_width,
+					height: rect_height,
+					depth_or_array_layers: 1,
+				},
+			);
+		}
+	}
+
+	#[cfg(feature = "accelerated_paint")]
+	pub(super) fn import_shared_texture(&self, info: &cef::AcceleratedPaintInfo) {
+		let texture = match cef::osr_texture_import::SharedTextureHandle::new(info).import_texture(&self.context.device) {
+			Ok(texture) => texture,
+			Err(e) => {
+				tracing::error!("Failed to import shared texture: {e}");
+				return;
+			}
+		};
+		let Ok(mut slot) = self.texture.lock() else {
+			tracing::error!("Failed to lock view texture");
+			return;
+		};
+		*slot = Some(texture);
+	}
+}

desktop/src/lib.rs

@@ -96,8 +96,8 @@ pub fn start() {
 		println!("UI acceleration is disabled");
 	}
 
-	let cef_handler = cef::CefHandler::new(wgpu_context.clone(), app_event_scheduler.clone(), cef_view_info_receiver);
+	let cef_handler = cef::CefHandler::new(app_event_scheduler.clone(), cef_view_info_receiver);
-	let cef_context = match cef_context_builder.create(cef_handler, prefs.disable_ui_acceleration) {
+	let cef_context = match cef_context_builder.create(cef_handler, wgpu_context.clone(), prefs.disable_ui_acceleration) {
 		Ok(context) => {
 			tracing::info!("CEF initialized successfully");
 			context

desktop/src/render.rs

@@ -1,5 +1,2 @@
-mod frame_buffer_ref;
-pub(crate) use frame_buffer_ref::FrameBufferRef;
-
 mod state;
 pub(crate) use state::{RenderError, RenderState};

desktop/src/render/frame_buffer_ref.rs

@@ -1,53 +0,0 @@
-use thiserror::Error;
-
-pub(crate) struct FrameBufferRef<'a> {
-	buffer: &'a [u8],
-	width: usize,
-	height: usize,
-}
-impl<'a> FrameBufferRef<'a> {
-	pub(crate) fn new(buffer: &'a [u8], width: usize, height: usize) -> Result<Self, FrameBufferError> {
-		let fb = Self { buffer, width, height };
-		fb.validate_size()?;
-		Ok(fb)
-	}
-	pub(crate) fn buffer(&self) -> &[u8] {
-		self.buffer
-	}
-
-	pub(crate) fn width(&self) -> usize {
-		self.width
-	}
-
-	pub(crate) fn height(&self) -> usize {
-		self.height
-	}
-
-	fn validate_size(&self) -> Result<(), FrameBufferError> {
-		if self.buffer.len() != self.width * self.height * 4 {
-			Err(FrameBufferError::InvalidSize {
-				buffer_size: self.buffer.len(),
-				expected_size: self.width * self.height * 4,
-				width: self.width,
-				height: self.height,
-			})
-		} else {
-			Ok(())
-		}
-	}
-}
-impl<'a> std::fmt::Debug for FrameBufferRef<'a> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		f.debug_struct("FrameBuffer")
-			.field("width", &self.width)
-			.field("height", &self.height)
-			.field("len", &self.buffer.len())
-			.finish()
-	}
-}
-
-#[derive(Error, Debug)]
-pub(crate) enum FrameBufferError {
-	#[error("Invalid buffer size {buffer_size}, expected {expected_size} for width {width} multiplied with height {height} multiplied by 4 channels")]
-	InvalidSize { buffer_size: usize, expected_size: usize, width: usize, height: usize },
-}

desktop/src/render/state.rs

@@ -206,8 +206,12 @@ impl RenderState {
 		self.update_bindgroup();
 	}
 
-	pub(crate) fn bind_ui_texture(&mut self, bind_ui_texture: wgpu::Texture) {
+	pub(crate) fn bind_ui_texture(&mut self, ui_texture: wgpu::Texture) {
-		self.ui_texture = Some(bind_ui_texture);
+		if self.ui_texture.as_ref() == Some(&ui_texture) {
+			self.surface_outdated = true;
+			return;
+		}
+		self.ui_texture = Some(ui_texture);
 		self.update_bindgroup();
 	}