F2: ui.screenshot captures the webview without focusing the window

On Windows the webview renders itself into a PNG through WebView2
CapturePreview (scheduled on the main thread via with_webview, 10 s
timeout, PNG signature and 64 MiB cap checked). macOS and Linux keep the
xcap screen-region capture as a documented fallback, now without focusing.
app.focus is the only control method that changes focus. A --control
instance opens its window without taking the foreground. An ignored live
test samples the foreground window while the app starts and takes five
screenshots and fails if Lutra ever owns it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-27 17:13:38 +01:00
parent d567f3957d
commit f49b42836a
12 changed files with 398 additions and 84 deletions

View file

@ -20,11 +20,20 @@ lutra-engine.workspace = true
serde_json = "1.0.154"
tauri = { version = "2.11.6", features = [] }
tauri-plugin-dialog = "2.7.3"
[target.'cfg(windows)'.dependencies]
webview2-com = "0.38.2"
windows = { version = "0.61.3", features = ["Win32_System_Com", "Win32_UI_Shell"] }
[target.'cfg(not(windows))'.dependencies]
xcap = "0.9.8"
[dev-dependencies]
tempfile.workspace = true
[target.'cfg(windows)'.dev-dependencies]
windows = { version = "0.61.3", features = ["Win32_UI_WindowsAndMessaging"] }
[build-dependencies]
png = "0.18.1"
tauri-build = { version = "2.6.3", features = [] }

View file

@ -8,7 +8,9 @@
//! start is a single instance (see [`instance`]): a second start forwards
//! its path to the running app and exits. A start with `--data-dir`,
//! `LUTRA_DATA_DIR` or `--control` keeps its files away from the user's
//! (see [`datadir`]).
//! (see [`datadir`]). With `--control` the window opens without taking the
//! foreground, so an agent starting Lutra does not catch the user's typing;
//! `app.focus` brings it to the front on request.
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
@ -85,6 +87,7 @@ fn run(args: &args::Args) -> Result<(), String> {
let slot: Arc<OnceLock<Arc<Host>>> = Arc::new(OnceLock::new());
let setup_slot = slot.clone();
let initial_path = args.path.clone();
let focus_on_start = args.control.is_none();
let app = tauri::Builder::default()
.plugin(tauri_plugin_dialog::init())
.invoke_handler(tauri::generate_handler![
@ -102,6 +105,7 @@ fn run(args: &args::Args) -> Result<(), String> {
.title(WINDOW_TITLE)
.inner_size(1200.0, 800.0)
.min_inner_size(640.0, 400.0)
.focused(focus_on_start)
.initialization_script(theme::init_script(&setting));
if let Some(dir) = &webview_dir {
window = window.data_directory(dir.clone());

View file

@ -1,30 +1,45 @@
//! Screenshots of Lutra's main window for `ui.screenshot`.
//!
//! `xcap` refuses to enumerate windows of the calling process (querying their
//! titles can deadlock), so the screenshot is taken from the screen instead:
//! the main window is brought to the front, its client area is located in
//! physical pixels through Tauri, and that region of the monitor is captured.
//! Anything covering the window at that moment shows up in the image.
//! A screenshot never focuses, raises or shows the window: `app.focus` is the
//! only control method that does. Taking one while the user types into
//! another app must not pull their keystrokes into Lutra.
//!
//! - Windows: the webview renders itself into a PNG through WebView2's
//! `CapturePreview` ([`webview2`]). The image is the page as the webview
//! draws it, whether or not other windows cover Lutra; the native title bar
//! is not part of it.
//! - Other platforms (fallback): the window's client area is located in
//! physical pixels through Tauri and that region of its monitor is captured
//! with `xcap` ([`region`]). Windows covering Lutra at that moment appear in
//! the image. WKWebView's and WebKitGTK's own snapshot APIs can replace it.
use std::path::Path;
use std::time::Duration;
use tauri::{AppHandle, Manager};
use xcap::Monitor;
use xcap::image::ImageFormat;
/// Time for the compositor to show the window after it was focused.
const SETTLE: Duration = Duration::from_millis(150);
#[cfg(not(windows))]
mod region;
#[cfg(windows)]
mod webview2;
/// Captures the client area of the window labelled `label` and writes it to
/// `path` as PNG. Must not be called on the main thread, which has to stay
/// free to answer the window queries. Focusing the window is best effort: a
/// failure is ignored because the capture still works when the window is
/// already in front.
/// The eight bytes every PNG file starts with.
const PNG_SIGNATURE: [u8; 8] = [0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n'];
/// Largest PNG accepted from the platform capture.
const MAX_PNG_BYTES: usize = 64 * 1024 * 1024;
/// How long to wait for the platform capture before giving up.
const CAPTURE_TIMEOUT: Duration = Duration::from_secs(10);
/// Captures the window labelled `label` and writes it to `path` as PNG,
/// without changing which window is in front or has the keyboard focus.
/// Must not be called on the main thread: the capture is scheduled there and
/// this function waits for it.
///
/// # Errors
/// A message if the window does not exist or is minimised, the monitor
/// cannot be captured, or the PNG cannot be written.
/// A message if the window does not exist or is minimised, the capture fails
/// or times out, the result is not a PNG, or the file cannot be written.
pub fn capture_window(app: &AppHandle, label: &str, path: &Path) -> Result<(), String> {
let window = app
.get_webview_window(label)
@ -32,76 +47,43 @@ pub fn capture_window(app: &AppHandle, label: &str, path: &Path) -> Result<(), S
if window.is_minimized().unwrap_or(false) {
return Err("the main window is minimised".into());
}
let _ = window.set_focus();
std::thread::sleep(SETTLE);
let pos = window.inner_position().map_err(|e| e.to_string())?;
let size = window.inner_size().map_err(|e| e.to_string())?;
let centre_x = pos
.x
.saturating_add(i32::try_from(size.width / 2).unwrap_or(0));
let centre_y = pos
.y
.saturating_add(i32::try_from(size.height / 2).unwrap_or(0));
let monitor = Monitor::from_point(centre_x, centre_y)
.map_err(|e| format!("cannot find the window's monitor: {e}"))?;
let (mx, my) = (
monitor.x().map_err(|e| e.to_string())?,
monitor.y().map_err(|e| e.to_string())?,
);
let (mw, mh) = (
monitor.width().map_err(|e| e.to_string())?,
monitor.height().map_err(|e| e.to_string())?,
);
let region = clip(
pos.x.saturating_sub(mx),
pos.y.saturating_sub(my),
size.width,
size.height,
mw,
mh,
)
.ok_or("the main window is outside its monitor")?;
let image = monitor
.capture_region(region.0, region.1, region.2, region.3)
.map_err(|e| format!("cannot capture the screen: {e}"))?;
#[cfg(windows)]
let png = webview2::capture_png(&window)?;
#[cfg(not(windows))]
let png = region::capture_png(&window)?;
check_png(&png)?;
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
std::fs::create_dir_all(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
}
image
.save_with_format(path, ImageFormat::Png)
.map_err(|e| format!("{}: {e}", path.display()))
std::fs::write(path, png).map_err(|e| format!("{}: {e}", path.display()))
}
/// Clips a rectangle given relative to a monitor's origin to the monitor's
/// bounds, returning `(x, y, width, height)` or `None` if nothing is left.
fn clip(x: i32, y: i32, w: u32, h: u32, mw: u32, mh: u32) -> Option<(u32, u32, u32, u32)> {
let left = i64::from(x).max(0);
let top = i64::from(y).max(0);
let right = (i64::from(x) + i64::from(w)).min(i64::from(mw));
let bottom = (i64::from(y) + i64::from(h)).min(i64::from(mh));
if right <= left || bottom <= top {
return None;
/// Checks that `bytes` start with the PNG signature and stay within
/// [`MAX_PNG_BYTES`].
///
/// # Errors
/// A message naming what is wrong.
fn check_png(bytes: &[u8]) -> Result<(), String> {
if bytes.len() > MAX_PNG_BYTES {
return Err(format!("the capture is larger than {MAX_PNG_BYTES} bytes"));
}
Some((
u32::try_from(left).ok()?,
u32::try_from(top).ok()?,
u32::try_from(right - left).ok()?,
u32::try_from(bottom - top).ok()?,
))
if !bytes.starts_with(&PNG_SIGNATURE) {
return Err("the capture is not a PNG image".into());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::clip;
use super::*;
#[test]
fn clips_to_the_monitor() {
assert_eq!(clip(10, 20, 100, 50, 1920, 1080), Some((10, 20, 100, 50)));
assert_eq!(clip(-10, -5, 100, 50, 1920, 1080), Some((0, 0, 90, 45)));
assert_eq!(
clip(1900, 1000, 100, 100, 1920, 1080),
Some((1900, 1000, 20, 80))
);
assert_eq!(clip(2000, 0, 100, 100, 1920, 1080), None);
fn accepts_png_and_rejects_the_rest() {
let mut png = PNG_SIGNATURE.to_vec();
png.extend_from_slice(b"rest");
assert!(check_png(&png).is_ok());
assert!(check_png(b"GIF89a").is_err());
assert!(check_png(&[]).is_err());
assert!(check_png(&vec![0; MAX_PNG_BYTES + 1]).is_err());
}
}

View file

@ -0,0 +1,91 @@
//! Fallback for platforms without a webview capture yet (macOS, Linux):
//! capture the window's client area from the screen with `xcap`.
//!
//! `xcap` refuses to enumerate windows of the calling process (querying their
//! titles can deadlock), so the client area is located in physical pixels
//! through Tauri and that region of its monitor is captured. The window is
//! not focused or raised: anything covering it at that moment shows up in
//! the image.
use std::io::Cursor;
use tauri::WebviewWindow;
use xcap::Monitor;
use xcap::image::ImageFormat;
/// Captures the screen region under `window`'s client area as PNG bytes.
///
/// # Errors
/// A message if the window's geometry or monitor cannot be read, the window
/// is outside its monitor, or the capture or encoding fails.
pub fn capture_png(window: &WebviewWindow) -> Result<Vec<u8>, String> {
let pos = window.inner_position().map_err(|e| e.to_string())?;
let size = window.inner_size().map_err(|e| e.to_string())?;
let centre_x = pos
.x
.saturating_add(i32::try_from(size.width / 2).unwrap_or(0));
let centre_y = pos
.y
.saturating_add(i32::try_from(size.height / 2).unwrap_or(0));
let monitor = Monitor::from_point(centre_x, centre_y)
.map_err(|e| format!("cannot find the window's monitor: {e}"))?;
let (mx, my) = (
monitor.x().map_err(|e| e.to_string())?,
monitor.y().map_err(|e| e.to_string())?,
);
let (mw, mh) = (
monitor.width().map_err(|e| e.to_string())?,
monitor.height().map_err(|e| e.to_string())?,
);
let region = clip(
pos.x.saturating_sub(mx),
pos.y.saturating_sub(my),
size.width,
size.height,
mw,
mh,
)
.ok_or("the main window is outside its monitor")?;
let image = monitor
.capture_region(region.0, region.1, region.2, region.3)
.map_err(|e| format!("cannot capture the screen: {e}"))?;
let mut png = Cursor::new(Vec::new());
image
.write_to(&mut png, ImageFormat::Png)
.map_err(|e| format!("cannot encode the capture: {e}"))?;
Ok(png.into_inner())
}
/// Clips a rectangle given relative to a monitor's origin to the monitor's
/// bounds, returning `(x, y, width, height)` or `None` if nothing is left.
fn clip(x: i32, y: i32, w: u32, h: u32, mw: u32, mh: u32) -> Option<(u32, u32, u32, u32)> {
let left = i64::from(x).max(0);
let top = i64::from(y).max(0);
let right = (i64::from(x) + i64::from(w)).min(i64::from(mw));
let bottom = (i64::from(y) + i64::from(h)).min(i64::from(mh));
if right <= left || bottom <= top {
return None;
}
Some((
u32::try_from(left).ok()?,
u32::try_from(top).ok()?,
u32::try_from(right - left).ok()?,
u32::try_from(bottom - top).ok()?,
))
}
#[cfg(test)]
mod tests {
use super::clip;
#[test]
fn clips_to_the_monitor() {
assert_eq!(clip(10, 20, 100, 50, 1920, 1080), Some((10, 20, 100, 50)));
assert_eq!(clip(-10, -5, 100, 50, 1920, 1080), Some((0, 0, 90, 45)));
assert_eq!(
clip(1900, 1000, 100, 100, 1920, 1080),
Some((1900, 1000, 20, 80))
);
assert_eq!(clip(2000, 0, 100, 100, 1920, 1080), None);
}
}

View file

@ -0,0 +1,110 @@
//! Windows: the webview captures itself with WebView2's `CapturePreview`
//! (<https://learn.microsoft.com/microsoft-edge/webview2/reference/win32/icorewebview2#capturepreview>).
//!
//! The capture runs on the main thread, where Tauri hands out the WebView2
//! controller, and completes asynchronously through the main thread's message
//! loop. The caller waits on a channel for the PNG bytes. Nothing here
//! focuses, raises or shows a window.
//!
//! This module allows `unsafe` because the WebView2 COM interfaces are only
//! reachable through FFI. Every call gets valid arguments by construction:
//! the controller comes from Tauri, the stream from `SHCreateMemStream`, the
//! completion handler from `webview2-com`, and reads go into a buffer whose
//! length is passed alongside its pointer.
#![allow(unsafe_code)]
use std::sync::mpsc::{self, Sender};
use tauri::WebviewWindow;
use webview2_com::CapturePreviewCompletedHandler;
use webview2_com::Microsoft::Web::WebView2::Win32::{
COREWEBVIEW2_CAPTURE_PREVIEW_IMAGE_FORMAT_PNG, ICoreWebView2Controller,
};
use windows::Win32::System::Com::{IStream, STREAM_SEEK_SET};
use windows::Win32::UI::Shell::SHCreateMemStream;
use super::{CAPTURE_TIMEOUT, MAX_PNG_BYTES};
/// Bytes read from the memory stream per call.
const CHUNK: u32 = 64 * 1024;
/// What the main thread sends back: the PNG bytes or an error message.
type Capture = Result<Vec<u8>, String>;
/// Captures `window`'s webview as PNG bytes.
///
/// # Errors
/// A message if the capture cannot be scheduled, WebView2 reports a failure,
/// or no result arrives within [`CAPTURE_TIMEOUT`].
pub fn capture_png(window: &WebviewWindow) -> Result<Vec<u8>, String> {
let (tx, rx) = mpsc::channel::<Capture>();
window
.with_webview(move |webview| {
if let Err(e) = start(&webview.controller(), &tx) {
let _ = tx.send(Err(e));
}
})
.map_err(|e| format!("cannot reach the webview: {e}"))?;
rx.recv_timeout(CAPTURE_TIMEOUT)
.map_err(|_| "the webview did not deliver the capture in time".to_string())?
}
/// Starts `CapturePreview` into a memory stream; the completion handler
/// reads the stream and sends the result on `tx`. Runs on the main thread. A
/// failed send means the caller timed out and left, so it is ignored.
///
/// # Errors
/// A message if the webview, the stream or the capture call fails
/// synchronously; the handler then never runs.
fn start(controller: &ICoreWebView2Controller, tx: &Sender<Capture>) -> Result<(), String> {
let webview = unsafe { controller.CoreWebView2() }
.map_err(|e| format!("cannot reach the webview: {e}"))?;
let stream = unsafe { SHCreateMemStream(None) }.ok_or("cannot create a memory stream")?;
let target = stream.clone();
let tx = tx.clone();
let handler = CapturePreviewCompletedHandler::create(Box::new(move |result| {
let capture = result
.map_err(|e| format!("WebView2 could not capture the page: {e}"))
.and_then(|()| read_all(&target));
let _ = tx.send(capture);
Ok(())
}));
unsafe {
webview.CapturePreview(
COREWEBVIEW2_CAPTURE_PREVIEW_IMAGE_FORMAT_PNG,
&stream,
&handler,
)
}
.map_err(|e| format!("WebView2 could not start the capture: {e}"))
}
/// Reads `stream` from its start to its end.
///
/// # Errors
/// A message if seeking or reading fails or the data exceeds
/// [`MAX_PNG_BYTES`].
fn read_all(stream: &IStream) -> Result<Vec<u8>, String> {
unsafe { stream.Seek(0, STREAM_SEEK_SET, None) }
.map_err(|e| format!("cannot rewind the capture: {e}"))?;
let mut out = Vec::new();
let mut chunk = vec![0_u8; usize::try_from(CHUNK).unwrap_or(65_536)];
let len = u32::try_from(chunk.len()).unwrap_or(CHUNK);
loop {
let mut read = 0_u32;
unsafe { stream.Read(chunk.as_mut_ptr().cast(), len, Some(&mut read)) }
.ok()
.map_err(|e| format!("cannot read the capture: {e}"))?;
let filled = chunk
.get(..usize::try_from(read).unwrap_or(usize::MAX))
.ok_or("the capture stream reported more bytes than it read")?;
if filled.is_empty() {
return Ok(out);
}
if out.len().saturating_add(filled.len()) > MAX_PNG_BYTES {
return Err(format!("the capture is larger than {MAX_PNG_BYTES} bytes"));
}
out.extend_from_slice(filled);
}
}

View file

@ -0,0 +1,111 @@
//! Live check that `ui.screenshot` never takes the foreground (Windows).
//!
//! Starts a built desktop app (`LUTRA_DESKTOP_EXE`, built with
//! `--features bundled-ui`) with `--control` and an isolated data directory,
//! takes several screenshots over the control channel and samples the
//! foreground window every few milliseconds the whole time. The test fails
//! if the foreground window ever belongs to the app's process, or if a
//! screenshot is not a PNG. Ignored by default because it needs a desktop
//! session and a built app:
//!
//! `LUTRA_DESKTOP_EXE=… LUTRA_CONTROL_PORT=7962 cargo test -p lutra-desktop --test screenshot_focus -- --ignored`
#![cfg(windows)]
#![allow(unsafe_code)]
use std::process::{Child, Command};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::thread;
use std::time::{Duration, Instant};
use lutra_control::client::Client;
use serde_json::json;
use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
/// Kills the app when the test ends, also on failure.
struct App(Child);
impl Drop for App {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
/// Process id owning the current foreground window, 0 if there is none.
fn foreground_pid() -> u32 {
let mut pid = 0_u32;
unsafe {
let window = GetForegroundWindow();
GetWindowThreadProcessId(window, Some(&mut pid));
}
pid
}
#[test]
#[ignore = "starts the desktop app; set LUTRA_DESKTOP_EXE and run with --ignored"]
fn screenshots_never_take_the_foreground() {
let exe = std::env::var("LUTRA_DESKTOP_EXE").expect("LUTRA_DESKTOP_EXE");
let port: u16 = std::env::var("LUTRA_CONTROL_PORT")
.unwrap_or_else(|_| "7962".into())
.parse()
.unwrap();
let data = tempfile::tempdir().unwrap();
let shots = tempfile::tempdir().unwrap();
let app = App(Command::new(exe)
.arg("--control")
.arg(port.to_string())
.arg("--data-dir")
.arg(data.path())
.spawn()
.unwrap());
let pid = app.0.id();
let stop = Arc::new(AtomicBool::new(false));
let stolen = Arc::new(AtomicU32::new(0));
let sampler = {
let (stop, stolen) = (stop.clone(), stolen.clone());
thread::spawn(move || {
while !stop.load(Ordering::Relaxed) {
if foreground_pid() == pid {
stolen.fetch_add(1, Ordering::Relaxed);
}
thread::sleep(Duration::from_millis(5));
}
})
};
let deadline = Instant::now() + Duration::from_secs(90);
let mut client = loop {
match Client::connect(port, Duration::from_secs(2)) {
Ok(client) => break client,
Err(_) if Instant::now() < deadline => thread::sleep(Duration::from_millis(250)),
Err(e) => panic!("the app did not open its control port: {e:?}"),
}
};
client.request("app.ping", json!({})).unwrap();
thread::sleep(Duration::from_secs(3));
for i in 0..5 {
let path = shots.path().join(format!("shot-{i}.png"));
client
.request("ui.screenshot", json!({ "path": path }))
.unwrap();
let png = std::fs::read(&path).unwrap();
assert!(
png.starts_with(b"\x89PNG\r\n\x1a\n"),
"shot {i} is not a PNG"
);
assert!(png.len() > 1000, "shot {i} is suspiciously small");
thread::sleep(Duration::from_millis(200));
}
stop.store(true, Ordering::Relaxed);
sampler.join().unwrap();
assert_eq!(
stolen.load(Ordering::Relaxed),
0,
"the app's window was in the foreground during the check"
);
}