feat: focus the target window before ref-addressed physical input

Ref-action physical fallbacks (click/scroll/type) already brought the target
app frontmost before synthesizing CGEvents. The raw-input commands (drag, hover,
mouse-*) resolving a point from a ref did not — the resolver had the pid but
discarded it, so the adapter saw only coordinates and synthetic events could
land on whatever window happened to be frontmost.

Add a best-effort focus_app(pid) to PlatformAdapter (macOS uses
ensure_app_focused; other adapters default to not_supported). The point resolver
now focuses the ref's app before returning, so every physical interaction that
targets a known element raises its window first. Coordinate (--xy) input is
unchanged: the caller owns the target there.
This commit is contained in:
Lahfir 2026-06-10 03:54:10 -07:00
parent 776ee25e72
commit d0371caaec
3 changed files with 14 additions and 2 deletions

View file

@ -224,6 +224,14 @@ pub trait PlatformAdapter: Send + Sync {
Err(AdapterError::not_supported("focus_window"))
}
/// Brings the application owning `pid` to the foreground. Best-effort guard
/// invoked before physical (cursor/keyboard) input that targets a known
/// element, so synthetic events land on the intended window rather than
/// whatever happens to be frontmost.
fn focus_app(&self, _pid: i32) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("focus_app"))
}
fn launch_app(&self, _id: &str, _timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
Err(AdapterError::not_supported("launch_app"))
}

View file

@ -163,11 +163,11 @@ pub(crate) fn resolve_point_from_ref_or_xy_with_context(
context: &CommandContext,
) -> Result<Point, AppError> {
if let Some(ref_id) = args.ref_id {
let (_entry, handle) =
resolve_ref_with_context(ref_id, args.snapshot_id, adapter, context)?;
let (entry, handle) = resolve_ref_with_context(ref_id, args.snapshot_id, adapter, context)?;
let bounds = adapter
.get_element_bounds(handle.handle())?
.ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?;
let _ = adapter.focus_app(entry.pid);
return Ok(Point {
x: bounds.x + bounds.width / 2.0,
y: bounds.y + bounds.height / 2.0,

View file

@ -119,6 +119,10 @@ impl PlatformAdapter for MacOSAdapter {
crate::system::app_ops::focus_window_impl(win)
}
fn focus_app(&self, pid: i32) -> Result<(), AdapterError> {
crate::system::app_ops::ensure_app_focused(pid)
}
fn launch_app(&self, id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
crate::system::app_ops::launch_app_impl(id, timeout_ms)
}