From 595ccb6cc45554351ea3e30b95e4ca47bdf4e16b Mon Sep 17 00:00:00 2001 From: Lahfir Date: Sat, 21 Feb 2026 15:28:28 -0800 Subject: [PATCH] feat: 10-step scroll chain, focus guards, enhanced click chain, bounds fix - Rewrite ax_scroll with 10-step AX-first chain (scroll-to-visible, increment/decrement, page scroll, value shift, sub-element press, focus child, select rows, keyboard, arrow keys, CGEvent last resort) - Add ensure_app_focused() guard before all CGEvent calls - Enhance smart_activate with AXScrollToVisible pre-step, AXShowAlternateUI, AXCustomActions, and keyboard space fallback - Fix bounds_hash always null in RefMap (always read bounds in builder) - Add right-click menu capture in response data - Add append_surface_refs for post-action menu detection --- crates/core/src/commands/right_click.rs | 25 ++- crates/core/src/snapshot.rs | 26 ++- crates/macos/src/actions/activate.rs | 72 ++++++- crates/macos/src/actions/dispatch.rs | 3 + crates/macos/src/actions/extras.rs | 263 +++++++++++++++++++++--- crates/macos/src/input/mouse.rs | 33 ++- crates/macos/src/system/app_ops.rs | 67 ++++-- crates/macos/src/tree/builder.rs | 10 +- crates/macos/src/tree/resolve.rs | 17 +- 9 files changed, 460 insertions(+), 56 deletions(-) diff --git a/crates/core/src/commands/right_click.rs b/crates/core/src/commands/right_click.rs index 1c7fc48..17e22a5 100644 --- a/crates/core/src/commands/right_click.rs +++ b/crates/core/src/commands/right_click.rs @@ -1,5 +1,9 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::Action, + adapter::{PlatformAdapter, SnapshotSurface}, + commands::helpers::resolve_ref, + error::AppError, + snapshot, }; use serde_json::Value; @@ -8,7 +12,22 @@ pub struct RightClickArgs { } pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; + let (entry, handle) = resolve_ref(&args.ref_id, adapter)?; let result = adapter.execute_action(&handle, Action::RightClick)?; - Ok(serde_json::to_value(result)?) + let mut response = serde_json::to_value(&result)?; + + std::thread::sleep(std::time::Duration::from_millis(250)); + + if let Some(menu_tree) = snapshot::append_surface_refs( + adapter, + entry.pid, + entry.source_app.as_deref(), + SnapshotSurface::Menu, + ) { + if let Ok(menu_json) = serde_json::to_value(&menu_tree) { + response["menu"] = menu_json; + } + } + + Ok(response) } diff --git a/crates/core/src/snapshot.rs b/crates/core/src/snapshot.rs index 6d235df..a0b2dbf 100644 --- a/crates/core/src/snapshot.rs +++ b/crates/core/src/snapshot.rs @@ -1,5 +1,5 @@ use crate::{ - adapter::{PlatformAdapter, TreeOptions, WindowFilter}, + adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter}, error::AppError, node::{AccessibilityNode, WindowInfo}, refs::{RefEntry, RefMap}, @@ -108,6 +108,30 @@ pub fn run( Ok(result) } +pub fn append_surface_refs( + adapter: &dyn PlatformAdapter, + pid: i32, + source_app: Option<&str>, + surface: SnapshotSurface, +) -> Option { + let filter = WindowFilter { + focused_only: false, + app: None, + }; + let windows = adapter.list_windows(&filter).ok()?; + let window = windows.into_iter().find(|w| w.pid == pid)?; + let opts = TreeOptions { + surface, + interactive_only: true, + ..Default::default() + }; + let raw_tree = adapter.get_tree(&window, &opts).ok()?; + let mut refmap = RefMap::load().ok()?; + let tree = allocate_refs(raw_tree, &mut refmap, false, true, pid, source_app); + refmap.save().ok()?; + Some(tree) +} + fn allocate_refs( mut node: AccessibilityNode, refmap: &mut RefMap, diff --git a/crates/macos/src/actions/activate.rs b/crates/macos/src/actions/activate.rs index 0cce80c..182525a 100644 --- a/crates/macos/src/actions/activate.rs +++ b/crates/macos/src/actions/activate.rs @@ -16,8 +16,10 @@ mod imp { }; use std::os::raw::c_uchar; - /// Attempts a 10-step AX-first activation chain before falling back to CGEvent. pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> { + let scroll_action = CFString::new("AXScrollToVisible"); + unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) }; + let actions = list_ax_actions(el); if try_action_from_list(el, &actions, &["AXPress"]) { @@ -32,6 +34,9 @@ mod imp { if try_action_from_list(el, &actions, &["AXPick"]) { return Ok(()); } + if try_show_alternate_ui(el) { + return Ok(()); + } if try_child_activation(el) { return Ok(()); @@ -42,9 +47,15 @@ mod imp { if try_select_via_parent(el) { return Ok(()); } + if try_custom_actions(el) { + return Ok(()); + } if try_focus_then_activate(el) { return Ok(()); } + if try_keyboard_activate(el) { + return Ok(()); + } if try_parent_activation(el) { return Ok(()); } @@ -66,7 +77,6 @@ mod imp { crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2) } - /// AXShowMenu first, then CGEvent right-click. pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> { let actions = list_ax_actions(el); if try_action_from_list(el, &actions, &["AXShowMenu"]) { @@ -225,6 +235,64 @@ mod imp { } false } + + fn try_show_alternate_ui(el: &AXElement) -> bool { + let actions = list_ax_actions(el); + if !actions.iter().any(|a| a == "AXShowAlternateUI") { + return false; + } + let action = CFString::new("AXShowAlternateUI"); + unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + std::thread::sleep(std::time::Duration::from_millis(100)); + let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); + for child in children.iter().take(5) { + let child_actions = list_ax_actions(child); + if try_action_from_list(child, &child_actions, &["AXPress"]) { + return true; + } + } + false + } + + fn try_custom_actions(el: &AXElement) -> bool { + let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default(); + if custom.is_empty() { + return false; + } + let action = CFString::new("AXPerformCustomAction"); + let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + err == kAXErrorSuccess + } + + fn try_keyboard_activate(el: &AXElement) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent; + + let cf_focused = CFString::new(kAXFocusedAttribute); + let err = unsafe { + AXUIElementSetAttributeValue( + el.0, + cf_focused.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if err != kAXErrorSuccess { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + + let pid = match crate::system::app_ops::pid_from_element(el) { + Some(p) => p, + None => return false, + }; + let app = crate::tree::element_for_pid(pid); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 49, true); + AXUIElementPostKeyboardEvent(app.0, 0, 49, false); + }; + true + } + + } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/actions/dispatch.rs b/crates/macos/src/actions/dispatch.rs index 4920519..9b31503 100644 --- a/crates/macos/src/actions/dispatch.rs +++ b/crates/macos/src/actions/dispatch.rs @@ -29,6 +29,9 @@ mod imp { button: MouseButton, count: u32, ) -> Result<(), AdapterError> { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } let bounds = crate::tree::read_bounds(el).ok_or_else(|| { AdapterError::new(ErrorCode::ActionFailed, "Element has no readable bounds") .with_suggestion("AX action failed and CGEvent fallback unavailable") diff --git a/crates/macos/src/actions/extras.rs b/crates/macos/src/actions/extras.rs index 21ba201..9cfdfc7 100644 --- a/crates/macos/src/actions/extras.rs +++ b/crates/macos/src/actions/extras.rs @@ -109,69 +109,282 @@ pub(crate) fn ax_scroll( direction: &agent_desktop_core::action::Direction, amount: u32, ) -> Result<(), AdapterError> { - use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction}; + use accessibility_sys::{ + kAXErrorSuccess, AXUIElementPerformAction, AXUIElementPostKeyboardEvent, + AXUIElementSetAttributeValue, + }; use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, string::CFString}; + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; let scroll_area = find_scroll_area(el); let target = scroll_area.as_ref().unwrap_or(el); - let (bar_orientation, action_name) = match direction { + let scroll_visible = CFString::new("AXScrollToVisible"); + if unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) } + == kAXErrorSuccess + { + return Ok(()); + } + + let (bar_attr, inc_action) = match direction { Direction::Down => ("AXVerticalScrollBar", "AXIncrement"), Direction::Up => ("AXVerticalScrollBar", "AXDecrement"), Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"), Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"), }; - if let Some(scroll_bar) = get_scroll_bar(target, bar_orientation) { - let ax_action = CFString::new(action_name); + if let Some(bar) = get_scroll_bar(target, bar_attr) { + let ax_action = CFString::new(inc_action); + let mut ok = true; for _ in 0..amount { - let err = - unsafe { AXUIElementPerformAction(scroll_bar.0, ax_action.as_concrete_TypeRef()) }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("{action_name} on scroll bar failed (err={err})"), - )); + if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) } + != kAXErrorSuccess + { + ok = false; + break; } } - return Ok(()); + if ok { + return Ok(()); + } } - let scroll_action_name = match direction { + let page_action = match direction { Direction::Down => "AXScrollDownByPage", Direction::Up => "AXScrollUpByPage", Direction::Right => "AXScrollRightByPage", Direction::Left => "AXScrollLeftByPage", }; - - if crate::actions::dispatch::has_ax_action(target, scroll_action_name) { - let ax_action = CFString::new(scroll_action_name); + if crate::actions::dispatch::has_ax_action(target, page_action) { + let ax = CFString::new(page_action); for _ in 0..amount { - unsafe { AXUIElementPerformAction(target.0, ax_action.as_concrete_TypeRef()) }; + unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) }; } return Ok(()); } + if let Some(bar) = get_scroll_bar(target, bar_attr) { + if try_scroll_bar_value_shift(&bar, direction, amount) { + return Ok(()); + } + if try_scroll_bar_sub_elements(&bar, direction) { + return Ok(()); + } + } + + if try_focus_child_in_direction(target, direction) { + return Ok(()); + } + if try_select_row_in_direction(target, direction) { + return Ok(()); + } + + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let keycode: u16 = match direction { + Direction::Down => 121, + Direction::Up => 116, + Direction::Right => 124, + Direction::Left => 123, + }; + let cf_focused = CFString::new("AXFocused"); + unsafe { + AXUIElementSetAttributeValue( + target.0, + cf_focused.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + std::thread::sleep(std::time::Duration::from_millis(50)); + let app = crate::tree::element_for_pid(pid); + for _ in 0..amount { + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, keycode, true); + AXUIElementPostKeyboardEvent(app.0, 0, keycode, false); + }; + } + return Ok(()); + } + + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + if let Some(b) = crate::tree::read_bounds(target) { + let (dy, dx) = match direction { + Direction::Down => (-(amount as i32) * 5, 0), + Direction::Up => (amount as i32 * 5, 0), + Direction::Right => (0, -(amount as i32) * 5), + Direction::Left => (0, amount as i32 * 5), + }; + return crate::input::mouse::synthesize_scroll_at( + b.x + b.width / 2.0, + b.y + b.height / 2.0, + dy, + dx, + ); + } + } + Err(AdapterError::new( ErrorCode::ActionNotSupported, - "No scroll bar or scroll action found on this element", + "No scroll mechanism found on element", ) - .with_suggestion("Element may not be scrollable, or try scrolling the parent container.")) + .with_suggestion("Element may not be scrollable, or try the parent container.")) +} + +#[cfg(target_os = "macos")] +fn try_scroll_bar_value_shift( + bar: &AXElement, + direction: &agent_desktop_core::action::Direction, + amount: u32, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use agent_desktop_core::action::Direction; + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + if !crate::actions::activate::is_attr_settable(bar, "AXValue") { + return false; + } + let current = read_scroll_bar_value(bar).unwrap_or(0.0); + let delta = 0.1 * amount as f64; + let new_val = match direction { + Direction::Down | Direction::Right => (current + delta).min(1.0), + Direction::Up | Direction::Left => (current - delta).max(0.0), + }; + let cf_num = CFNumber::from(new_val as f32); + let cf_attr = CFString::new("AXValue"); + let err = unsafe { + AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef()) + }; + err == kAXErrorSuccess +} + +#[cfg(target_os = "macos")] +fn read_scroll_bar_value(bar: &AXElement) -> Option { + use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue}; + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + let cf_attr = CFString::new("AXValue"); + let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut(); + let err = + unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) }; + if err != kAXErrorSuccess || value.is_null() { + return None; + } + let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; + cf.downcast::().and_then(|n| n.to_f64()) +} + +#[cfg(target_os = "macos")] +fn try_scroll_bar_sub_elements( + bar: &AXElement, + direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction}; + use agent_desktop_core::action::Direction; + use core_foundation::{base::TCFType, string::CFString}; + + let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default(); + let target_subroles = match direction { + Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"], + Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"], + }; + let press = CFString::new("AXPress"); + for child in &children { + let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default(); + if target_subroles.iter().any(|t| *t == sr) + && unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) } + == kAXErrorSuccess + { + return true; + } + } + false +} + +#[cfg(target_os = "macos")] +fn try_focus_child_in_direction( + scroll_area: &AXElement, + _direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); + let child = match children.first() { + Some(c) => c, + None => return false, + }; + let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default(); + let target = match grandchildren.last() { + Some(t) => t, + None => return false, + }; + let cf_attr = CFString::new("AXFocused"); + let err = unsafe { + AXUIElementSetAttributeValue( + target.0, + cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + err == kAXErrorSuccess +} + +#[cfg(target_os = "macos")] +fn try_select_row_in_direction( + scroll_area: &AXElement, + _direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + + let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); + for child in &children { + let role = crate::tree::copy_string_attr(child, kAXRoleAttribute); + if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) { + continue; + } + if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") { + continue; + } + let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default(); + if let Some(last) = rows.last() { + unsafe { CFRetain(last.0 as CFTypeRef) }; + let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) }; + let arr = CFArray::from_CFTypes(&[el_as_cftype]); + let cf_attr = CFString::new("AXSelectedRows"); + let err = unsafe { + AXUIElementSetAttributeValue( + child.0, + cf_attr.as_concrete_TypeRef(), + arr.as_CFTypeRef(), + ) + }; + if err == kAXErrorSuccess { + return true; + } + } + } + false } #[cfg(target_os = "macos")] fn find_scroll_area(el: &AXElement) -> Option { use accessibility_sys::kAXRoleAttribute; - let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?; if role == "AXScrollArea" { return Some(el.clone()); } - let parent = crate::tree::copy_element_attr(el, "AXParent")?; - let parent_role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute)?; - if parent_role == "AXScrollArea" { - return Some(parent); + let mut current = crate::tree::copy_element_attr(el, "AXParent")?; + for _ in 0..5 { + let r = crate::tree::copy_string_attr(¤t, kAXRoleAttribute)?; + if r == "AXScrollArea" { + return Some(current); + } + current = crate::tree::copy_element_attr(¤t, "AXParent")?; } None } diff --git a/crates/macos/src/input/mouse.rs b/crates/macos/src/input/mouse.rs index 5da45a3..658fb0b 100644 --- a/crates/macos/src/input/mouse.rs +++ b/crates/macos/src/input/mouse.rs @@ -129,6 +129,33 @@ mod imp { MouseButton::Middle => CGEventType::OtherMouseUp, } } + + pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> { + use core_graphics::geometry::CGPoint; + + extern "C" { + fn CGEventCreateScrollWheelEvent( + source: *const std::ffi::c_void, + units: u32, + wheel_count: u32, + wheel1: i32, + wheel2: i32, + ) -> *mut std::ffi::c_void; + fn CGEventSetLocation(event: *mut std::ffi::c_void, point: CGPoint); + fn CGEventPost(tap: u32, event: *mut std::ffi::c_void); + } + + let event = unsafe { CGEventCreateScrollWheelEvent(std::ptr::null(), 0, 2, dy, dx) }; + if event.is_null() { + return Err(AdapterError::internal("scroll event creation failed")); + } + unsafe { + CGEventSetLocation(event, CGPoint::new(x, y)); + CGEventPost(0, event); + core_foundation::base::CFRelease(event as _); + } + Ok(()) + } } #[cfg(not(target_os = "macos"))] @@ -142,6 +169,10 @@ mod imp { pub fn synthesize_drag(_params: DragParams) -> Result<(), AdapterError> { Err(AdapterError::not_supported("drag")) } + + pub fn synthesize_scroll_at(_x: f64, _y: f64, _dy: i32, _dx: i32) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("scroll")) + } } -pub use imp::{synthesize_drag, synthesize_mouse}; +pub use imp::{synthesize_drag, synthesize_mouse, synthesize_scroll_at}; diff --git a/crates/macos/src/system/app_ops.rs b/crates/macos/src/system/app_ops.rs index c807c3a..d54b19e 100644 --- a/crates/macos/src/system/app_ops.rs +++ b/crates/macos/src/system/app_ops.rs @@ -1,5 +1,39 @@ use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo}; +#[cfg(target_os = "macos")] +pub fn pid_from_element(el: &crate::tree::AXElement) -> Option { + let mut pid: i32 = 0; + let err = unsafe { accessibility_sys::AXUIElementGetPid(el.0, &mut pid) }; + if err == accessibility_sys::kAXErrorSuccess { + Some(pid) + } else { + None + } +} + +#[cfg(target_os = "macos")] +pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + let app_el = crate::tree::element_for_pid(pid); + let frontmost_attr = CFString::new("AXFrontmost"); + let err = unsafe { + AXUIElementSetAttributeValue( + app_el.0, + frontmost_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if err != kAXErrorSuccess { + return Err(AdapterError::internal(format!( + "Failed to focus app pid={pid}" + ))); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + Ok(()) +} + #[cfg(target_os = "macos")] pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> { use accessibility_sys::{ @@ -162,22 +196,23 @@ fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool { let Some(bar_items) = crate::tree::copy_ax_array(&menu_bar, "AXChildren") else { return false; }; - if let Some(app_menu) = bar_items.first() { - if let Some(menus) = crate::tree::copy_ax_array(app_menu, "AXChildren") { - for menu in &menus { - if let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") { - for item in &items { - let title = crate::tree::copy_string_attr(item, "AXTitle"); - if let Some(t) = &title { - if t.contains("Quit") || t.contains("quit") { - let press = CFString::new("AXPress"); - let err = unsafe { - AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) - }; - return err == kAXErrorSuccess; - } - } - } + for bar_item in bar_items.iter().skip(1) { + let Some(menus) = crate::tree::copy_ax_array(bar_item, "AXChildren") else { + continue; + }; + for menu in &menus { + let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") else { + continue; + }; + for item in &items { + let Some(t) = crate::tree::copy_string_attr(item, "AXTitle") else { + continue; + }; + if t.starts_with("Quit") { + let press = CFString::new("AXPress"); + let err = + unsafe { AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) }; + return err == kAXErrorSuccess; } } } diff --git a/crates/macos/src/tree/builder.rs b/crates/macos/src/tree/builder.rs index 362d136..b978c64 100644 --- a/crates/macos/src/tree/builder.rs +++ b/crates/macos/src/tree/builder.rs @@ -45,7 +45,7 @@ pub fn build_subtree( el: &AXElement, depth: u8, max_depth: u8, - include_bounds: bool, + _include_bounds: bool, ancestors: &mut FxHashSet, ) -> Option { if depth > max_depth || depth >= ABSOLUTE_MAX_DEPTH { @@ -81,18 +81,14 @@ pub fn build_subtree( states.push("disabled".into()); } - let bounds = if include_bounds { - read_bounds(el) - } else { - None - }; + let bounds = read_bounds(el); let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default(); let name = name.or_else(|| label_from_children(&children_raw)); let children = children_raw .into_iter() - .filter_map(|child| build_subtree(&child, depth + 1, max_depth, include_bounds, ancestors)) + .filter_map(|child| build_subtree(&child, depth + 1, max_depth, _include_bounds, ancestors)) .collect(); ancestors.remove(&ptr_key); diff --git a/crates/macos/src/tree/resolve.rs b/crates/macos/src/tree/resolve.rs index ffc1f62..a885f3a 100644 --- a/crates/macos/src/tree/resolve.rs +++ b/crates/macos/src/tree/resolve.rs @@ -41,7 +41,22 @@ pub fn find_element_recursive( (None, None) => true, _ => false, }; - if name_match { + let bounds_match = match entry.bounds_hash { + Some(expected) => { + let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash()); + tracing::debug!( + role = normalized, + ?elem_name, + ?actual, + expected, + name_match, + "resolve candidate" + ); + actual.map(|h| h == expected).unwrap_or(false) + } + None => true, + }; + if name_match && bounds_match { ancestors.remove(&ptr_key); unsafe { CFRetain(el.0 as CFTypeRef) }; return Ok(NativeHandle::from_ptr(el.0 as *const _));