feat: 10-step scroll chain, focus guards, enhanced click chain, bounds fix

- Rewrite ax_scroll with 10-step AX-first chain (scroll-to-visible,
  increment/decrement, page scroll, value shift, sub-element press,
  focus child, select rows, keyboard, arrow keys, CGEvent last resort)
- Add ensure_app_focused() guard before all CGEvent calls
- Enhance smart_activate with AXScrollToVisible pre-step,
  AXShowAlternateUI, AXCustomActions, and keyboard space fallback
- Fix bounds_hash always null in RefMap (always read bounds in builder)
- Add right-click menu capture in response data
- Add append_surface_refs for post-action menu detection
This commit is contained in:
Lahfir 2026-02-21 15:28:28 -08:00
parent 2f495ffb69
commit 595ccb6cc4
9 changed files with 460 additions and 56 deletions

View file

@ -1,5 +1,9 @@
use crate::{
action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError,
action::Action,
adapter::{PlatformAdapter, SnapshotSurface},
commands::helpers::resolve_ref,
error::AppError,
snapshot,
};
use serde_json::Value;
@ -8,7 +12,22 @@ pub struct RightClickArgs {
}
pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?;
let (entry, handle) = resolve_ref(&args.ref_id, adapter)?;
let result = adapter.execute_action(&handle, Action::RightClick)?;
Ok(serde_json::to_value(result)?)
let mut response = serde_json::to_value(&result)?;
std::thread::sleep(std::time::Duration::from_millis(250));
if let Some(menu_tree) = snapshot::append_surface_refs(
adapter,
entry.pid,
entry.source_app.as_deref(),
SnapshotSurface::Menu,
) {
if let Ok(menu_json) = serde_json::to_value(&menu_tree) {
response["menu"] = menu_json;
}
}
Ok(response)
}

View file

@ -1,5 +1,5 @@
use crate::{
adapter::{PlatformAdapter, TreeOptions, WindowFilter},
adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter},
error::AppError,
node::{AccessibilityNode, WindowInfo},
refs::{RefEntry, RefMap},
@ -108,6 +108,30 @@ pub fn run(
Ok(result)
}
pub fn append_surface_refs(
adapter: &dyn PlatformAdapter,
pid: i32,
source_app: Option<&str>,
surface: SnapshotSurface,
) -> Option<AccessibilityNode> {
let filter = WindowFilter {
focused_only: false,
app: None,
};
let windows = adapter.list_windows(&filter).ok()?;
let window = windows.into_iter().find(|w| w.pid == pid)?;
let opts = TreeOptions {
surface,
interactive_only: true,
..Default::default()
};
let raw_tree = adapter.get_tree(&window, &opts).ok()?;
let mut refmap = RefMap::load().ok()?;
let tree = allocate_refs(raw_tree, &mut refmap, false, true, pid, source_app);
refmap.save().ok()?;
Some(tree)
}
fn allocate_refs(
mut node: AccessibilityNode,
refmap: &mut RefMap,

View file

@ -16,8 +16,10 @@ mod imp {
};
use std::os::raw::c_uchar;
/// Attempts a 10-step AX-first activation chain before falling back to CGEvent.
pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> {
let scroll_action = CFString::new("AXScrollToVisible");
unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) };
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXPress"]) {
@ -32,6 +34,9 @@ mod imp {
if try_action_from_list(el, &actions, &["AXPick"]) {
return Ok(());
}
if try_show_alternate_ui(el) {
return Ok(());
}
if try_child_activation(el) {
return Ok(());
@ -42,9 +47,15 @@ mod imp {
if try_select_via_parent(el) {
return Ok(());
}
if try_custom_actions(el) {
return Ok(());
}
if try_focus_then_activate(el) {
return Ok(());
}
if try_keyboard_activate(el) {
return Ok(());
}
if try_parent_activation(el) {
return Ok(());
}
@ -66,7 +77,6 @@ mod imp {
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
}
/// AXShowMenu first, then CGEvent right-click.
pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> {
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXShowMenu"]) {
@ -225,6 +235,64 @@ mod imp {
}
false
}
fn try_show_alternate_ui(el: &AXElement) -> bool {
let actions = list_ax_actions(el);
if !actions.iter().any(|a| a == "AXShowAlternateUI") {
return false;
}
let action = CFString::new("AXShowAlternateUI");
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
std::thread::sleep(std::time::Duration::from_millis(100));
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(5) {
let child_actions = list_ax_actions(child);
if try_action_from_list(child, &child_actions, &["AXPress"]) {
return true;
}
}
false
}
fn try_custom_actions(el: &AXElement) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
let action = CFString::new("AXPerformCustomAction");
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_keyboard_activate(el: &AXElement) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
let cf_focused = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_focused.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
}
#[cfg(not(target_os = "macos"))]

View file

@ -29,6 +29,9 @@ mod imp {
button: MouseButton,
count: u32,
) -> Result<(), AdapterError> {
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
}
let bounds = crate::tree::read_bounds(el).ok_or_else(|| {
AdapterError::new(ErrorCode::ActionFailed, "Element has no readable bounds")
.with_suggestion("AX action failed and CGEvent fallback unavailable")

View file

@ -109,69 +109,282 @@ pub(crate) fn ax_scroll(
direction: &agent_desktop_core::action::Direction,
amount: u32,
) -> Result<(), AdapterError> {
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
use accessibility_sys::{
kAXErrorSuccess, AXUIElementPerformAction, AXUIElementPostKeyboardEvent,
AXUIElementSetAttributeValue,
};
use agent_desktop_core::action::Direction;
use core_foundation::{base::TCFType, string::CFString};
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
let scroll_area = find_scroll_area(el);
let target = scroll_area.as_ref().unwrap_or(el);
let (bar_orientation, action_name) = match direction {
let scroll_visible = CFString::new("AXScrollToVisible");
if unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) }
== kAXErrorSuccess
{
return Ok(());
}
let (bar_attr, inc_action) = match direction {
Direction::Down => ("AXVerticalScrollBar", "AXIncrement"),
Direction::Up => ("AXVerticalScrollBar", "AXDecrement"),
Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"),
Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"),
};
if let Some(scroll_bar) = get_scroll_bar(target, bar_orientation) {
let ax_action = CFString::new(action_name);
if let Some(bar) = get_scroll_bar(target, bar_attr) {
let ax_action = CFString::new(inc_action);
let mut ok = true;
for _ in 0..amount {
let err =
unsafe { AXUIElementPerformAction(scroll_bar.0, ax_action.as_concrete_TypeRef()) };
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("{action_name} on scroll bar failed (err={err})"),
));
if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) }
!= kAXErrorSuccess
{
ok = false;
break;
}
}
return Ok(());
if ok {
return Ok(());
}
}
let scroll_action_name = match direction {
let page_action = match direction {
Direction::Down => "AXScrollDownByPage",
Direction::Up => "AXScrollUpByPage",
Direction::Right => "AXScrollRightByPage",
Direction::Left => "AXScrollLeftByPage",
};
if crate::actions::dispatch::has_ax_action(target, scroll_action_name) {
let ax_action = CFString::new(scroll_action_name);
if crate::actions::dispatch::has_ax_action(target, page_action) {
let ax = CFString::new(page_action);
for _ in 0..amount {
unsafe { AXUIElementPerformAction(target.0, ax_action.as_concrete_TypeRef()) };
unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) };
}
return Ok(());
}
if let Some(bar) = get_scroll_bar(target, bar_attr) {
if try_scroll_bar_value_shift(&bar, direction, amount) {
return Ok(());
}
if try_scroll_bar_sub_elements(&bar, direction) {
return Ok(());
}
}
if try_focus_child_in_direction(target, direction) {
return Ok(());
}
if try_select_row_in_direction(target, direction) {
return Ok(());
}
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let keycode: u16 = match direction {
Direction::Down => 121,
Direction::Up => 116,
Direction::Right => 124,
Direction::Left => 123,
};
let cf_focused = CFString::new("AXFocused");
unsafe {
AXUIElementSetAttributeValue(
target.0,
cf_focused.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
std::thread::sleep(std::time::Duration::from_millis(50));
let app = crate::tree::element_for_pid(pid);
for _ in 0..amount {
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, keycode, true);
AXUIElementPostKeyboardEvent(app.0, 0, keycode, false);
};
}
return Ok(());
}
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
if let Some(b) = crate::tree::read_bounds(target) {
let (dy, dx) = match direction {
Direction::Down => (-(amount as i32) * 5, 0),
Direction::Up => (amount as i32 * 5, 0),
Direction::Right => (0, -(amount as i32) * 5),
Direction::Left => (0, amount as i32 * 5),
};
return crate::input::mouse::synthesize_scroll_at(
b.x + b.width / 2.0,
b.y + b.height / 2.0,
dy,
dx,
);
}
}
Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"No scroll bar or scroll action found on this element",
"No scroll mechanism found on element",
)
.with_suggestion("Element may not be scrollable, or try scrolling the parent container."))
.with_suggestion("Element may not be scrollable, or try the parent container."))
}
#[cfg(target_os = "macos")]
fn try_scroll_bar_value_shift(
bar: &AXElement,
direction: &agent_desktop_core::action::Direction,
amount: u32,
) -> bool {
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
use agent_desktop_core::action::Direction;
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
if !crate::actions::activate::is_attr_settable(bar, "AXValue") {
return false;
}
let current = read_scroll_bar_value(bar).unwrap_or(0.0);
let delta = 0.1 * amount as f64;
let new_val = match direction {
Direction::Down | Direction::Right => (current + delta).min(1.0),
Direction::Up | Direction::Left => (current - delta).max(0.0),
};
let cf_num = CFNumber::from(new_val as f32);
let cf_attr = CFString::new("AXValue");
let err = unsafe {
AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef())
};
err == kAXErrorSuccess
}
#[cfg(target_os = "macos")]
fn read_scroll_bar_value(bar: &AXElement) -> Option<f64> {
use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue};
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
let cf_attr = CFString::new("AXValue");
let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut();
let err =
unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) };
if err != kAXErrorSuccess || value.is_null() {
return None;
}
let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) };
cf.downcast::<CFNumber>().and_then(|n| n.to_f64())
}
#[cfg(target_os = "macos")]
fn try_scroll_bar_sub_elements(
bar: &AXElement,
direction: &agent_desktop_core::action::Direction,
) -> bool {
use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction};
use agent_desktop_core::action::Direction;
use core_foundation::{base::TCFType, string::CFString};
let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default();
let target_subroles = match direction {
Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"],
Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"],
};
let press = CFString::new("AXPress");
for child in &children {
let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default();
if target_subroles.iter().any(|t| *t == sr)
&& unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) }
== kAXErrorSuccess
{
return true;
}
}
false
}
#[cfg(target_os = "macos")]
fn try_focus_child_in_direction(
scroll_area: &AXElement,
_direction: &agent_desktop_core::action::Direction,
) -> bool {
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default();
let child = match children.first() {
Some(c) => c,
None => return false,
};
let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default();
let target = match grandchildren.last() {
Some(t) => t,
None => return false,
};
let cf_attr = CFString::new("AXFocused");
let err = unsafe {
AXUIElementSetAttributeValue(
target.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
#[cfg(target_os = "macos")]
fn try_select_row_in_direction(
scroll_area: &AXElement,
_direction: &agent_desktop_core::action::Direction,
) -> bool {
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
string::CFString,
};
let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default();
for child in &children {
let role = crate::tree::copy_string_attr(child, kAXRoleAttribute);
if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) {
continue;
}
if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") {
continue;
}
let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default();
if let Some(last) = rows.last() {
unsafe { CFRetain(last.0 as CFTypeRef) };
let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_as_cftype]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
child.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
if err == kAXErrorSuccess {
return true;
}
}
}
false
}
#[cfg(target_os = "macos")]
fn find_scroll_area(el: &AXElement) -> Option<AXElement> {
use accessibility_sys::kAXRoleAttribute;
let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?;
if role == "AXScrollArea" {
return Some(el.clone());
}
let parent = crate::tree::copy_element_attr(el, "AXParent")?;
let parent_role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute)?;
if parent_role == "AXScrollArea" {
return Some(parent);
let mut current = crate::tree::copy_element_attr(el, "AXParent")?;
for _ in 0..5 {
let r = crate::tree::copy_string_attr(&current, kAXRoleAttribute)?;
if r == "AXScrollArea" {
return Some(current);
}
current = crate::tree::copy_element_attr(&current, "AXParent")?;
}
None
}

View file

@ -129,6 +129,33 @@ mod imp {
MouseButton::Middle => CGEventType::OtherMouseUp,
}
}
pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> {
use core_graphics::geometry::CGPoint;
extern "C" {
fn CGEventCreateScrollWheelEvent(
source: *const std::ffi::c_void,
units: u32,
wheel_count: u32,
wheel1: i32,
wheel2: i32,
) -> *mut std::ffi::c_void;
fn CGEventSetLocation(event: *mut std::ffi::c_void, point: CGPoint);
fn CGEventPost(tap: u32, event: *mut std::ffi::c_void);
}
let event = unsafe { CGEventCreateScrollWheelEvent(std::ptr::null(), 0, 2, dy, dx) };
if event.is_null() {
return Err(AdapterError::internal("scroll event creation failed"));
}
unsafe {
CGEventSetLocation(event, CGPoint::new(x, y));
CGEventPost(0, event);
core_foundation::base::CFRelease(event as _);
}
Ok(())
}
}
#[cfg(not(target_os = "macos"))]
@ -142,6 +169,10 @@ mod imp {
pub fn synthesize_drag(_params: DragParams) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("drag"))
}
pub fn synthesize_scroll_at(_x: f64, _y: f64, _dy: i32, _dx: i32) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("scroll"))
}
}
pub use imp::{synthesize_drag, synthesize_mouse};
pub use imp::{synthesize_drag, synthesize_mouse, synthesize_scroll_at};

View file

@ -1,5 +1,39 @@
use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo};
#[cfg(target_os = "macos")]
pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> {
let mut pid: i32 = 0;
let err = unsafe { accessibility_sys::AXUIElementGetPid(el.0, &mut pid) };
if err == accessibility_sys::kAXErrorSuccess {
Some(pid)
} else {
None
}
}
#[cfg(target_os = "macos")]
pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> {
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
let app_el = crate::tree::element_for_pid(pid);
let frontmost_attr = CFString::new("AXFrontmost");
let err = unsafe {
AXUIElementSetAttributeValue(
app_el.0,
frontmost_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::internal(format!(
"Failed to focus app pid={pid}"
)));
}
std::thread::sleep(std::time::Duration::from_millis(50));
Ok(())
}
#[cfg(target_os = "macos")]
pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> {
use accessibility_sys::{
@ -162,22 +196,23 @@ fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool {
let Some(bar_items) = crate::tree::copy_ax_array(&menu_bar, "AXChildren") else {
return false;
};
if let Some(app_menu) = bar_items.first() {
if let Some(menus) = crate::tree::copy_ax_array(app_menu, "AXChildren") {
for menu in &menus {
if let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") {
for item in &items {
let title = crate::tree::copy_string_attr(item, "AXTitle");
if let Some(t) = &title {
if t.contains("Quit") || t.contains("quit") {
let press = CFString::new("AXPress");
let err = unsafe {
AXUIElementPerformAction(item.0, press.as_concrete_TypeRef())
};
return err == kAXErrorSuccess;
}
}
}
for bar_item in bar_items.iter().skip(1) {
let Some(menus) = crate::tree::copy_ax_array(bar_item, "AXChildren") else {
continue;
};
for menu in &menus {
let Some(items) = crate::tree::copy_ax_array(menu, "AXChildren") else {
continue;
};
for item in &items {
let Some(t) = crate::tree::copy_string_attr(item, "AXTitle") else {
continue;
};
if t.starts_with("Quit") {
let press = CFString::new("AXPress");
let err =
unsafe { AXUIElementPerformAction(item.0, press.as_concrete_TypeRef()) };
return err == kAXErrorSuccess;
}
}
}

View file

@ -45,7 +45,7 @@ pub fn build_subtree(
el: &AXElement,
depth: u8,
max_depth: u8,
include_bounds: bool,
_include_bounds: bool,
ancestors: &mut FxHashSet<usize>,
) -> Option<AccessibilityNode> {
if depth > max_depth || depth >= ABSOLUTE_MAX_DEPTH {
@ -81,18 +81,14 @@ pub fn build_subtree(
states.push("disabled".into());
}
let bounds = if include_bounds {
read_bounds(el)
} else {
None
};
let bounds = read_bounds(el);
let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default();
let name = name.or_else(|| label_from_children(&children_raw));
let children = children_raw
.into_iter()
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, include_bounds, ancestors))
.filter_map(|child| build_subtree(&child, depth + 1, max_depth, _include_bounds, ancestors))
.collect();
ancestors.remove(&ptr_key);

View file

@ -41,7 +41,22 @@ pub fn find_element_recursive(
(None, None) => true,
_ => false,
};
if name_match {
let bounds_match = match entry.bounds_hash {
Some(expected) => {
let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash());
tracing::debug!(
role = normalized,
?elem_name,
?actual,
expected,
name_match,
"resolve candidate"
);
actual.map(|h| h == expected).unwrap_or(false)
}
None => true,
};
if name_match && bounds_match {
ancestors.remove(&ptr_key);
unsafe { CFRetain(el.0 as CFTypeRef) };
return Ok(NativeHandle::from_ptr(el.0 as *const _));