From cddc5d3547f058a78f8b398fa982e39a1fcbf6b1 Mon Sep 17 00:00:00 2001 From: Lahfir Date: Sat, 21 Feb 2026 15:40:03 -0800 Subject: [PATCH] feat: AX-first right-click chain with inline context menu capture - Add 7-step right-click chain: AXShowMenu direct, focus-app + AXShowMenu, select + AXShowMenu, focus-element + AXShowMenu, parent AXShowMenu, child AXShowMenu, CGEvent last resort - Key fix: AXShowMenu returns -25204 (CannotComplete) when app isn't frontmost; ensure_app_focused via AX resolves this - Right-click command now returns full context menu tree inline with ref_ids on all menuitems for immediate agent interaction - Remove surface-based menu detection (menu lives in regular tree) - Clean up debug tracing from resolve.rs --- crates/core/src/commands/right_click.rs | 35 +++++-- crates/macos/src/actions/activate.rs | 121 +++++++++++++++++++----- crates/macos/src/tree/resolve.rs | 8 -- 3 files changed, 122 insertions(+), 42 deletions(-) diff --git a/crates/core/src/commands/right_click.rs b/crates/core/src/commands/right_click.rs index 17e22a5d..e1e8b5cc 100644 --- a/crates/core/src/commands/right_click.rs +++ b/crates/core/src/commands/right_click.rs @@ -1,8 +1,9 @@ use crate::{ action::Action, - adapter::{PlatformAdapter, SnapshotSurface}, + adapter::{PlatformAdapter, TreeOptions}, commands::helpers::resolve_ref, error::AppError, + node::AccessibilityNode, snapshot, }; use serde_json::Value; @@ -16,18 +17,32 @@ pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result Option<&AccessibilityNode> { + if node.role == "menu" && node.children.iter().any(|c| c.role == "menuitem") { + return Some(node); + } + for child in &node.children { + if let Some(menu) = find_context_menu(child) { + return Some(menu); + } + } + None +} diff --git a/crates/macos/src/actions/activate.rs b/crates/macos/src/actions/activate.rs index 182525a1..96ecb600 100644 --- a/crates/macos/src/actions/activate.rs +++ b/crates/macos/src/actions/activate.rs @@ -78,13 +78,102 @@ mod imp { } pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> { - let actions = list_ax_actions(el); - if try_action_from_list(el, &actions, &["AXShowMenu"]) { + if ax_show_menu(el) { + return Ok(()); + } + + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + std::thread::sleep(std::time::Duration::from_millis(50)); + if ax_show_menu(el) { + return Ok(()); + } + } + + if try_select_then_show_menu(el) { + return Ok(()); + } + + if try_focus_then_show_menu(el) { + return Ok(()); + } + + if try_parent_show_menu(el) { + return Ok(()); + } + + if try_child_show_menu(el) { return Ok(()); } crate::actions::dispatch::click_via_bounds(el, MouseButton::Right, 1) } + fn ax_show_menu(el: &AXElement) -> bool { + let show = CFString::new("AXShowMenu"); + let err = unsafe { AXUIElementPerformAction(el.0, show.as_concrete_TypeRef()) }; + err == kAXErrorSuccess + } + + fn try_select_then_show_menu(el: &AXElement) -> bool { + if !is_attr_settable(el, "AXSelected") { + return false; + } + let cf_attr = CFString::new("AXSelected"); + let err = unsafe { + AXUIElementSetAttributeValue( + el.0, + cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if err != kAXErrorSuccess { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_show_menu(el) + } + + fn try_focus_then_show_menu(el: &AXElement) -> bool { + let cf_attr = CFString::new(kAXFocusedAttribute); + let err = unsafe { + AXUIElementSetAttributeValue( + el.0, + cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if err != kAXErrorSuccess { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_show_menu(el) + } + + fn try_parent_show_menu(el: &AXElement) -> bool { + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..3 { + let ancestor = match ¤t { + Some(a) => a, + None => return false, + }; + if ax_show_menu(ancestor) { + return true; + } + current = crate::tree::copy_element_attr(ancestor, "AXParent"); + } + false + } + + fn try_child_show_menu(el: &AXElement) -> bool { + let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); + for child in children.iter().take(5) { + if ax_show_menu(child) { + return true; + } + } + false + } + /// Three smart_activate calls with gaps, then CGEvent triple-click. pub fn smart_triple_activate(el: &AXElement) -> Result<(), AdapterError> { for _ in 0..3 { @@ -202,15 +291,10 @@ mod imp { fn try_child_activation(el: &AXElement) -> bool { let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - let targets = ["AXPress", "AXConfirm", "AXOpen", "AXShowDefaultUI"]; for child in children.iter().take(3) { let child_actions = list_ax_actions(child); - for target in &targets { - if child_actions.iter().any(|a| a == target) { - let action = CFString::new(target); - unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) }; - return true; - } + if try_action_from_list(child, &child_actions, &["AXPress", "AXConfirm", "AXOpen"]) { + return true; } } false @@ -219,17 +303,10 @@ mod imp { fn try_parent_activation(el: &AXElement) -> bool { let mut current = crate::tree::copy_element_attr(el, "AXParent"); for _ in 0..2 { - let ancestor = match ¤t { - Some(a) => a, - None => return false, - }; - for action_name in &["AXPress", "AXConfirm"] { - let action = CFString::new(action_name); - if unsafe { AXUIElementPerformAction(ancestor.0, action.as_concrete_TypeRef()) } - == kAXErrorSuccess - { - return true; - } + let ancestor = match ¤t { Some(a) => a, None => return false }; + let actions = list_ax_actions(ancestor); + if try_action_from_list(ancestor, &actions, &["AXPress", "AXConfirm"]) { + return true; } current = crate::tree::copy_element_attr(ancestor, "AXParent"); } @@ -266,7 +343,6 @@ mod imp { fn try_keyboard_activate(el: &AXElement) -> bool { use accessibility_sys::AXUIElementPostKeyboardEvent; - let cf_focused = CFString::new(kAXFocusedAttribute); let err = unsafe { AXUIElementSetAttributeValue( @@ -279,7 +355,6 @@ mod imp { return false; } std::thread::sleep(std::time::Duration::from_millis(50)); - let pid = match crate::system::app_ops::pid_from_element(el) { Some(p) => p, None => return false, @@ -291,8 +366,6 @@ mod imp { }; true } - - } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/tree/resolve.rs b/crates/macos/src/tree/resolve.rs index a885f3a7..e7a1b438 100644 --- a/crates/macos/src/tree/resolve.rs +++ b/crates/macos/src/tree/resolve.rs @@ -44,14 +44,6 @@ pub fn find_element_recursive( let bounds_match = match entry.bounds_hash { Some(expected) => { let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash()); - tracing::debug!( - role = normalized, - ?elem_name, - ?actual, - expected, - name_match, - "resolve candidate" - ); actual.map(|h| h == expected).unwrap_or(false) } None => true,