mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-15 11:37:47 +00:00
feat: AX-first right-click chain with inline context menu capture
- Add 7-step right-click chain: AXShowMenu direct, focus-app + AXShowMenu, select + AXShowMenu, focus-element + AXShowMenu, parent AXShowMenu, child AXShowMenu, CGEvent last resort - Key fix: AXShowMenu returns -25204 (CannotComplete) when app isn't frontmost; ensure_app_focused via AX resolves this - Right-click command now returns full context menu tree inline with ref_ids on all menuitems for immediate agent interaction - Remove surface-based menu detection (menu lives in regular tree) - Clean up debug tracing from resolve.rs
This commit is contained in:
parent
595ccb6cc4
commit
cddc5d3547
3 changed files with 122 additions and 42 deletions
|
|
@ -1,8 +1,9 @@
|
|||
use crate::{
|
||||
action::Action,
|
||||
adapter::{PlatformAdapter, SnapshotSurface},
|
||||
adapter::{PlatformAdapter, TreeOptions},
|
||||
commands::helpers::resolve_ref,
|
||||
error::AppError,
|
||||
node::AccessibilityNode,
|
||||
snapshot,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
|
@ -16,18 +17,32 @@ pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result<Va
|
|||
let result = adapter.execute_action(&handle, Action::RightClick)?;
|
||||
let mut response = serde_json::to_value(&result)?;
|
||||
|
||||
std::thread::sleep(std::time::Duration::from_millis(250));
|
||||
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||
|
||||
if let Some(menu_tree) = snapshot::append_surface_refs(
|
||||
adapter,
|
||||
entry.pid,
|
||||
entry.source_app.as_deref(),
|
||||
SnapshotSurface::Menu,
|
||||
) {
|
||||
if let Ok(menu_json) = serde_json::to_value(&menu_tree) {
|
||||
response["menu"] = menu_json;
|
||||
let opts = TreeOptions {
|
||||
interactive_only: true,
|
||||
..Default::default()
|
||||
};
|
||||
if let Ok(snap) = snapshot::build(adapter, &opts, entry.source_app.as_deref(), None) {
|
||||
snap.refmap.save().ok();
|
||||
if let Some(menu) = find_context_menu(&snap.tree) {
|
||||
if let Ok(menu_json) = serde_json::to_value(menu) {
|
||||
response["menu"] = menu_json;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn find_context_menu(node: &AccessibilityNode) -> Option<&AccessibilityNode> {
|
||||
if node.role == "menu" && node.children.iter().any(|c| c.role == "menuitem") {
|
||||
return Some(node);
|
||||
}
|
||||
for child in &node.children {
|
||||
if let Some(menu) = find_context_menu(child) {
|
||||
return Some(menu);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,13 +78,102 @@ mod imp {
|
|||
}
|
||||
|
||||
pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> {
|
||||
let actions = list_ax_actions(el);
|
||||
if try_action_from_list(el, &actions, &["AXShowMenu"]) {
|
||||
if ax_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
if ax_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
if try_select_then_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if try_focus_then_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if try_parent_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if try_child_show_menu(el) {
|
||||
return Ok(());
|
||||
}
|
||||
crate::actions::dispatch::click_via_bounds(el, MouseButton::Right, 1)
|
||||
}
|
||||
|
||||
fn ax_show_menu(el: &AXElement) -> bool {
|
||||
let show = CFString::new("AXShowMenu");
|
||||
let err = unsafe { AXUIElementPerformAction(el.0, show.as_concrete_TypeRef()) };
|
||||
err == kAXErrorSuccess
|
||||
}
|
||||
|
||||
fn try_select_then_show_menu(el: &AXElement) -> bool {
|
||||
if !is_attr_settable(el, "AXSelected") {
|
||||
return false;
|
||||
}
|
||||
let cf_attr = CFString::new("AXSelected");
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_show_menu(el)
|
||||
}
|
||||
|
||||
fn try_focus_then_show_menu(el: &AXElement) -> bool {
|
||||
let cf_attr = CFString::new(kAXFocusedAttribute);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
el.0,
|
||||
cf_attr.as_concrete_TypeRef(),
|
||||
CFBoolean::true_value().as_CFTypeRef(),
|
||||
)
|
||||
};
|
||||
if err != kAXErrorSuccess {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
ax_show_menu(el)
|
||||
}
|
||||
|
||||
fn try_parent_show_menu(el: &AXElement) -> bool {
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent");
|
||||
for _ in 0..3 {
|
||||
let ancestor = match ¤t {
|
||||
Some(a) => a,
|
||||
None => return false,
|
||||
};
|
||||
if ax_show_menu(ancestor) {
|
||||
return true;
|
||||
}
|
||||
current = crate::tree::copy_element_attr(ancestor, "AXParent");
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn try_child_show_menu(el: &AXElement) -> bool {
|
||||
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
|
||||
for child in children.iter().take(5) {
|
||||
if ax_show_menu(child) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Three smart_activate calls with gaps, then CGEvent triple-click.
|
||||
pub fn smart_triple_activate(el: &AXElement) -> Result<(), AdapterError> {
|
||||
for _ in 0..3 {
|
||||
|
|
@ -202,15 +291,10 @@ mod imp {
|
|||
|
||||
fn try_child_activation(el: &AXElement) -> bool {
|
||||
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
|
||||
let targets = ["AXPress", "AXConfirm", "AXOpen", "AXShowDefaultUI"];
|
||||
for child in children.iter().take(3) {
|
||||
let child_actions = list_ax_actions(child);
|
||||
for target in &targets {
|
||||
if child_actions.iter().any(|a| a == target) {
|
||||
let action = CFString::new(target);
|
||||
unsafe { AXUIElementPerformAction(child.0, action.as_concrete_TypeRef()) };
|
||||
return true;
|
||||
}
|
||||
if try_action_from_list(child, &child_actions, &["AXPress", "AXConfirm", "AXOpen"]) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
|
|
@ -219,17 +303,10 @@ mod imp {
|
|||
fn try_parent_activation(el: &AXElement) -> bool {
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent");
|
||||
for _ in 0..2 {
|
||||
let ancestor = match ¤t {
|
||||
Some(a) => a,
|
||||
None => return false,
|
||||
};
|
||||
for action_name in &["AXPress", "AXConfirm"] {
|
||||
let action = CFString::new(action_name);
|
||||
if unsafe { AXUIElementPerformAction(ancestor.0, action.as_concrete_TypeRef()) }
|
||||
== kAXErrorSuccess
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let ancestor = match ¤t { Some(a) => a, None => return false };
|
||||
let actions = list_ax_actions(ancestor);
|
||||
if try_action_from_list(ancestor, &actions, &["AXPress", "AXConfirm"]) {
|
||||
return true;
|
||||
}
|
||||
current = crate::tree::copy_element_attr(ancestor, "AXParent");
|
||||
}
|
||||
|
|
@ -266,7 +343,6 @@ mod imp {
|
|||
|
||||
fn try_keyboard_activate(el: &AXElement) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
let cf_focused = CFString::new(kAXFocusedAttribute);
|
||||
let err = unsafe {
|
||||
AXUIElementSetAttributeValue(
|
||||
|
|
@ -279,7 +355,6 @@ mod imp {
|
|||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
|
|
@ -291,8 +366,6 @@ mod imp {
|
|||
};
|
||||
true
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -44,14 +44,6 @@ pub fn find_element_recursive(
|
|||
let bounds_match = match entry.bounds_hash {
|
||||
Some(expected) => {
|
||||
let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash());
|
||||
tracing::debug!(
|
||||
role = normalized,
|
||||
?elem_name,
|
||||
?actual,
|
||||
expected,
|
||||
name_match,
|
||||
"resolve candidate"
|
||||
);
|
||||
actual.map(|h| h == expected).unwrap_or(false)
|
||||
}
|
||||
None => true,
|
||||
|
|
|
|||
Loading…
Reference in a new issue