Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-07-27 01:22:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge pull request #8 from lahfir/refactor/centralized-ax-chain-executor

Browse source 
refactor: centralize AX chain executor with verbose logging
This commit is contained in:
Lahfir 2026-02-23 04:42:47 -08:00 committed by GitHub
commit 7f3232a012
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 1237 additions and 631 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
crates/core/src/commands/focus_window.rs
View file

@ -34,10 +34,13 @@ pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result<V
};
let window = window.ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No matching window found",
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No matching window found",
)
.with_suggestion("Run 'list-windows' to see available windows and their IDs."),
)
})?;
adapter.focus_window(&window)?;

8
crates/core/src/commands/helpers.rs
View file

@ -18,7 +18,15 @@ pub fn resolve_ref(
.get(ref_id)
.ok_or_else(|| AppError::stale_ref(ref_id))?
.clone();
tracing::debug!(
"resolve: {} -> pid={} role={} name={:?}",
ref_id,
entry.pid,
entry.role,
entry.name.as_deref().unwrap_or("(none)")
);
let handle = adapter.resolve_element(&entry)?;
tracing::debug!("resolve: {} resolved successfully", ref_id);
Ok((entry, handle))
}

15
crates/core/src/commands/snapshot.rs
View file

@ -16,6 +16,14 @@ pub struct SnapshotArgs {
}
pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
tracing::debug!(
"tree: snapshot app={:?} window_id={:?} max_depth={} interactive_only={}",
args.app.as_deref().unwrap_or("(focused)"),
args.window_id.as_deref().unwrap_or("(auto)"),
args.max_depth,
args.interactive_only
);
let opts = crate::adapter::TreeOptions {
max_depth: args.max_depth,
include_bounds: args.include_bounds,
@ -35,6 +43,13 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result<Valu
let tree = serde_json::to_value(&result.tree)?;
let win = &result.window;
tracing::debug!(
"tree: snapshot complete app={:?} window={:?} refs={}",
win.app,
win.title,
ref_count
);
Ok(json!({
"app": win.app,
"window": {

37
crates/core/src/snapshot.rs
View file

@ -45,10 +45,13 @@ pub fn build(
let window = if let Some(wid) = window_id {
windows.into_iter().find(|w| w.id == wid).ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
format!("No window with id {wid}"),
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
format!("No window with id {wid}"),
)
.with_suggestion("Run 'list-windows' to see available window IDs."),
)
})?
} else if let Some(app) = app_name {
windows
@ -64,17 +67,27 @@ pub fn build(
.and_then(|ws| ws.into_iter().next())
})
.ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::AppNotFound,
format!("No window found for app '{app}'"),
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::AppNotFound,
format!("No window found for app '{app}'"),
)
.with_suggestion(
"Verify the app is running. Use 'list-apps' to see running applications.",
),
)
})?
} else {
windows.into_iter().find(|w| w.is_focused).ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No focused window found. Use --app to specify an application.",
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No focused window found",
)
.with_suggestion(
"Use --app to specify an application, or click a window to focus it.",
),
)
})?
};

399
crates/macos/src/actions/activate.rs
View file

@ -1,399 +0,0 @@
use agent_desktop_core::{action::MouseButton, error::AdapterError};
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute, AXUIElementCopyActionNames,
AXUIElementIsAttributeSettable, AXUIElementPerformAction, AXUIElementSetAttributeValue,
};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
boolean::CFBoolean,
string::CFString,
};
use std::os::raw::c_uchar;
pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> {
let scroll_action = CFString::new("AXScrollToVisible");
unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) };
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXPress"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXConfirm"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXOpen"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXPick"]) {
return Ok(());
}
if try_show_alternate_ui(el) {
return Ok(());
}
if try_child_activation(el) {
return Ok(());
}
if try_set_selected(el) {
return Ok(());
}
if try_select_via_parent(el) {
return Ok(());
}
if try_custom_actions(el) {
return Ok(());
}
if try_focus_then_activate(el) {
return Ok(());
}
if try_keyboard_activate(el) {
return Ok(());
}
if try_parent_activation(el) {
return Ok(());
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 1)
}
/// AXOpen first, then two smart_activate calls with a gap, then CGEvent double-click.
pub fn smart_double_activate(el: &AXElement) -> Result<(), AdapterError> {
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXOpen"]) {
return Ok(());
}
let _ = smart_activate(el);
std::thread::sleep(std::time::Duration::from_millis(50));
let _ = smart_activate(el);
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
}
pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> {
if ax_show_menu(el) {
return Ok(());
}
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
if ax_show_menu(el) {
return Ok(());
}
}
if try_select_then_show_menu(el) {
return Ok(());
}
if try_focus_then_show_menu(el) {
return Ok(());
}
if try_parent_show_menu(el) {
return Ok(());
}
if try_child_show_menu(el) {
return Ok(());
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Right, 1)
}
fn ax_show_menu(el: &AXElement) -> bool {
let show = CFString::new("AXShowMenu");
let err = unsafe { AXUIElementPerformAction(el.0, show.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_select_then_show_menu(el: &AXElement) -> bool {
if !is_attr_settable(el, "AXSelected") {
return false;
}
let cf_attr = CFString::new("AXSelected");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_show_menu(el)
}
fn try_focus_then_show_menu(el: &AXElement) -> bool {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_show_menu(el)
}
fn try_parent_show_menu(el: &AXElement) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..3 {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
if ax_show_menu(ancestor) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
fn try_child_show_menu(el: &AXElement) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(5) {
if ax_show_menu(child) {
return true;
}
}
false
}
/// Three smart_activate calls with gaps, then CGEvent triple-click.
pub fn smart_triple_activate(el: &AXElement) -> Result<(), AdapterError> {
for _ in 0..3 {
let _ = smart_activate(el);
std::thread::sleep(std::time::Duration::from_millis(30));
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3)
}
fn list_ax_actions(el: &AXElement) -> Vec<String> {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return Vec::new();
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let mut result = Vec::new();
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
result.push(name.to_string());
}
}
result
}
pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool {
let cf_attr = CFString::new(attr);
let mut settable: c_uchar = 0;
let err = unsafe {
AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable)
};
err == kAXErrorSuccess && settable != 0
}
fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool {
for target in targets {
if actions.iter().any(|a| a == target) {
let action = CFString::new(target);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err == kAXErrorSuccess {
return true;
}
}
}
false
}
fn try_set_selected(el: &AXElement) -> bool {
if !is_attr_settable(el, "AXSelected") {
return false;
}
let cf_attr = CFString::new("AXSelected");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_select_via_parent(el: &AXElement) -> bool {
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
Some(p) => p,
None => return false,
};
let parent_role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
Some(r) => r,
None => return false,
};
if !matches!(parent_role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_as_cftype = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_as_cftype]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_focus_then_activate(el: &AXElement) -> bool {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let confirm = CFString::new("AXConfirm");
if unsafe { AXUIElementPerformAction(el.0, confirm.as_concrete_TypeRef()) }
== kAXErrorSuccess
{
return true;
}
let press = CFString::new("AXPress");
let err = unsafe { AXUIElementPerformAction(el.0, press.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_child_activation(el: &AXElement) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(3) {
let child_actions = list_ax_actions(child);
if try_action_from_list(child, &child_actions, &["AXPress", "AXConfirm", "AXOpen"]) {
return true;
}
}
false
}
fn try_parent_activation(el: &AXElement) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..2 {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
let actions = list_ax_actions(ancestor);
if try_action_from_list(ancestor, &actions, &["AXPress", "AXConfirm"]) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
fn try_show_alternate_ui(el: &AXElement) -> bool {
let actions = list_ax_actions(el);
if !actions.iter().any(|a| a == "AXShowAlternateUI") {
return false;
}
let action = CFString::new("AXShowAlternateUI");
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
std::thread::sleep(std::time::Duration::from_millis(100));
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(5) {
let child_actions = list_ax_actions(child);
if try_action_from_list(child, &child_actions, &["AXPress"]) {
return true;
}
}
false
}
fn try_custom_actions(el: &AXElement) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
let action = CFString::new("AXPerformCustomAction");
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_keyboard_activate(el: &AXElement) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
let cf_focused = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_focused.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
use crate::tree::AXElement;
pub fn smart_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_activate"))
}
pub fn smart_double_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_double_activate"))
}
pub fn smart_right_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_right_activate"))
}
pub fn smart_triple_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_triple_activate"))
}
pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool {
false
}
}
pub(crate) use imp::{
is_attr_settable, smart_activate, smart_double_activate, smart_right_activate,
smart_triple_activate,
};

227
crates/macos/src/actions/ax_helpers.rs Normal file
View file

@ -0,0 +1,227 @@
use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute,
AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, AXUIElementPerformAction,
AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout,
};
use core_foundation::{
array::CFArray,
base::{CFType, TCFType},
boolean::CFBoolean,
string::CFString,
};
use std::os::raw::c_uchar;
pub fn try_ax_action(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
pub fn try_ax_action_retried(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err == kAXErrorSuccess {
return true;
}
if err == kAXErrorCannotComplete {
std::thread::sleep(std::time::Duration::from_millis(100));
let retry = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
return retry == kAXErrorSuccess;
}
false
}
pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool {
let cf_attr = CFString::new(attr);
let cf_val = if value {
CFBoolean::true_value()
} else {
CFBoolean::false_value()
};
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
err == kAXErrorSuccess
}
pub fn set_ax_string_or_err(
el: &AXElement,
attr: &str,
value: &str,
) -> Result<(), AdapterError> {
let cf_attr = CFString::new(attr);
let cf_val = CFString::new(value);
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::ActionFailed,
format!("AXSetAttributeValue({attr}) failed (err={err})"),
)
.with_suggestion("Attribute may be read-only. Try 'click' or 'type' instead."));
}
Ok(())
}
pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool {
let cf_attr = CFString::new(attr);
let mut settable: c_uchar = 0;
let err = unsafe {
AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable)
};
err == kAXErrorSuccess && settable != 0
}
pub fn list_ax_actions(el: &AXElement) -> Vec<String> {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return Vec::new();
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let mut result = Vec::with_capacity(actions.len() as usize);
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
result.push(name.to_string());
}
}
result
}
pub fn has_ax_action(el: &AXElement, target: &str) -> bool {
list_ax_actions(el).iter().any(|a| a == target)
}
pub fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool {
for target in targets {
if actions.iter().any(|a| a == target) && try_ax_action(el, target) {
return true;
}
}
false
}
pub fn try_each_child(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(limit) {
if f(child) {
return true;
}
}
false
}
pub fn try_each_ancestor(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..limit {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
if f(ancestor) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
pub fn ensure_visible(el: &AXElement) {
let action = CFString::new("AXScrollToVisible");
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
}
pub fn set_messaging_timeout(el: &AXElement, seconds: f32) {
unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) };
}
pub fn ax_focus(el: &AXElement) -> bool {
set_ax_bool(el, kAXFocusedAttribute, true)
}
pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> {
set_ax_string_or_err(el, kAXValueAttribute, val)
}
pub fn ax_press(el: &AXElement) -> bool {
try_ax_action(el, "AXPress")
}
pub fn element_role(el: &AXElement) -> Option<String> {
use accessibility_sys::kAXRoleAttribute;
crate::tree::copy_string_attr(el, kAXRoleAttribute)
.map(|r| crate::tree::roles::ax_role_to_str(&r).to_string())
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
use crate::tree::AXElement;
pub fn try_ax_action(_el: &AXElement, _name: &str) -> bool {
false
}
pub fn try_ax_action_retried(_el: &AXElement, _name: &str) -> bool {
false
}
pub fn set_ax_bool(_el: &AXElement, _attr: &str, _value: bool) -> bool {
false
}
pub fn set_ax_string_or_err(
_el: &AXElement,
_attr: &str,
_value: &str,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("set_ax_string_or_err"))
}
pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool {
false
}
pub fn list_ax_actions(_el: &AXElement) -> Vec<String> {
Vec::new()
}
pub fn has_ax_action(_el: &AXElement, _target: &str) -> bool {
false
}
pub fn try_action_from_list(_el: &AXElement, _actions: &[String], _targets: &[&str]) -> bool {
false
}
pub fn try_each_child(_el: &AXElement, _f: impl Fn(&AXElement) -> bool, _limit: usize) -> bool {
false
}
pub fn try_each_ancestor(
_el: &AXElement,
_f: impl Fn(&AXElement) -> bool,
_limit: usize,
) -> bool {
false
}
pub fn ensure_visible(_el: &AXElement) {}
pub fn set_messaging_timeout(_el: &AXElement, _seconds: f32) {}
pub fn ax_focus(_el: &AXElement) -> bool {
false
}
pub fn ax_set_value(_el: &AXElement, _val: &str) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("ax_set_value"))
}
pub fn ax_press(_el: &AXElement) -> bool {
false
}
pub fn element_role(_el: &AXElement) -> Option<String> {
None
}
}
pub(crate) use imp::{
ax_focus, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action,
is_attr_settable, list_ax_actions, set_ax_bool, set_ax_string_or_err, set_messaging_timeout,
try_action_from_list, try_ax_action, try_ax_action_retried, try_each_ancestor, try_each_child,
};

209
crates/macos/src/actions/chain.rs Normal file
View file

@ -0,0 +1,209 @@
use agent_desktop_core::action::MouseButton;
use agent_desktop_core::error::{AdapterError, ErrorCode};
use crate::actions::discovery::ElementCaps;
use crate::tree::AXElement;
#[allow(dead_code)]
pub enum ChainStep {
Action(&'static str),
SetBool {
attr: &'static str,
value: bool,
},
SetDynamic {
attr: &'static str,
},
FocusThenAction(&'static str),
FocusThenSetDynamic {
attr: &'static str,
},
FocusThenConfirmOrPress,
ChildActions {
actions: &'static [&'static str],
limit: usize,
},
AncestorActions {
actions: &'static [&'static str],
limit: usize,
},
Custom {
label: &'static str,
func: fn(&AXElement, &ElementCaps) -> bool,
},
CGClick {
button: MouseButton,
count: u32,
},
}
pub struct ChainDef {
pub pre_scroll: bool,
pub steps: &'static [ChainStep],
pub suggestion: &'static str,
}
pub struct ChainContext<'a> {
pub dynamic_value: Option<&'a str>,
}
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::ax_helpers;
use std::time::{Duration, Instant};
const CHAIN_TIMEOUT: Duration = Duration::from_secs(10);
pub fn execute_chain(
el: &AXElement,
caps: &ElementCaps,
def: &ChainDef,
ctx: &ChainContext,
) -> Result<(), AdapterError> {
let deadline = Instant::now() + CHAIN_TIMEOUT;
let total = def.steps.len();
ax_helpers::set_messaging_timeout(el, 3.0);
if def.pre_scroll {
tracing::debug!("chain: pre-scroll AXScrollToVisible");
ax_helpers::ensure_visible(el);
}
for (i, step) in def.steps.iter().enumerate() {
if Instant::now() > deadline {
tracing::debug!("chain: timeout after {i}/{total} steps");
return Err(AdapterError::timeout("Chain execution exceeded 10s"));
}
let label = step_label(step);
if execute_step(el, caps, step, ctx) {
tracing::debug!("chain: [{}/{}] {} -> success", i + 1, total, label);
return Ok(());
}
tracing::debug!("chain: [{}/{}] {} -> skip", i + 1, total, label);
}
tracing::debug!("chain: all {total} steps exhausted");
Err(
AdapterError::new(ErrorCode::ActionFailed, "All chain steps exhausted")
.with_suggestion(def.suggestion),
)
}
fn step_label(step: &ChainStep) -> &'static str {
match step {
ChainStep::Action(name) => name,
ChainStep::SetBool { attr, .. } => attr,
ChainStep::SetDynamic { attr } => attr,
ChainStep::FocusThenAction(name) => name,
ChainStep::FocusThenSetDynamic { attr } => attr,
ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress",
ChainStep::ChildActions { .. } => "ChildActions",
ChainStep::AncestorActions { .. } => "AncestorActions",
ChainStep::Custom { label, .. } => label,
ChainStep::CGClick { .. } => "CGClick",
}
}
fn execute_step(
el: &AXElement,
caps: &ElementCaps,
step: &ChainStep,
ctx: &ChainContext,
) -> bool {
match step {
ChainStep::Action(name) => ax_helpers::try_ax_action_retried(el, name),
ChainStep::SetBool { attr, value } => {
let settable = match *attr {
"AXSelected" => caps.settable_selected,
"AXDisclosing" => caps.settable_disclosing,
"AXFocused" => caps.settable_focus,
_ => ax_helpers::is_attr_settable(el, attr),
};
settable && ax_helpers::set_ax_bool(el, attr, *value)
}
ChainStep::SetDynamic { attr } => {
let value = match ctx.dynamic_value {
Some(v) => v,
None => return false,
};
ax_helpers::set_ax_string_or_err(el, attr, value).is_ok()
}
ChainStep::FocusThenAction(name) => {
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::try_ax_action_retried(el, name)
}
ChainStep::FocusThenSetDynamic { attr } => {
let value = match ctx.dynamic_value {
Some(v) => v,
None => return false,
};
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::set_ax_string_or_err(el, attr, value).is_ok()
}
ChainStep::FocusThenConfirmOrPress => {
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::try_ax_action_retried(el, "AXConfirm")
|| ax_helpers::try_ax_action_retried(el, "AXPress")
}
ChainStep::ChildActions { actions, limit } => ax_helpers::try_each_child(
el,
|child| {
let child_actions = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &child_actions, actions)
},
*limit,
),
ChainStep::AncestorActions { actions, limit } => ax_helpers::try_each_ancestor(
el,
|ancestor| {
let al = ax_helpers::list_ax_actions(ancestor);
ax_helpers::try_action_from_list(ancestor, &al, actions)
},
*limit,
),
ChainStep::Custom { label: _, func } => func(el, caps),
ChainStep::CGClick { button, count } => {
crate::actions::dispatch::click_via_bounds(el, button.clone(), *count).is_ok()
}
}
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
pub fn execute_chain(
_el: &AXElement,
_caps: &ElementCaps,
def: &ChainDef,
_ctx: &ChainContext,
) -> Result<(), AdapterError> {
Err(AdapterError::new(
ErrorCode::ActionFailed,
"Chain execution not supported on this platform",
)
.with_suggestion(def.suggestion))
}
}
pub(crate) use imp::execute_chain;

361
crates/macos/src/actions/chain_defs.rs Normal file
View file

@ -0,0 +1,361 @@
use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::{
ax_helpers,
chain::{execute_chain, ChainContext, ChainDef, ChainStep},
discovery::ElementCaps,
};
use crate::tree::AXElement;
use agent_desktop_core::action::MouseButton;
pub static CLICK_CHAIN: ChainDef = ChainDef {
pre_scroll: true,
steps: &[
ChainStep::Action("AXPress"),
ChainStep::Action("AXConfirm"),
ChainStep::Action("AXOpen"),
ChainStep::Action("AXPick"),
ChainStep::Custom {
label: "show_alternate_ui",
func: try_show_alternate_ui,
},
ChainStep::ChildActions {
actions: &["AXPress", "AXConfirm", "AXOpen"],
limit: 3,
},
ChainStep::SetBool {
attr: "AXSelected",
value: true,
},
ChainStep::Custom {
label: "select_via_parent",
func: try_select_via_parent,
},
ChainStep::Custom {
label: "custom_actions",
func: try_custom_actions,
},
ChainStep::FocusThenConfirmOrPress,
ChainStep::Custom {
label: "keyboard_activate",
func: try_keyboard_activate,
},
ChainStep::AncestorActions {
actions: &["AXPress", "AXConfirm"],
limit: 2,
},
ChainStep::CGClick {
button: MouseButton::Left,
count: 1,
},
],
suggestion: "Element may not be interactable. Try 'mouse-click --xy X,Y'.",
};
pub static RIGHT_CLICK_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXShowMenu"),
ChainStep::Custom {
label: "focus_app_show_menu",
func: focus_app_then_show_menu,
},
ChainStep::Custom {
label: "select_then_show_menu",
func: select_then_show_menu,
},
ChainStep::FocusThenAction("AXShowMenu"),
ChainStep::AncestorActions {
actions: &["AXShowMenu"],
limit: 3,
},
ChainStep::ChildActions {
actions: &["AXShowMenu"],
limit: 5,
},
ChainStep::CGClick {
button: MouseButton::Right,
count: 1,
},
],
suggestion: "Try 'mouse-click --button right --xy X,Y'.",
};
pub static EXPAND_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXExpand"),
ChainStep::SetBool {
attr: "AXDisclosing",
value: true,
},
],
suggestion: "Try 'click' to open it instead.",
};
pub static COLLAPSE_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXCollapse"),
ChainStep::SetBool {
attr: "AXDisclosing",
value: false,
},
],
suggestion: "Try 'click' to close it instead.",
};
pub static SET_VALUE_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetDynamic { attr: "AXValue" },
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
],
suggestion: "Try 'clear' then 'type', or check element is a text field.",
};
pub static CLEAR_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetDynamic { attr: "AXValue" },
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
ChainStep::Custom {
label: "select_all_delete",
func: select_all_then_delete,
},
],
suggestion: "Try 'press cmd+a' then 'press delete'.",
};
pub static FOCUS_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetBool {
attr: "AXFocused",
value: true,
},
ChainStep::Action("AXRaise"),
ChainStep::Action("AXPress"),
ChainStep::SetBool {
attr: "AXSelected",
value: true,
},
ChainStep::CGClick {
button: MouseButton::Left,
count: 1,
},
],
suggestion: "Try 'click' to focus the element instead.",
};
pub static SCROLL_TO_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXScrollToVisible"),
ChainStep::Custom {
label: "walk_parents_scroll",
func: walk_parents_and_scroll,
},
],
suggestion: "Element may not be in a scrollable container.",
};
fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 55, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, false);
AXUIElementPostKeyboardEvent(app.0, 0, 55, false);
};
std::thread::sleep(std::time::Duration::from_millis(30));
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 51, true);
AXUIElementPostKeyboardEvent(app.0, 0, 51, false);
};
true
}
fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let bounds = match crate::tree::read_bounds(el) {
Some(b) => b,
None => return false,
};
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..8 {
let parent = match &current {
Some(p) => p,
None => return false,
};
let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute);
if role.as_deref() == Some("AXScrollArea") {
let parent_bounds = match crate::tree::read_bounds(parent) {
Some(b) => b,
None => return false,
};
let target_y = bounds.y + bounds.height / 2.0;
let visible_mid = parent_bounds.y + parent_bounds.height / 2.0;
if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height {
let dy = if target_y > visible_mid { -5 } else { 5 };
let cx = parent_bounds.x + parent_bounds.width / 2.0;
let cy = parent_bounds.y + parent_bounds.height / 2.0;
for _ in 0..20 {
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
std::thread::sleep(std::time::Duration::from_millis(16));
}
}
return true;
}
current = crate::tree::copy_element_attr(parent, "AXParent");
}
false
}
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
return false;
}
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
std::thread::sleep(std::time::Duration::from_millis(100));
ax_helpers::try_each_child(
el,
|child| {
let ca = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
},
5,
)
}
fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
string::CFString,
};
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
Some(p) => p,
None => return false,
};
let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
Some(r) => r,
None => return false,
};
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_cf]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
ax_helpers::try_ax_action(el, "AXPerformCustomAction")
}
fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::is_attr_settable(el, "AXSelected") {
return false;
}
if !ax_helpers::set_ax_bool(el, "AXSelected", true) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
if ax_helpers::try_ax_action(el, "AXOpen") {
return Ok(());
}
let ctx = ChainContext {
dynamic_value: None,
};
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
std::thread::sleep(std::time::Duration::from_millis(50));
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
}
pub fn triple_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
let ctx = ChainContext {
dynamic_value: None,
};
for _ in 0..3 {
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
std::thread::sleep(std::time::Duration::from_millis(30));
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3)
}
}
#[cfg(not(target_os = "macos"))]
mod imp {}
#[cfg(target_os = "macos")]
pub(crate) use imp::{
double_click, triple_click, CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN,
FOCUS_CHAIN, RIGHT_CLICK_CHAIN, SCROLL_TO_CHAIN, SET_VALUE_CHAIN,
};

26
crates/macos/src/actions/discovery.rs Normal file
View file

@ -0,0 +1,26 @@
use crate::tree::AXElement;
pub struct ElementCaps {
pub settable_focus: bool,
pub settable_selected: bool,
pub settable_disclosing: bool,
}
#[cfg(target_os = "macos")]
pub fn discover(el: &AXElement) -> ElementCaps {
use crate::actions::ax_helpers;
ElementCaps {
settable_focus: ax_helpers::is_attr_settable(el, "AXFocused"),
settable_selected: ax_helpers::is_attr_settable(el, "AXSelected"),
settable_disclosing: ax_helpers::is_attr_settable(el, "AXDisclosing"),
}
}
#[cfg(not(target_os = "macos"))]
pub fn discover(_el: &AXElement) -> ElementCaps {
ElementCaps {
settable_focus: false,
settable_selected: false,
settable_disclosing: false,
}
}

303
crates/macos/src/actions/dispatch.rs
View file

@ -6,23 +6,12 @@ use agent_desktop_core::{
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::{
ax_helpers,
chain::{execute_chain, ChainContext},
chain_defs, discovery,
};
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorSuccess, kAXFocusedAttribute, kAXPressAction, kAXValueAttribute,
AXUIElementCopyActionNames, AXUIElementPerformAction, AXUIElementSetAttributeValue,
};
use core_foundation::{
array::CFArray,
base::{CFType, TCFType},
boolean::CFBoolean,
string::CFString,
};
pub fn try_ax_action(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
pub fn click_via_bounds(
el: &AXElement,
@ -37,10 +26,10 @@ mod imp {
.with_suggestion("AX action failed and CGEvent fallback unavailable")
})?;
if bounds.width <= 0.0 || bounds.height <= 0.0 {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
"Element has zero-size bounds",
));
return Err(
AdapterError::new(ErrorCode::ActionFailed, "Element has zero-size bounds")
.with_suggestion("Element may be hidden or off-screen. Try 'scroll-to' first."),
);
}
let center = Point {
x: bounds.x + bounds.width / 2.0,
@ -71,21 +60,31 @@ mod imp {
pub fn perform_action(el: &AXElement, action: &Action) -> Result<ActionResult, AdapterError> {
let label = action_label(action);
tracing::debug!("action: perform {label}");
match action {
Action::Click => {
crate::actions::activate::smart_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?;
}
Action::DoubleClick => {
crate::actions::activate::smart_double_activate(el)?;
let caps = discovery::discover(el);
chain_defs::double_click(el, &caps)?;
}
Action::RightClick => {
crate::actions::activate::smart_right_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::RIGHT_CLICK_CHAIN, &ctx)?;
}
Action::Toggle => {
let role = element_role(el);
let role = ax_helpers::element_role(el);
if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
@ -98,40 +97,31 @@ mod imp {
"Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.",
));
}
crate::actions::activate::smart_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?;
}
Action::SetValue(val) => {
ax_set_value(el, val)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: Some(val),
};
execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?;
}
Action::SetFocus => {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("SetFocus failed (err={err})"),
));
}
execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?;
}
Action::TypeText(text) => {
let cf_attr = CFString::new(kAXFocusedAttribute);
unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
crate::input::keyboard::synthesize_text(text)?;
execute_type(el, text)?;
}
Action::PressKey(combo) => {
@ -139,57 +129,19 @@ mod imp {
}
Action::Expand => {
if !try_ax_action(el, "AXExpand") {
if crate::actions::activate::is_attr_settable(el, "AXDisclosing") {
let cf_attr = CFString::new("AXDisclosing");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXDisclosing set to true failed (err={err})"),
));
}
} else {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"AXExpand failed and AXDisclosing not settable",
)
.with_suggestion("Try 'click' to open it instead."));
}
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::EXPAND_CHAIN, &ctx)?;
}
Action::Collapse => {
if !try_ax_action(el, "AXCollapse") {
if crate::actions::activate::is_attr_settable(el, "AXDisclosing") {
let cf_attr = CFString::new("AXDisclosing");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::false_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXDisclosing set to false failed (err={err})"),
));
}
} else {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"AXCollapse failed and AXDisclosing not settable",
)
.with_suggestion("Try 'click' to close it instead."));
}
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::COLLAPSE_CHAIN, &ctx)?;
}
Action::Select(value) => {
@ -209,24 +161,24 @@ mod imp {
}
Action::TripleClick => {
crate::actions::activate::smart_triple_activate(el)?;
let caps = discovery::discover(el);
chain_defs::triple_click(el, &caps)?;
}
Action::ScrollTo => {
let ax_action = CFString::new("AXScrollToVisible");
let err =
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXScrollToVisible failed (err={err})"),
)
.with_suggestion("Element may not be inside a scrollable area"));
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?;
}
Action::Clear => {
ax_set_value(el, "")?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: Some(""),
};
execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?;
}
Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => {
@ -236,7 +188,8 @@ mod imp {
"{} requires adapter-level handling, not element action",
label
),
));
)
.with_suggestion("Use the top-level command (e.g. 'hover', 'drag', 'key-down') instead of targeting an element."));
}
_ => {
@ -244,62 +197,91 @@ mod imp {
}
}
Ok(ActionResult::new(label))
let mut result = ActionResult::new(label);
if let Some(state) = read_post_state(el, action) {
result = result.with_state(state);
}
Ok(result)
}
fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> {
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
}
ax_helpers::ax_focus(el);
std::thread::sleep(std::time::Duration::from_millis(50));
let has_non_ascii = !text.is_ascii();
if has_non_ascii {
type_via_clipboard_paste(el, text)
} else {
crate::input::keyboard::synthesize_text(text)
}
}
fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> {
let saved = crate::input::clipboard::get().ok();
crate::input::clipboard::set(text)?;
std::thread::sleep(std::time::Duration::from_millis(50));
crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo {
key: "v".into(),
modifiers: vec![agent_desktop_core::action::Modifier::Cmd],
})?;
std::thread::sleep(std::time::Duration::from_millis(100));
if let Some(prev) = saved {
let _ = crate::input::clipboard::set(&prev);
}
Ok(())
}
fn read_post_state(
el: &AXElement,
action: &Action,
) -> Option<agent_desktop_core::action::ElementState> {
let delay_ms = match action {
Action::Click
| Action::Toggle
| Action::Check
| Action::Uncheck
| Action::TypeText(_) => 50,
Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0,
_ => return None,
};
if delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
}
let value = crate::tree::copy_value_typed(el);
let role = ax_helpers::element_role(el).unwrap_or_default();
let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false);
let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true);
let mut states = Vec::new();
if focused {
states.push("focused".into());
}
if !enabled {
states.push("disabled".into());
}
Some(agent_desktop_core::action::ElementState {
role,
states,
value,
})
}
pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> {
let action = CFString::new(kAXPressAction);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err != kAXErrorSuccess {
if !ax_helpers::ax_press(el) {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("{context}: AXPress failed (err={err})"),
));
format!("{context}: AXPress failed"),
)
.with_suggestion("Element may not be pressable. Try 'click' instead."));
}
Ok(())
}
pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> {
let cf_attr = CFString::new(kAXValueAttribute);
let cf_val = CFString::new(val);
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("SetValue failed (err={err})"),
));
}
Ok(())
}
pub fn element_role(el: &AXElement) -> Option<String> {
use accessibility_sys::kAXRoleAttribute;
crate::tree::copy_string_attr(el, kAXRoleAttribute)
.map(|r| crate::tree::roles::ax_role_to_str(&r).to_string())
}
pub fn has_ax_action(el: &AXElement, action_name: &str) -> bool {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return false;
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let target = CFString::new(action_name);
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
if name == target {
return true;
}
}
}
false
}
fn check_uncheck(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> {
let role = element_role(el);
let role = ax_helpers::element_role(el);
if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
@ -315,8 +297,11 @@ mod imp {
if is_checked == want_checked {
return Ok(());
}
crate::actions::activate::smart_activate(el)?;
Ok(())
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)
}
}
@ -333,9 +318,7 @@ mod imp {
pub use imp::perform_action;
#[cfg(target_os = "macos")]
pub(crate) use imp::{
ax_press_or_fail, ax_set_value, click_via_bounds, element_role, has_ax_action,
};
pub(crate) use imp::{ax_press_or_fail, click_via_bounds};
fn action_label(action: &Action) -> String {
match action {

15
crates/macos/src/actions/extras.rs
View file

@ -6,12 +6,13 @@ use crate::tree::AXElement;
#[cfg(target_os = "macos")]
pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> {
use crate::actions::dispatch::{ax_press_or_fail, ax_set_value, element_role};
use crate::actions::ax_helpers;
use crate::actions::dispatch::ax_press_or_fail;
let role = element_role(el);
let role = ax_helpers::element_role(el);
match role.as_deref() {
Some("combobox") => {
ax_set_value(el, value)?;
ax_helpers::ax_set_value(el, value)?;
}
Some("popupbutton") | Some("menubutton") => {
ax_press_or_fail(el, "select (open popup)")?;
@ -37,7 +38,7 @@ pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterErr
}
}
_ => {
if ax_set_value(el, value).is_err() {
if ax_helpers::ax_set_value(el, value).is_err() {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
format!(
@ -155,7 +156,7 @@ pub(crate) fn ax_scroll(
Direction::Right => "AXScrollRightByPage",
Direction::Left => "AXScrollLeftByPage",
};
if crate::actions::dispatch::has_ax_action(target, page_action) {
if crate::actions::ax_helpers::has_ax_action(target, page_action) {
let ax = CFString::new(page_action);
for _ in 0..amount {
unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) };
@ -240,7 +241,7 @@ fn try_scroll_bar_value_shift(
use agent_desktop_core::action::Direction;
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
if !crate::actions::activate::is_attr_settable(bar, "AXValue") {
if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") {
return false;
}
let current = read_scroll_bar_value(bar).unwrap_or(0.0);
@ -347,7 +348,7 @@ fn try_select_row_in_direction(
if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) {
continue;
}
if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") {
if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") {
continue;
}
let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default();

5
crates/macos/src/actions/mod.rs
View file

@ -1,4 +1,7 @@
pub mod activate;
pub mod ax_helpers;
pub mod chain;
pub mod chain_defs;
pub mod discovery;
pub mod dispatch;
pub mod extras;

97
crates/macos/src/input/clipboard.rs
View file

@ -3,35 +3,92 @@ use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use std::process::Command;
use core_foundation::base::TCFType;
use std::ffi::c_void;
type Id = *mut c_void;
type Class = *mut c_void;
type Sel = *mut c_void;
extern "C" {
fn objc_getClass(name: *const core::ffi::c_char) -> Class;
fn sel_registerName(name: *const core::ffi::c_char) -> Sel;
fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id;
static NSPasteboardTypeString: Id;
}
fn pasteboard() -> Result<Id, AdapterError> {
unsafe {
let cls = objc_getClass(c"NSPasteboard".as_ptr());
if cls.is_null() {
return Err(AdapterError::internal("NSPasteboard class not found"));
}
let sel = sel_registerName(c"generalPasteboard".as_ptr());
let send: unsafe extern "C" fn(Class, Sel) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let pb = send(cls, sel);
if pb.is_null() {
return Err(AdapterError::internal("generalPasteboard returned null"));
}
Ok(pb)
}
}
pub fn get() -> Result<String, AdapterError> {
let output = Command::new("pbpaste")
.output()
.map_err(|e| AdapterError::internal(format!("pbpaste failed: {e}")))?;
String::from_utf8(output.stdout)
.map_err(|_| AdapterError::internal("Clipboard contains non-UTF8 data"))
tracing::debug!("clipboard: get");
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"stringForType:".as_ptr());
let send: unsafe extern "C" fn(Id, Sel, Id) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let ns_string = send(pb, sel, NSPasteboardTypeString);
if ns_string.is_null() {
tracing::debug!("clipboard: get -> empty");
return Ok(String::new());
}
let cf_str = core_foundation::string::CFString::wrap_under_get_rule(
ns_string as core_foundation_sys::string::CFStringRef,
);
let result = cf_str.to_string();
tracing::debug!("clipboard: get -> {} chars", result.len());
Ok(result)
}
}
pub fn set(text: &str) -> Result<(), AdapterError> {
use std::io::Write;
let mut child = Command::new("pbcopy")
.stdin(std::process::Stdio::piped())
.spawn()
.map_err(|e| AdapterError::internal(format!("pbcopy failed: {e}")))?;
if let Some(stdin) = child.stdin.as_mut() {
stdin
.write_all(text.as_bytes())
.map_err(|e| AdapterError::internal(format!("Write to pbcopy failed: {e}")))?;
tracing::debug!("clipboard: set {} chars", text.len());
unsafe {
let pb = pasteboard()?;
let clear_sel = sel_registerName(c"clearContents".as_ptr());
let send_void: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send_void(pb, clear_sel);
let cf_text = core_foundation::string::CFString::new(text);
let ns_text = cf_text.as_concrete_TypeRef() as Id;
let set_sel = sel_registerName(c"setString:forType:".as_ptr());
let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool =
std::mem::transmute(objc_msgSend as *const c_void);
let ok = send_two(pb, set_sel, ns_text, NSPasteboardTypeString);
if !ok {
return Err(AdapterError::internal(
"NSPasteboard setString:forType: failed",
));
}
Ok(())
}
child
.wait()
.map_err(|e| AdapterError::internal(format!("pbcopy wait failed: {e}")))?;
Ok(())
}
pub fn clear() -> Result<(), AdapterError> {
set("")
tracing::debug!("clipboard: clear");
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"clearContents".as_ptr());
let send: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send(pb, sel);
Ok(())
}
}
}

21
crates/macos/src/input/keyboard.rs
View file

@ -8,6 +8,23 @@ mod imp {
};
pub fn synthesize_key(combo: &KeyCombo) -> Result<(), AdapterError> {
tracing::debug!(
"keyboard: synthesize_key {}{}",
if combo.modifiers.is_empty() {
String::new()
} else {
format!(
"{}+",
combo
.modifiers
.iter()
.map(|m| format!("{m:?}"))
.collect::<Vec<_>>()
.join("+")
)
},
combo.key
);
let key_code = key_name_to_code(&combo.key)?;
let sys_wide = unsafe { AXUIElementCreateSystemWide() };
@ -54,6 +71,7 @@ mod imp {
}
pub fn synthesize_text(text: &str) -> Result<(), AdapterError> {
tracing::debug!("keyboard: synthesize_text {} chars", text.len());
let sys_wide = unsafe { AXUIElementCreateSystemWide() };
if sys_wide.is_null() {
return Err(AdapterError::internal(
@ -258,7 +276,8 @@ mod imp {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Unknown key: '{other}'"),
))
)
.with_suggestion("Valid keys: a-z, 0-9, return, escape, tab, space, delete, left, right, up, down, f1-f12"))
}
};
Ok(code)

19
crates/macos/src/input/mouse.rs
View file

@ -11,6 +11,13 @@ mod imp {
use core_graphics::geometry::CGPoint;
pub fn synthesize_mouse(event: MouseEvent) -> Result<(), AdapterError> {
tracing::debug!(
"mouse: {:?} {:?} at ({:.0}, {:.0})",
event.kind,
event.button,
event.point.x,
event.point.y
);
let point = CGPoint::new(event.point.x, event.point.y);
let cg_button = to_cg_button(&event.button);
match event.kind {
@ -24,6 +31,14 @@ mod imp {
}
pub fn synthesize_drag(params: DragParams) -> Result<(), AdapterError> {
tracing::debug!(
"mouse: drag ({:.0},{:.0}) -> ({:.0},{:.0}) duration={}ms",
params.from.x,
params.from.y,
params.to.x,
params.to.y,
params.duration_ms.unwrap_or(300)
);
let from = CGPoint::new(params.from.x, params.from.y);
let to = CGPoint::new(params.to.x, params.to.y);
let duration_ms = params.duration_ms.unwrap_or(300);
@ -31,7 +46,7 @@ mod imp {
let step_delay = std::time::Duration::from_millis(duration_ms / steps as u64);
post_event(CGEventType::LeftMouseDown, from, CGMouseButton::Left)?;
std::thread::sleep(std::time::Duration::from_millis(50));
std::thread::sleep(std::time::Duration::from_millis(200));
for i in 1..=steps {
let t = i as f64 / steps as f64;
@ -45,6 +60,7 @@ mod imp {
std::thread::sleep(step_delay);
}
std::thread::sleep(std::time::Duration::from_millis(500));
post_event(CGEventType::LeftMouseUp, to, CGMouseButton::Left)
}
@ -131,6 +147,7 @@ mod imp {
}
pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> {
tracing::debug!("mouse: scroll at ({x:.0},{y:.0}) dy={dy} dx={dx}");
use core_graphics::geometry::CGPoint;
extern "C" {

14
crates/macos/src/system/app_ops.rs
View file

@ -13,6 +13,7 @@ pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> {
#[cfg(target_os = "macos")]
pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> {
tracing::debug!("system: ensure_app_focused pid={pid}");
use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue};
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
@ -36,6 +37,11 @@ pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> {
#[cfg(target_os = "macos")]
pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> {
tracing::debug!(
"system: focus_window app={:?} title={:?}",
win.app,
win.title
);
use accessibility_sys::{
kAXErrorSuccess, AXUIElementCreateApplication, AXUIElementPerformAction,
AXUIElementSetAttributeValue,
@ -85,6 +91,7 @@ pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> {
#[cfg(target_os = "macos")]
pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
tracing::debug!("system: launch app={id:?} timeout={timeout_ms}ms");
use crate::adapter::list_windows_impl;
use std::process::Command;
use std::time::{Duration, Instant};
@ -93,7 +100,8 @@ pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterE
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Invalid app identifier: '{id}'"),
));
)
.with_suggestion("Use an app name like 'Safari' or bundle ID like 'com.apple.Safari'."));
}
let filter = WindowFilter {
@ -148,6 +156,7 @@ pub fn launch_app_impl(_id: &str, _timeout_ms: u64) -> Result<WindowInfo, Adapte
#[cfg(target_os = "macos")]
pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> {
tracing::debug!("system: close app={id:?} force={force}");
use std::process::Command;
if force {
Command::new("pkill")
@ -167,7 +176,8 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Invalid app name '{id}'"),
));
)
.with_suggestion("App name should only contain letters, numbers, spaces, hyphens, dots, or underscores."));
}
let script = format!(
r#"tell application "System Events"

2
crates/macos/src/system/key_dispatch.rs
View file

@ -11,6 +11,7 @@ pub fn press_for_app_impl(app_name: &str, combo: &KeyCombo) -> Result<ActionResu
use accessibility_sys::AXUIElementSetAttributeValue;
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
tracing::debug!("system: press_for_app app={app_name:?} key={:?}", combo.key);
let pid = find_pid_by_name(app_name)?;
let app_el = crate::tree::element_for_pid(pid);
if app_el.0.is_null() {
@ -302,6 +303,7 @@ pub(crate) fn find_pid_by_name(app_name: &str) -> Result<i32, AdapterError> {
agent_desktop_core::error::ErrorCode::AppNotFound,
format!("App '{app_name}' not found"),
)
.with_suggestion("Verify the app is running. Use 'list-apps' to see running applications.")
})
}

2
crates/macos/src/system/screenshot.rs
View file

@ -6,6 +6,7 @@ mod imp {
use std::process::Command;
pub fn capture_app(pid: i32) -> Result<ImageBuffer, AdapterError> {
tracing::debug!("system: screenshot app pid={pid}");
let temp = temp_path();
let cg_id = find_cg_window_id_for_pid(pid);
@ -31,6 +32,7 @@ mod imp {
}
pub fn capture_screen(_idx: usize) -> Result<ImageBuffer, AdapterError> {
tracing::debug!("system: screenshot screen");
let temp = temp_path();
let status = Command::new("screencapture")
.args(["-x", "-t", "png"])

18
crates/macos/src/system/window_ops.rs
View file

@ -17,6 +17,12 @@ mod imp {
}
pub fn execute(win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> {
tracing::debug!(
"system: window_op {:?} app={:?} title={:?}",
op,
win.app,
win.title
);
let win_el = crate::tree::window_element_for(win.pid, &win.title);
match op {
WindowOp::Resize { width, height } => set_size(&win_el, width, height),
@ -43,7 +49,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Resize failed (err={err})"),
));
)
.with_suggestion("Window may not support resizing. Try a different size."));
}
Ok(())
}
@ -66,6 +73,9 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Move failed (err={err})"),
)
.with_suggestion(
"Window may not support repositioning. Verify coordinates are on-screen.",
));
}
Ok(())
@ -86,7 +96,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("{op} failed (err={err})"),
));
)
.with_suggestion("Window may not support this operation. Try 'focus-window' first."));
}
Ok(())
}
@ -101,7 +112,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Zoom button press failed (err={err})"),
));
)
.with_suggestion("Try 'resize-window' with explicit dimensions instead."));
}
Ok(())
}

9
crates/macos/src/tree/element.rs
View file

@ -161,7 +161,7 @@ mod imp {
cf_type.downcast::<CFString>().map(|s| s.to_string())
}
fn copy_value_typed(el: &AXElement) -> Option<String> {
pub fn copy_value_typed(el: &AXElement) -> Option<String> {
let cf_attr = CFString::new(kAXValueAttribute);
let mut val_ref: CFTypeRef = std::ptr::null_mut();
let err = unsafe {
@ -335,6 +335,9 @@ mod imp {
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
None
}
pub fn copy_value_typed(_el: &AXElement) -> Option<String> {
None
}
pub fn fetch_node_attrs(
_el: &AXElement,
) -> (
@ -350,6 +353,6 @@ mod imp {
}
pub use imp::{
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid,
fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed,
element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
};

4
crates/macos/src/tree/mod.rs
View file

@ -6,8 +6,8 @@ pub mod surfaces;
pub use builder::{build_subtree, window_element_for};
pub use element::{
copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, read_bounds,
resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid,
read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
};
pub use resolve::{find_element_recursive, resolve_element_impl};
pub use roles::{ax_role_to_str, is_interactive_role};

34
crates/macos/src/tree/resolve.rs
View file

@ -7,9 +7,41 @@ use super::element::{
#[cfg(target_os = "macos")]
pub fn resolve_element_impl(entry: &RefEntry) -> Result<NativeHandle, AdapterError> {
tracing::debug!(
"resolve: searching pid={} role={} name={:?} bounds_hash={:?}",
entry.pid,
entry.role,
entry.name.as_deref().unwrap_or("(none)"),
entry.bounds_hash
);
let root = element_for_pid(entry.pid);
let mut visited = FxHashSet::default();
find_element_recursive(&root, entry, 0, 20, &mut visited)
if let Ok(handle) = find_element_recursive(&root, entry, 0, 20, &mut visited) {
tracing::debug!("resolve: found exact match");
return Ok(handle);
}
if entry.bounds_hash.is_some() && entry.name.is_some() {
tracing::debug!("resolve: exact match failed, trying relaxed (name-only)");
let relaxed = RefEntry {
bounds_hash: None,
..entry.clone()
};
visited.clear();
if let Ok(handle) = find_element_recursive(&root, &relaxed, 0, 20, &mut visited) {
tracing::debug!("resolve: found via relaxed match (bounds changed)");
return Ok(handle);
}
}
tracing::debug!("resolve: element not found");
Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::StaleRef,
format!(
"Element not found: role={}, name={:?}",
entry.role,
entry.name.as_deref().unwrap_or("(none)")
),
)
.with_suggestion("Run 'snapshot' to refresh, then retry with the updated ref."))
}
#[cfg(target_os = "macos")]

15
src/cli_args.rs
View file

@ -12,6 +12,21 @@ pub enum Surface {
Alert,
}
impl Surface {
pub fn to_core(&self) -> agent_desktop_core::adapter::SnapshotSurface {
use agent_desktop_core::adapter::SnapshotSurface;
match self {
Self::Window => SnapshotSurface::Window,
Self::Focused => SnapshotSurface::Focused,
Self::Menu => SnapshotSurface::Menu,
Self::Menubar => SnapshotSurface::Menubar,
Self::Sheet => SnapshotSurface::Sheet,
Self::Popover => SnapshotSurface::Popover,
Self::Alert => SnapshotSurface::Alert,
}
}
}
#[derive(Parser, Debug)]
pub struct SnapshotArgs {
#[arg(long, help = "Filter to application by name")]

17
src/dispatch.rs
View file

@ -16,6 +16,7 @@ use serde_json::Value;
use crate::cli::Commands;
pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
tracing::debug!("dispatch: {}", cmd.name());
match cmd {
Commands::Snapshot(a) => snapshot::execute(
snapshot::SnapshotArgs {
@ -25,7 +26,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result<Value, A
include_bounds: a.include_bounds,
interactive_only: a.interactive_only,
compact: a.compact,
surface: cli_surface_to_core(&a.surface),
surface: a.surface.to_core(),
},
adapter,
),
@ -389,17 +390,3 @@ fn parse_xy_opt(s: Option<&str>) -> Result<Option<(f64, f64)>, AppError> {
None => Ok(None),
}
}
fn cli_surface_to_core(s: &crate::cli::Surface) -> agent_desktop_core::adapter::SnapshotSurface {
use crate::cli::Surface;
use agent_desktop_core::adapter::SnapshotSurface;
match s {
Surface::Window => SnapshotSurface::Window,
Surface::Focused => SnapshotSurface::Focused,
Surface::Menu => SnapshotSurface::Menu,
Surface::Menubar => SnapshotSurface::Menubar,
Surface::Sheet => SnapshotSurface::Sheet,
Surface::Popover => SnapshotSurface::Popover,
Surface::Alert => SnapshotSurface::Alert,
}
}