Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-08-19 05:25:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: centralize AX chain executor with error suggestions and resilience

Browse source 
- Add ax_helpers.rs: 16 shared AX utility functions, eliminating ~150 LOC duplication
- Add chain.rs: declarative ChainStep executor with 9 step variants and 10s deadline
- Add chain_defs.rs: static chain definitions for click (14-step), right-click, expand, collapse
- Add discovery.rs: one-time ElementCaps query shared across chain steps
- Replace NSPasteboard subprocess calls with direct objc_msgSend FFI (zero new deps)
- Add .with_suggestion() to 16 high-priority error paths across 8 files
- Fix FocusThenAction to use retried variant for kAXErrorCannotComplete resilience
- Add relaxed stale ref matching (name-only fallback when bounds change)
- Delete activate.rs, absorb logic into chain_defs and ax_helpers (-388 net LOC)
This commit is contained in:
Lahfir 2026-02-23 03:32:17 -08:00
parent ada8f4f0ee
commit 4fe91e3a96
16 changed files with 997 additions and 603 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
crates/core/src/commands/focus_window.rs
View file

@ -34,10 +34,13 @@ pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result<V
};
let window = window.ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No matching window found",
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No matching window found",
)
.with_suggestion("Run 'list-windows' to see available windows and their IDs."),
)
})?;
adapter.focus_window(&window)?;

37
crates/core/src/snapshot.rs
View file

@ -45,10 +45,13 @@ pub fn build(
let window = if let Some(wid) = window_id {
windows.into_iter().find(|w| w.id == wid).ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
format!("No window with id {wid}"),
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
format!("No window with id {wid}"),
)
.with_suggestion("Run 'list-windows' to see available window IDs."),
)
})?
} else if let Some(app) = app_name {
windows
@ -64,17 +67,27 @@ pub fn build(
.and_then(|ws| ws.into_iter().next())
})
.ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::AppNotFound,
format!("No window found for app '{app}'"),
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::AppNotFound,
format!("No window found for app '{app}'"),
)
.with_suggestion(
"Verify the app is running. Use 'list-apps' to see running applications.",
),
)
})?
} else {
windows.into_iter().find(|w| w.is_focused).ok_or_else(|| {
AppError::Adapter(crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No focused window found. Use --app to specify an application.",
))
AppError::Adapter(
crate::error::AdapterError::new(
crate::error::ErrorCode::WindowNotFound,
"No focused window found",
)
.with_suggestion(
"Use --app to specify an application, or click a window to focus it.",
),
)
})?
};

399
crates/macos/src/actions/activate.rs
View file

@ -1,399 +0,0 @@
use agent_desktop_core::{action::MouseButton, error::AdapterError};
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorSuccess, kAXFocusedAttribute, kAXRoleAttribute, AXUIElementCopyActionNames,
AXUIElementIsAttributeSettable, AXUIElementPerformAction, AXUIElementSetAttributeValue,
};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
boolean::CFBoolean,
string::CFString,
};
use std::os::raw::c_uchar;
pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> {
let scroll_action = CFString::new("AXScrollToVisible");
unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) };
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXPress"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXConfirm"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXOpen"]) {
return Ok(());
}
if try_action_from_list(el, &actions, &["AXPick"]) {
return Ok(());
}
if try_show_alternate_ui(el) {
return Ok(());
}
if try_child_activation(el) {
return Ok(());
}
if try_set_selected(el) {
return Ok(());
}
if try_select_via_parent(el) {
return Ok(());
}
if try_custom_actions(el) {
return Ok(());
}
if try_focus_then_activate(el) {
return Ok(());
}
if try_keyboard_activate(el) {
return Ok(());
}
if try_parent_activation(el) {
return Ok(());
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 1)
}
/// AXOpen first, then two smart_activate calls with a gap, then CGEvent double-click.
pub fn smart_double_activate(el: &AXElement) -> Result<(), AdapterError> {
let actions = list_ax_actions(el);
if try_action_from_list(el, &actions, &["AXOpen"]) {
return Ok(());
}
let _ = smart_activate(el);
std::thread::sleep(std::time::Duration::from_millis(50));
let _ = smart_activate(el);
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
}
pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> {
if ax_show_menu(el) {
return Ok(());
}
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
if ax_show_menu(el) {
return Ok(());
}
}
if try_select_then_show_menu(el) {
return Ok(());
}
if try_focus_then_show_menu(el) {
return Ok(());
}
if try_parent_show_menu(el) {
return Ok(());
}
if try_child_show_menu(el) {
return Ok(());
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Right, 1)
}
fn ax_show_menu(el: &AXElement) -> bool {
let show = CFString::new("AXShowMenu");
let err = unsafe { AXUIElementPerformAction(el.0, show.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_select_then_show_menu(el: &AXElement) -> bool {
if !is_attr_settable(el, "AXSelected") {
return false;
}
let cf_attr = CFString::new("AXSelected");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_show_menu(el)
}
fn try_focus_then_show_menu(el: &AXElement) -> bool {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_show_menu(el)
}
fn try_parent_show_menu(el: &AXElement) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..3 {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
if ax_show_menu(ancestor) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
fn try_child_show_menu(el: &AXElement) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(5) {
if ax_show_menu(child) {
return true;
}
}
false
}
/// Three smart_activate calls with gaps, then CGEvent triple-click.
pub fn smart_triple_activate(el: &AXElement) -> Result<(), AdapterError> {
for _ in 0..3 {
let _ = smart_activate(el);
std::thread::sleep(std::time::Duration::from_millis(30));
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3)
}
fn list_ax_actions(el: &AXElement) -> Vec<String> {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return Vec::new();
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let mut result = Vec::new();
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
result.push(name.to_string());
}
}
result
}
pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool {
let cf_attr = CFString::new(attr);
let mut settable: c_uchar = 0;
let err = unsafe {
AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable)
};
err == kAXErrorSuccess && settable != 0
}
fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool {
for target in targets {
if actions.iter().any(|a| a == target) {
let action = CFString::new(target);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err == kAXErrorSuccess {
return true;
}
}
}
false
}
fn try_set_selected(el: &AXElement) -> bool {
if !is_attr_settable(el, "AXSelected") {
return false;
}
let cf_attr = CFString::new("AXSelected");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_select_via_parent(el: &AXElement) -> bool {
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
Some(p) => p,
None => return false,
};
let parent_role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
Some(r) => r,
None => return false,
};
if !matches!(parent_role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_as_cftype = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_as_cftype]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_focus_then_activate(el: &AXElement) -> bool {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let confirm = CFString::new("AXConfirm");
if unsafe { AXUIElementPerformAction(el.0, confirm.as_concrete_TypeRef()) }
== kAXErrorSuccess
{
return true;
}
let press = CFString::new("AXPress");
let err = unsafe { AXUIElementPerformAction(el.0, press.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_child_activation(el: &AXElement) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(3) {
let child_actions = list_ax_actions(child);
if try_action_from_list(child, &child_actions, &["AXPress", "AXConfirm", "AXOpen"]) {
return true;
}
}
false
}
fn try_parent_activation(el: &AXElement) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..2 {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
let actions = list_ax_actions(ancestor);
if try_action_from_list(ancestor, &actions, &["AXPress", "AXConfirm"]) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
fn try_show_alternate_ui(el: &AXElement) -> bool {
let actions = list_ax_actions(el);
if !actions.iter().any(|a| a == "AXShowAlternateUI") {
return false;
}
let action = CFString::new("AXShowAlternateUI");
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
std::thread::sleep(std::time::Duration::from_millis(100));
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(5) {
let child_actions = list_ax_actions(child);
if try_action_from_list(child, &child_actions, &["AXPress"]) {
return true;
}
}
false
}
fn try_custom_actions(el: &AXElement) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
let action = CFString::new("AXPerformCustomAction");
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
fn try_keyboard_activate(el: &AXElement) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
let cf_focused = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_focused.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
use crate::tree::AXElement;
pub fn smart_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_activate"))
}
pub fn smart_double_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_double_activate"))
}
pub fn smart_right_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_right_activate"))
}
pub fn smart_triple_activate(_el: &AXElement) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("smart_triple_activate"))
}
pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool {
false
}
}
pub(crate) use imp::{
is_attr_settable, smart_activate, smart_double_activate, smart_right_activate,
smart_triple_activate,
};

240
crates/macos/src/actions/ax_helpers.rs Normal file
View file

@ -0,0 +1,240 @@
use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute,
AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, AXUIElementPerformAction,
AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout,
};
use core_foundation::{
array::CFArray,
base::{CFType, TCFType},
boolean::CFBoolean,
string::CFString,
};
use std::os::raw::c_uchar;
pub fn try_ax_action(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
pub fn try_ax_action_retried(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err == kAXErrorSuccess {
return true;
}
if err == kAXErrorCannotComplete {
std::thread::sleep(std::time::Duration::from_millis(100));
let retry = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
return retry == kAXErrorSuccess;
}
false
}
pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool {
let cf_attr = CFString::new(attr);
let cf_val = if value {
CFBoolean::true_value()
} else {
CFBoolean::false_value()
};
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
err == kAXErrorSuccess
}
#[allow(dead_code)]
pub fn set_ax_string(el: &AXElement, attr: &str, value: &str) -> bool {
let cf_attr = CFString::new(attr);
let cf_val = CFString::new(value);
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
err == kAXErrorSuccess
}
pub fn set_ax_string_or_err(
el: &AXElement,
attr: &str,
value: &str,
) -> Result<(), AdapterError> {
let cf_attr = CFString::new(attr);
let cf_val = CFString::new(value);
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::ActionFailed,
format!("AXSetAttributeValue({attr}) failed (err={err})"),
)
.with_suggestion("Attribute may be read-only. Try 'click' or 'type' instead."));
}
Ok(())
}
pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool {
let cf_attr = CFString::new(attr);
let mut settable: c_uchar = 0;
let err = unsafe {
AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable)
};
err == kAXErrorSuccess && settable != 0
}
pub fn list_ax_actions(el: &AXElement) -> Vec<String> {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return Vec::new();
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let mut result = Vec::with_capacity(actions.len() as usize);
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
result.push(name.to_string());
}
}
result
}
pub fn has_ax_action(el: &AXElement, target: &str) -> bool {
list_ax_actions(el).iter().any(|a| a == target)
}
pub fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool {
for target in targets {
if actions.iter().any(|a| a == target) && try_ax_action(el, target) {
return true;
}
}
false
}
pub fn try_each_child(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool {
let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default();
for child in children.iter().take(limit) {
if f(child) {
return true;
}
}
false
}
pub fn try_each_ancestor(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool {
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..limit {
let ancestor = match &current {
Some(a) => a,
None => return false,
};
if f(ancestor) {
return true;
}
current = crate::tree::copy_element_attr(ancestor, "AXParent");
}
false
}
pub fn ensure_visible(el: &AXElement) {
let action = CFString::new("AXScrollToVisible");
unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
}
pub fn set_messaging_timeout(el: &AXElement, seconds: f32) {
unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) };
}
pub fn ax_focus(el: &AXElement) -> bool {
set_ax_bool(el, kAXFocusedAttribute, true)
}
pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> {
set_ax_string_or_err(el, kAXValueAttribute, val)
}
pub fn ax_press(el: &AXElement) -> bool {
try_ax_action(el, "AXPress")
}
pub fn element_role(el: &AXElement) -> Option<String> {
use accessibility_sys::kAXRoleAttribute;
crate::tree::copy_string_attr(el, kAXRoleAttribute)
.map(|r| crate::tree::roles::ax_role_to_str(&r).to_string())
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
use crate::tree::AXElement;
pub fn try_ax_action(_el: &AXElement, _name: &str) -> bool {
false
}
pub fn try_ax_action_retried(_el: &AXElement, _name: &str) -> bool {
false
}
pub fn set_ax_bool(_el: &AXElement, _attr: &str, _value: bool) -> bool {
false
}
pub fn set_ax_string(_el: &AXElement, _attr: &str, _value: &str) -> bool {
false
}
pub fn set_ax_string_or_err(
_el: &AXElement,
_attr: &str,
_value: &str,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("set_ax_string_or_err"))
}
pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool {
false
}
pub fn list_ax_actions(_el: &AXElement) -> Vec<String> {
Vec::new()
}
pub fn has_ax_action(_el: &AXElement, _target: &str) -> bool {
false
}
pub fn try_action_from_list(_el: &AXElement, _actions: &[String], _targets: &[&str]) -> bool {
false
}
pub fn try_each_child(_el: &AXElement, _f: impl Fn(&AXElement) -> bool, _limit: usize) -> bool {
false
}
pub fn try_each_ancestor(
_el: &AXElement,
_f: impl Fn(&AXElement) -> bool,
_limit: usize,
) -> bool {
false
}
pub fn ensure_visible(_el: &AXElement) {}
pub fn set_messaging_timeout(_el: &AXElement, _seconds: f32) {}
pub fn ax_focus(_el: &AXElement) -> bool {
false
}
pub fn ax_set_value(_el: &AXElement, _val: &str) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("ax_set_value"))
}
pub fn ax_press(_el: &AXElement) -> bool {
false
}
pub fn element_role(_el: &AXElement) -> Option<String> {
None
}
}
pub(crate) use imp::{
ax_focus, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action,
is_attr_settable, list_ax_actions, set_ax_bool, set_ax_string_or_err, set_messaging_timeout,
try_action_from_list, try_ax_action, try_ax_action_retried, try_each_ancestor, try_each_child,
};

214
crates/macos/src/actions/chain.rs Normal file
View file

@ -0,0 +1,214 @@
use agent_desktop_core::error::{AdapterError, ErrorCode};
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::{ax_helpers, discovery::ElementCaps};
use crate::tree::AXElement;
use agent_desktop_core::action::MouseButton;
use std::time::{Duration, Instant};
#[allow(dead_code)]
pub enum ChainStep {
Action(&'static str),
SetBool {
attr: &'static str,
value: bool,
},
SetDynamic {
attr: &'static str,
},
FocusThenAction(&'static str),
FocusThenConfirmOrPress,
ChildActions {
actions: &'static [&'static str],
limit: usize,
},
AncestorActions {
actions: &'static [&'static str],
limit: usize,
},
Custom {
label: &'static str,
func: fn(&AXElement, &ElementCaps) -> bool,
},
CGClick {
button: MouseButton,
count: u32,
},
}
pub struct ChainDef {
pub pre_scroll: bool,
pub steps: &'static [ChainStep],
pub suggestion: &'static str,
}
pub struct ChainContext<'a> {
pub dynamic_value: Option<&'a str>,
}
const CHAIN_TIMEOUT: Duration = Duration::from_secs(10);
pub fn execute_chain(
el: &AXElement,
caps: &ElementCaps,
def: &ChainDef,
ctx: &ChainContext,
) -> Result<(), AdapterError> {
let deadline = Instant::now() + CHAIN_TIMEOUT;
ax_helpers::set_messaging_timeout(el, 3.0);
if def.pre_scroll {
ax_helpers::ensure_visible(el);
}
for step in def.steps {
if Instant::now() > deadline {
return Err(AdapterError::timeout("Chain execution exceeded 10s"));
}
if execute_step(el, caps, step, ctx) {
return Ok(());
}
}
Err(
AdapterError::new(ErrorCode::ActionFailed, "All chain steps exhausted")
.with_suggestion(def.suggestion),
)
}
fn execute_step(
el: &AXElement,
caps: &ElementCaps,
step: &ChainStep,
ctx: &ChainContext,
) -> bool {
match step {
ChainStep::Action(name) => ax_helpers::try_ax_action_retried(el, name),
ChainStep::SetBool { attr, value } => {
let settable = match *attr {
"AXSelected" => caps.settable_selected,
"AXDisclosing" => caps.settable_disclosing,
"AXFocused" => caps.settable_focus,
_ => ax_helpers::is_attr_settable(el, attr),
};
settable && ax_helpers::set_ax_bool(el, attr, *value)
}
ChainStep::SetDynamic { attr } => {
let value = match ctx.dynamic_value {
Some(v) => v,
None => return false,
};
ax_helpers::set_ax_string_or_err(el, attr, value).is_ok()
}
ChainStep::FocusThenAction(name) => {
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::try_ax_action_retried(el, name)
}
ChainStep::FocusThenConfirmOrPress => {
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::try_ax_action_retried(el, "AXConfirm")
|| ax_helpers::try_ax_action_retried(el, "AXPress")
}
ChainStep::ChildActions { actions, limit } => ax_helpers::try_each_child(
el,
|child| {
let child_actions = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &child_actions, actions)
},
*limit,
),
ChainStep::AncestorActions { actions, limit } => ax_helpers::try_each_ancestor(
el,
|ancestor| {
let al = ax_helpers::list_ax_actions(ancestor);
ax_helpers::try_action_from_list(ancestor, &al, actions)
},
*limit,
),
ChainStep::Custom { label: _, func } => func(el, caps),
ChainStep::CGClick { button, count } => {
crate::actions::dispatch::click_via_bounds(el, button.clone(), *count).is_ok()
}
}
}
}
#[cfg(not(target_os = "macos"))]
mod imp {
use super::*;
use crate::actions::discovery::ElementCaps;
use crate::tree::AXElement;
use agent_desktop_core::action::MouseButton;
#[allow(dead_code)]
pub enum ChainStep {
Action(&'static str),
SetBool {
attr: &'static str,
value: bool,
},
SetDynamic {
attr: &'static str,
},
FocusThenAction(&'static str),
FocusThenConfirmOrPress,
ChildActions {
actions: &'static [&'static str],
limit: usize,
},
AncestorActions {
actions: &'static [&'static str],
limit: usize,
},
Custom {
label: &'static str,
func: fn(&AXElement, &ElementCaps) -> bool,
},
CGClick {
button: MouseButton,
count: u32,
},
}
pub struct ChainDef {
pub pre_scroll: bool,
pub steps: &'static [ChainStep],
pub suggestion: &'static str,
}
pub struct ChainContext<'a> {
pub dynamic_value: Option<&'a str>,
}
pub fn execute_chain(
_el: &AXElement,
_caps: &ElementCaps,
def: &ChainDef,
_ctx: &ChainContext,
) -> Result<(), AdapterError> {
Err(AdapterError::new(
ErrorCode::ActionFailed,
"Chain execution not supported on this platform",
)
.with_suggestion(def.suggestion))
}
}
#[allow(unused_imports)]
pub(crate) use imp::{execute_chain, ChainContext, ChainDef, ChainStep};

242
crates/macos/src/actions/chain_defs.rs Normal file
View file

@ -0,0 +1,242 @@
use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::{
ax_helpers,
chain::{execute_chain, ChainContext, ChainDef, ChainStep},
discovery::ElementCaps,
};
use crate::tree::AXElement;
use agent_desktop_core::action::MouseButton;
pub static CLICK_CHAIN: ChainDef = ChainDef {
pre_scroll: true,
steps: &[
ChainStep::Action("AXPress"),
ChainStep::Action("AXConfirm"),
ChainStep::Action("AXOpen"),
ChainStep::Action("AXPick"),
ChainStep::Custom {
label: "show_alternate_ui",
func: try_show_alternate_ui,
},
ChainStep::ChildActions {
actions: &["AXPress", "AXConfirm", "AXOpen"],
limit: 3,
},
ChainStep::SetBool {
attr: "AXSelected",
value: true,
},
ChainStep::Custom {
label: "select_via_parent",
func: try_select_via_parent,
},
ChainStep::Custom {
label: "custom_actions",
func: try_custom_actions,
},
ChainStep::FocusThenConfirmOrPress,
ChainStep::Custom {
label: "keyboard_activate",
func: try_keyboard_activate,
},
ChainStep::AncestorActions {
actions: &["AXPress", "AXConfirm"],
limit: 2,
},
ChainStep::CGClick {
button: MouseButton::Left,
count: 1,
},
],
suggestion: "Element may not be interactable. Try 'mouse-click --xy X,Y'.",
};
pub static RIGHT_CLICK_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXShowMenu"),
ChainStep::Custom {
label: "focus_app_show_menu",
func: focus_app_then_show_menu,
},
ChainStep::Custom {
label: "select_then_show_menu",
func: select_then_show_menu,
},
ChainStep::FocusThenAction("AXShowMenu"),
ChainStep::AncestorActions {
actions: &["AXShowMenu"],
limit: 3,
},
ChainStep::ChildActions {
actions: &["AXShowMenu"],
limit: 5,
},
ChainStep::CGClick {
button: MouseButton::Right,
count: 1,
},
],
suggestion: "Try 'mouse-click --button right --xy X,Y'.",
};
pub static EXPAND_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXExpand"),
ChainStep::SetBool {
attr: "AXDisclosing",
value: true,
},
],
suggestion: "Try 'click' to open it instead.",
};
pub static COLLAPSE_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXCollapse"),
ChainStep::SetBool {
attr: "AXDisclosing",
value: false,
},
],
suggestion: "Try 'click' to close it instead.",
};
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
return false;
}
ax_helpers::try_ax_action(el, "AXShowAlternateUI");
std::thread::sleep(std::time::Duration::from_millis(100));
ax_helpers::try_each_child(
el,
|child| {
let ca = ax_helpers::list_ax_actions(child);
ax_helpers::try_action_from_list(child, &ca, &["AXPress"])
},
5,
)
}
fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue};
use core_foundation::{
array::CFArray,
base::{CFRetain, CFType, CFTypeRef, TCFType},
string::CFString,
};
let parent = match crate::tree::copy_element_attr(el, "AXParent") {
Some(p) => p,
None => return false,
};
let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) {
Some(r) => r,
None => return false,
};
if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") {
return false;
}
if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") {
return false;
}
unsafe { CFRetain(el.0 as CFTypeRef) };
let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) };
let arr = CFArray::from_CFTypes(&[el_cf]);
let cf_attr = CFString::new("AXSelectedRows");
let err = unsafe {
AXUIElementSetAttributeValue(
parent.0,
cf_attr.as_concrete_TypeRef(),
arr.as_CFTypeRef(),
)
};
err == kAXErrorSuccess
}
fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool {
let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default();
if custom.is_empty() {
return false;
}
ax_helpers::try_ax_action(el, "AXPerformCustomAction")
}
fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 49, true);
AXUIElementPostKeyboardEvent(app.0, 0, 49, false);
};
true
}
fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let _ = crate::system::app_ops::ensure_app_focused(pid);
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::is_attr_settable(el, "AXSelected") {
return false;
}
if !ax_helpers::set_ax_bool(el, "AXSelected", true) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
ax_helpers::try_ax_action(el, "AXShowMenu")
}
pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
if ax_helpers::try_ax_action(el, "AXOpen") {
return Ok(());
}
let ctx = ChainContext {
dynamic_value: None,
};
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
std::thread::sleep(std::time::Duration::from_millis(50));
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2)
}
pub fn triple_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> {
let ctx = ChainContext {
dynamic_value: None,
};
for _ in 0..3 {
let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx);
std::thread::sleep(std::time::Duration::from_millis(30));
}
crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3)
}
}
#[cfg(not(target_os = "macos"))]
mod imp {}
#[cfg(target_os = "macos")]
pub(crate) use imp::{
double_click, triple_click, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, RIGHT_CLICK_CHAIN,
};

86
crates/macos/src/actions/discovery.rs Normal file
View file

@ -0,0 +1,86 @@
#[cfg(target_os = "macos")]
#[allow(dead_code)]
mod imp {
use crate::actions::ax_helpers;
use crate::tree::AXElement;
pub struct ElementCaps {
pub actions: Vec<String>,
pub settable_value: bool,
pub settable_focus: bool,
pub settable_selected: bool,
pub settable_disclosing: bool,
pub role: Option<String>,
pub has_children: bool,
pub pid: Option<i32>,
}
pub fn discover(el: &AXElement) -> ElementCaps {
let actions = ax_helpers::list_ax_actions(el);
let settable_value = ax_helpers::is_attr_settable(el, "AXValue");
let settable_focus = ax_helpers::is_attr_settable(el, "AXFocused");
let settable_selected = ax_helpers::is_attr_settable(el, "AXSelected");
let settable_disclosing = ax_helpers::is_attr_settable(el, "AXDisclosing");
let role = ax_helpers::element_role(el);
let has_children = crate::tree::copy_ax_array(el, "AXChildren")
.map(|c| !c.is_empty())
.unwrap_or(false);
let pid = crate::system::app_ops::pid_from_element(el);
ElementCaps {
actions,
settable_value,
settable_focus,
settable_selected,
settable_disclosing,
role,
has_children,
pid,
}
}
impl ElementCaps {
pub fn has_action(&self, name: &str) -> bool {
self.actions.iter().any(|a| a == name)
}
}
}
#[cfg(not(target_os = "macos"))]
#[allow(dead_code)]
mod imp {
use crate::tree::AXElement;
pub struct ElementCaps {
pub actions: Vec<String>,
pub settable_value: bool,
pub settable_focus: bool,
pub settable_selected: bool,
pub settable_disclosing: bool,
pub role: Option<String>,
pub has_children: bool,
pub pid: Option<i32>,
}
pub fn discover(_el: &AXElement) -> ElementCaps {
ElementCaps {
actions: Vec::new(),
settable_value: false,
settable_focus: false,
settable_selected: false,
settable_disclosing: false,
role: None,
has_children: false,
pid: None,
}
}
impl ElementCaps {
pub fn has_action(&self, _name: &str) -> bool {
false
}
}
}
#[allow(unused_imports)]
pub(crate) use imp::{discover, ElementCaps};

215
crates/macos/src/actions/dispatch.rs
View file

@ -6,23 +6,12 @@ use agent_desktop_core::{
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use crate::actions::{
ax_helpers,
chain::{execute_chain, ChainContext},
chain_defs, discovery,
};
use crate::tree::AXElement;
use accessibility_sys::{
kAXErrorSuccess, kAXFocusedAttribute, kAXPressAction, kAXValueAttribute,
AXUIElementCopyActionNames, AXUIElementPerformAction, AXUIElementSetAttributeValue,
};
use core_foundation::{
array::CFArray,
base::{CFType, TCFType},
boolean::CFBoolean,
string::CFString,
};
pub fn try_ax_action(el: &AXElement, name: &str) -> bool {
let action = CFString::new(name);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
err == kAXErrorSuccess
}
pub fn click_via_bounds(
el: &AXElement,
@ -37,10 +26,10 @@ mod imp {
.with_suggestion("AX action failed and CGEvent fallback unavailable")
})?;
if bounds.width <= 0.0 || bounds.height <= 0.0 {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
"Element has zero-size bounds",
));
return Err(
AdapterError::new(ErrorCode::ActionFailed, "Element has zero-size bounds")
.with_suggestion("Element may be hidden or off-screen. Try 'scroll-to' first."),
);
}
let center = Point {
x: bounds.x + bounds.width / 2.0,
@ -73,19 +62,28 @@ mod imp {
let label = action_label(action);
match action {
Action::Click => {
crate::actions::activate::smart_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?;
}
Action::DoubleClick => {
crate::actions::activate::smart_double_activate(el)?;
let caps = discovery::discover(el);
chain_defs::double_click(el, &caps)?;
}
Action::RightClick => {
crate::actions::activate::smart_right_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::RIGHT_CLICK_CHAIN, &ctx)?;
}
Action::Toggle => {
let role = element_role(el);
let role = ax_helpers::element_role(el);
if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
@ -98,39 +96,28 @@ mod imp {
"Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.",
));
}
crate::actions::activate::smart_activate(el)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?;
}
Action::SetValue(val) => {
ax_set_value(el, val)?;
ax_helpers::ax_set_value(el, val)?;
}
Action::SetFocus => {
let cf_attr = CFString::new(kAXFocusedAttribute);
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("SetFocus failed (err={err})"),
));
if !ax_helpers::ax_focus(el) {
return Err(
AdapterError::new(ErrorCode::ActionFailed, "SetFocus failed")
.with_suggestion("Element may not support focus. Try 'click' instead."),
);
}
}
Action::TypeText(text) => {
let cf_attr = CFString::new(kAXFocusedAttribute);
unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
ax_helpers::ax_focus(el);
crate::input::keyboard::synthesize_text(text)?;
}
@ -139,57 +126,19 @@ mod imp {
}
Action::Expand => {
if !try_ax_action(el, "AXExpand") {
if crate::actions::activate::is_attr_settable(el, "AXDisclosing") {
let cf_attr = CFString::new("AXDisclosing");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXDisclosing set to true failed (err={err})"),
));
}
} else {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"AXExpand failed and AXDisclosing not settable",
)
.with_suggestion("Try 'click' to open it instead."));
}
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::EXPAND_CHAIN, &ctx)?;
}
Action::Collapse => {
if !try_ax_action(el, "AXCollapse") {
if crate::actions::activate::is_attr_settable(el, "AXDisclosing") {
let cf_attr = CFString::new("AXDisclosing");
let err = unsafe {
AXUIElementSetAttributeValue(
el.0,
cf_attr.as_concrete_TypeRef(),
CFBoolean::false_value().as_CFTypeRef(),
)
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXDisclosing set to false failed (err={err})"),
));
}
} else {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"AXCollapse failed and AXDisclosing not settable",
)
.with_suggestion("Try 'click' to close it instead."));
}
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::COLLAPSE_CHAIN, &ctx)?;
}
Action::Select(value) => {
@ -209,24 +158,22 @@ mod imp {
}
Action::TripleClick => {
crate::actions::activate::smart_triple_activate(el)?;
let caps = discovery::discover(el);
chain_defs::triple_click(el, &caps)?;
}
Action::ScrollTo => {
let ax_action = CFString::new("AXScrollToVisible");
let err =
unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) };
if err != kAXErrorSuccess {
if !ax_helpers::try_ax_action(el, "AXScrollToVisible") {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("AXScrollToVisible failed (err={err})"),
"AXScrollToVisible failed",
)
.with_suggestion("Element may not be inside a scrollable area"));
}
}
Action::Clear => {
ax_set_value(el, "")?;
ax_helpers::ax_set_value(el, "")?;
}
Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => {
@ -236,7 +183,8 @@ mod imp {
"{} requires adapter-level handling, not element action",
label
),
));
)
.with_suggestion("Use the top-level command (e.g. 'hover', 'drag', 'key-down') instead of targeting an element."));
}
_ => {
@ -248,58 +196,18 @@ mod imp {
}
pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> {
let action = CFString::new(kAXPressAction);
let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) };
if err != kAXErrorSuccess {
if !ax_helpers::ax_press(el) {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("{context}: AXPress failed (err={err})"),
));
format!("{context}: AXPress failed"),
)
.with_suggestion("Element may not be pressable. Try 'click' instead."));
}
Ok(())
}
pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> {
let cf_attr = CFString::new(kAXValueAttribute);
let cf_val = CFString::new(val);
let err = unsafe {
AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef())
};
if err != kAXErrorSuccess {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("SetValue failed (err={err})"),
));
}
Ok(())
}
pub fn element_role(el: &AXElement) -> Option<String> {
use accessibility_sys::kAXRoleAttribute;
crate::tree::copy_string_attr(el, kAXRoleAttribute)
.map(|r| crate::tree::roles::ax_role_to_str(&r).to_string())
}
pub fn has_ax_action(el: &AXElement, action_name: &str) -> bool {
let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null();
let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) };
if err != kAXErrorSuccess || actions_ref.is_null() {
return false;
}
let actions: CFArray<CFType> = unsafe { TCFType::wrap_under_create_rule(actions_ref) };
let target = CFString::new(action_name);
for i in 0..actions.len() {
if let Some(name) = actions.get(i).and_then(|v| v.downcast::<CFString>()) {
if name == target {
return true;
}
}
}
false
}
fn check_uncheck(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> {
let role = element_role(el);
let role = ax_helpers::element_role(el);
if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
@ -315,8 +223,11 @@ mod imp {
if is_checked == want_checked {
return Ok(());
}
crate::actions::activate::smart_activate(el)?;
Ok(())
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)
}
}
@ -333,9 +244,7 @@ mod imp {
pub use imp::perform_action;
#[cfg(target_os = "macos")]
pub(crate) use imp::{
ax_press_or_fail, ax_set_value, click_via_bounds, element_role, has_ax_action,
};
pub(crate) use imp::{ax_press_or_fail, click_via_bounds};
fn action_label(action: &Action) -> String {
match action {

15
crates/macos/src/actions/extras.rs
View file

@ -6,12 +6,13 @@ use crate::tree::AXElement;
#[cfg(target_os = "macos")]
pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> {
use crate::actions::dispatch::{ax_press_or_fail, ax_set_value, element_role};
use crate::actions::ax_helpers;
use crate::actions::dispatch::ax_press_or_fail;
let role = element_role(el);
let role = ax_helpers::element_role(el);
match role.as_deref() {
Some("combobox") => {
ax_set_value(el, value)?;
ax_helpers::ax_set_value(el, value)?;
}
Some("popupbutton") | Some("menubutton") => {
ax_press_or_fail(el, "select (open popup)")?;
@ -37,7 +38,7 @@ pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterErr
}
}
_ => {
if ax_set_value(el, value).is_err() {
if ax_helpers::ax_set_value(el, value).is_err() {
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
format!(
@ -155,7 +156,7 @@ pub(crate) fn ax_scroll(
Direction::Right => "AXScrollRightByPage",
Direction::Left => "AXScrollLeftByPage",
};
if crate::actions::dispatch::has_ax_action(target, page_action) {
if crate::actions::ax_helpers::has_ax_action(target, page_action) {
let ax = CFString::new(page_action);
for _ in 0..amount {
unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) };
@ -240,7 +241,7 @@ fn try_scroll_bar_value_shift(
use agent_desktop_core::action::Direction;
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
if !crate::actions::activate::is_attr_settable(bar, "AXValue") {
if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") {
return false;
}
let current = read_scroll_bar_value(bar).unwrap_or(0.0);
@ -347,7 +348,7 @@ fn try_select_row_in_direction(
if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) {
continue;
}
if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") {
if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") {
continue;
}
let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default();

5
crates/macos/src/actions/mod.rs
View file

@ -1,4 +1,7 @@
pub mod activate;
pub mod ax_helpers;
pub mod chain;
pub mod chain_defs;
pub mod discovery;
pub mod dispatch;
pub mod extras;

91
crates/macos/src/input/clipboard.rs
View file

@ -3,35 +3,86 @@ use agent_desktop_core::error::AdapterError;
#[cfg(target_os = "macos")]
mod imp {
use super::*;
use std::process::Command;
use core_foundation::base::TCFType;
use std::ffi::c_void;
type Id = *mut c_void;
type Class = *mut c_void;
type Sel = *mut c_void;
extern "C" {
fn objc_getClass(name: *const core::ffi::c_char) -> Class;
fn sel_registerName(name: *const core::ffi::c_char) -> Sel;
fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id;
static NSPasteboardTypeString: Id;
}
fn pasteboard() -> Result<Id, AdapterError> {
unsafe {
let cls = objc_getClass(c"NSPasteboard".as_ptr());
if cls.is_null() {
return Err(AdapterError::internal("NSPasteboard class not found"));
}
let sel = sel_registerName(c"generalPasteboard".as_ptr());
let send: unsafe extern "C" fn(Class, Sel) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let pb = send(cls, sel);
if pb.is_null() {
return Err(AdapterError::internal("generalPasteboard returned null"));
}
Ok(pb)
}
}
pub fn get() -> Result<String, AdapterError> {
let output = Command::new("pbpaste")
.output()
.map_err(|e| AdapterError::internal(format!("pbpaste failed: {e}")))?;
String::from_utf8(output.stdout)
.map_err(|_| AdapterError::internal("Clipboard contains non-UTF8 data"))
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"stringForType:".as_ptr());
let send: unsafe extern "C" fn(Id, Sel, Id) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let ns_string = send(pb, sel, NSPasteboardTypeString);
if ns_string.is_null() {
return Ok(String::new());
}
let cf_str = core_foundation::string::CFString::wrap_under_get_rule(
ns_string as core_foundation_sys::string::CFStringRef,
);
Ok(cf_str.to_string())
}
}
pub fn set(text: &str) -> Result<(), AdapterError> {
use std::io::Write;
let mut child = Command::new("pbcopy")
.stdin(std::process::Stdio::piped())
.spawn()
.map_err(|e| AdapterError::internal(format!("pbcopy failed: {e}")))?;
if let Some(stdin) = child.stdin.as_mut() {
stdin
.write_all(text.as_bytes())
.map_err(|e| AdapterError::internal(format!("Write to pbcopy failed: {e}")))?;
unsafe {
let pb = pasteboard()?;
let clear_sel = sel_registerName(c"clearContents".as_ptr());
let send_void: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send_void(pb, clear_sel);
let cf_text = core_foundation::string::CFString::new(text);
let ns_text = cf_text.as_concrete_TypeRef() as Id;
let set_sel = sel_registerName(c"setString:forType:".as_ptr());
let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool =
std::mem::transmute(objc_msgSend as *const c_void);
let ok = send_two(pb, set_sel, ns_text, NSPasteboardTypeString);
if !ok {
return Err(AdapterError::internal(
"NSPasteboard setString:forType: failed",
));
}
Ok(())
}
child
.wait()
.map_err(|e| AdapterError::internal(format!("pbcopy wait failed: {e}")))?;
Ok(())
}
pub fn clear() -> Result<(), AdapterError> {
set("")
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"clearContents".as_ptr());
let send: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send(pb, sel);
Ok(())
}
}
}

3
crates/macos/src/input/keyboard.rs
View file

@ -258,7 +258,8 @@ mod imp {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Unknown key: '{other}'"),
))
)
.with_suggestion("Valid keys: a-z, 0-9, return, escape, tab, space, delete, left, right, up, down, f1-f12"))
}
};
Ok(code)

6
crates/macos/src/system/app_ops.rs
View file

@ -93,7 +93,8 @@ pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result<WindowInfo, AdapterE
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Invalid app identifier: '{id}'"),
));
)
.with_suggestion("Use an app name like 'Safari' or bundle ID like 'com.apple.Safari'."));
}
let filter = WindowFilter {
@ -167,7 +168,8 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> {
return Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
format!("Invalid app name '{id}'"),
));
)
.with_suggestion("App name should only contain letters, numbers, spaces, hyphens, dots, or underscores."));
}
let script = format!(
r#"tell application "System Events"

1
crates/macos/src/system/key_dispatch.rs
View file

@ -302,6 +302,7 @@ pub(crate) fn find_pid_by_name(app_name: &str) -> Result<i32, AdapterError> {
agent_desktop_core::error::ErrorCode::AppNotFound,
format!("App '{app_name}' not found"),
)
.with_suggestion("Verify the app is running. Use 'list-apps' to see running applications.")
})
}

12
crates/macos/src/system/window_ops.rs
View file

@ -43,7 +43,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Resize failed (err={err})"),
));
)
.with_suggestion("Window may not support resizing. Try a different size."));
}
Ok(())
}
@ -66,6 +67,9 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Move failed (err={err})"),
)
.with_suggestion(
"Window may not support repositioning. Verify coordinates are on-screen.",
));
}
Ok(())
@ -86,7 +90,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("{op} failed (err={err})"),
));
)
.with_suggestion("Window may not support this operation. Try 'focus-window' first."));
}
Ok(())
}
@ -101,7 +106,8 @@ mod imp {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Zoom button press failed (err={err})"),
));
)
.with_suggestion("Try 'resize-window' with explicit dimensions instead."));
}
Ok(())
}

23
crates/macos/src/tree/resolve.rs
View file

@ -9,7 +9,28 @@ use super::element::{
pub fn resolve_element_impl(entry: &RefEntry) -> Result<NativeHandle, AdapterError> {
let root = element_for_pid(entry.pid);
let mut visited = FxHashSet::default();
find_element_recursive(&root, entry, 0, 20, &mut visited)
if let Ok(handle) = find_element_recursive(&root, entry, 0, 20, &mut visited) {
return Ok(handle);
}
if entry.bounds_hash.is_some() && entry.name.is_some() {
let relaxed = RefEntry {
bounds_hash: None,
..entry.clone()
};
visited.clear();
if let Ok(handle) = find_element_recursive(&root, &relaxed, 0, 20, &mut visited) {
return Ok(handle);
}
}
Err(AdapterError::new(
agent_desktop_core::error::ErrorCode::StaleRef,
format!(
"Element not found: role={}, name={:?}",
entry.role,
entry.name.as_deref().unwrap_or("(none)")
),
)
.with_suggestion("Run 'snapshot' to refresh, then retry with the updated ref."))
}
#[cfg(target_os = "macos")]