diff --git a/crates/core/src/commands/focus_window.rs b/crates/core/src/commands/focus_window.rs index cad7e52..72f87e7 100644 --- a/crates/core/src/commands/focus_window.rs +++ b/crates/core/src/commands/focus_window.rs @@ -34,10 +34,13 @@ pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result pid={} role={} name={:?}", + ref_id, + entry.pid, + entry.role, + entry.name.as_deref().unwrap_or("(none)") + ); let handle = adapter.resolve_element(&entry)?; + tracing::debug!("resolve: {} resolved successfully", ref_id); Ok((entry, handle)) } diff --git a/crates/core/src/commands/snapshot.rs b/crates/core/src/commands/snapshot.rs index a2331aa..6c4f067 100644 --- a/crates/core/src/commands/snapshot.rs +++ b/crates/core/src/commands/snapshot.rs @@ -16,6 +16,14 @@ pub struct SnapshotArgs { } pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result { + tracing::debug!( + "tree: snapshot app={:?} window_id={:?} max_depth={} interactive_only={}", + args.app.as_deref().unwrap_or("(focused)"), + args.window_id.as_deref().unwrap_or("(auto)"), + args.max_depth, + args.interactive_only + ); + let opts = crate::adapter::TreeOptions { max_depth: args.max_depth, include_bounds: args.include_bounds, @@ -35,6 +43,13 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result Result<(), AdapterError> { - let scroll_action = CFString::new("AXScrollToVisible"); - unsafe { AXUIElementPerformAction(el.0, scroll_action.as_concrete_TypeRef()) }; - - let actions = list_ax_actions(el); - - if try_action_from_list(el, &actions, &["AXPress"]) { - return Ok(()); - } - if try_action_from_list(el, &actions, &["AXConfirm"]) { - return Ok(()); - } - if try_action_from_list(el, &actions, &["AXOpen"]) { - return Ok(()); - } - if try_action_from_list(el, &actions, &["AXPick"]) { - return Ok(()); - } - if try_show_alternate_ui(el) { - return Ok(()); - } - - if try_child_activation(el) { - return Ok(()); - } - if try_set_selected(el) { - return Ok(()); - } - if try_select_via_parent(el) { - return Ok(()); - } - if try_custom_actions(el) { - return Ok(()); - } - if try_focus_then_activate(el) { - return Ok(()); - } - if try_keyboard_activate(el) { - return Ok(()); - } - if try_parent_activation(el) { - return Ok(()); - } - - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 1) - } - - /// AXOpen first, then two smart_activate calls with a gap, then CGEvent double-click. - pub fn smart_double_activate(el: &AXElement) -> Result<(), AdapterError> { - let actions = list_ax_actions(el); - if try_action_from_list(el, &actions, &["AXOpen"]) { - return Ok(()); - } - - let _ = smart_activate(el); - std::thread::sleep(std::time::Duration::from_millis(50)); - let _ = smart_activate(el); - - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2) - } - - pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> { - if ax_show_menu(el) { - return Ok(()); - } - - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let _ = crate::system::app_ops::ensure_app_focused(pid); - std::thread::sleep(std::time::Duration::from_millis(50)); - if ax_show_menu(el) { - return Ok(()); - } - } - - if try_select_then_show_menu(el) { - return Ok(()); - } - - if try_focus_then_show_menu(el) { - return Ok(()); - } - - if try_parent_show_menu(el) { - return Ok(()); - } - - if try_child_show_menu(el) { - return Ok(()); - } - crate::actions::dispatch::click_via_bounds(el, MouseButton::Right, 1) - } - - fn ax_show_menu(el: &AXElement) -> bool { - let show = CFString::new("AXShowMenu"); - let err = unsafe { AXUIElementPerformAction(el.0, show.as_concrete_TypeRef()) }; - err == kAXErrorSuccess - } - - fn try_select_then_show_menu(el: &AXElement) -> bool { - if !is_attr_settable(el, "AXSelected") { - return false; - } - let cf_attr = CFString::new("AXSelected"); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_show_menu(el) - } - - fn try_focus_then_show_menu(el: &AXElement) -> bool { - let cf_attr = CFString::new(kAXFocusedAttribute); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_show_menu(el) - } - - fn try_parent_show_menu(el: &AXElement) -> bool { - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..3 { - let ancestor = match ¤t { - Some(a) => a, - None => return false, - }; - if ax_show_menu(ancestor) { - return true; - } - current = crate::tree::copy_element_attr(ancestor, "AXParent"); - } - false - } - - fn try_child_show_menu(el: &AXElement) -> bool { - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in children.iter().take(5) { - if ax_show_menu(child) { - return true; - } - } - false - } - - /// Three smart_activate calls with gaps, then CGEvent triple-click. - pub fn smart_triple_activate(el: &AXElement) -> Result<(), AdapterError> { - for _ in 0..3 { - let _ = smart_activate(el); - std::thread::sleep(std::time::Duration::from_millis(30)); - } - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3) - } - - fn list_ax_actions(el: &AXElement) -> Vec { - let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null(); - let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) }; - if err != kAXErrorSuccess || actions_ref.is_null() { - return Vec::new(); - } - let actions: CFArray = unsafe { TCFType::wrap_under_create_rule(actions_ref) }; - let mut result = Vec::new(); - for i in 0..actions.len() { - if let Some(name) = actions.get(i).and_then(|v| v.downcast::()) { - result.push(name.to_string()); - } - } - result - } - - pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool { - let cf_attr = CFString::new(attr); - let mut settable: c_uchar = 0; - let err = unsafe { - AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable) - }; - err == kAXErrorSuccess && settable != 0 - } - - fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool { - for target in targets { - if actions.iter().any(|a| a == target) { - let action = CFString::new(target); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - if err == kAXErrorSuccess { - return true; - } - } - } - false - } - - fn try_set_selected(el: &AXElement) -> bool { - if !is_attr_settable(el, "AXSelected") { - return false; - } - let cf_attr = CFString::new("AXSelected"); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess - } - - fn try_select_via_parent(el: &AXElement) -> bool { - let parent = match crate::tree::copy_element_attr(el, "AXParent") { - Some(p) => p, - None => return false, - }; - let parent_role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) { - Some(r) => r, - None => return false, - }; - if !matches!(parent_role.as_str(), "AXTable" | "AXOutline" | "AXList") { - return false; - } - if !is_attr_settable(&parent, "AXSelectedRows") { - return false; - } - unsafe { CFRetain(el.0 as CFTypeRef) }; - let el_as_cftype = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; - let arr = CFArray::from_CFTypes(&[el_as_cftype]); - let cf_attr = CFString::new("AXSelectedRows"); - let err = unsafe { - AXUIElementSetAttributeValue( - parent.0, - cf_attr.as_concrete_TypeRef(), - arr.as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess - } - - fn try_focus_then_activate(el: &AXElement) -> bool { - let cf_attr = CFString::new(kAXFocusedAttribute); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let confirm = CFString::new("AXConfirm"); - if unsafe { AXUIElementPerformAction(el.0, confirm.as_concrete_TypeRef()) } - == kAXErrorSuccess - { - return true; - } - let press = CFString::new("AXPress"); - let err = unsafe { AXUIElementPerformAction(el.0, press.as_concrete_TypeRef()) }; - err == kAXErrorSuccess - } - - fn try_child_activation(el: &AXElement) -> bool { - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in children.iter().take(3) { - let child_actions = list_ax_actions(child); - if try_action_from_list(child, &child_actions, &["AXPress", "AXConfirm", "AXOpen"]) { - return true; - } - } - false - } - - fn try_parent_activation(el: &AXElement) -> bool { - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..2 { - let ancestor = match ¤t { - Some(a) => a, - None => return false, - }; - let actions = list_ax_actions(ancestor); - if try_action_from_list(ancestor, &actions, &["AXPress", "AXConfirm"]) { - return true; - } - current = crate::tree::copy_element_attr(ancestor, "AXParent"); - } - false - } - - fn try_show_alternate_ui(el: &AXElement) -> bool { - let actions = list_ax_actions(el); - if !actions.iter().any(|a| a == "AXShowAlternateUI") { - return false; - } - let action = CFString::new("AXShowAlternateUI"); - unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - std::thread::sleep(std::time::Duration::from_millis(100)); - let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); - for child in children.iter().take(5) { - let child_actions = list_ax_actions(child); - if try_action_from_list(child, &child_actions, &["AXPress"]) { - return true; - } - } - false - } - - fn try_custom_actions(el: &AXElement) -> bool { - let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default(); - if custom.is_empty() { - return false; - } - let action = CFString::new("AXPerformCustomAction"); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - err == kAXErrorSuccess - } - - fn try_keyboard_activate(el: &AXElement) -> bool { - use accessibility_sys::AXUIElementPostKeyboardEvent; - let cf_focused = CFString::new(kAXFocusedAttribute); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_focused.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let pid = match crate::system::app_ops::pid_from_element(el) { - Some(p) => p, - None => return false, - }; - let app = crate::tree::element_for_pid(pid); - unsafe { - AXUIElementPostKeyboardEvent(app.0, 0, 49, true); - AXUIElementPostKeyboardEvent(app.0, 0, 49, false); - }; - true - } -} - -#[cfg(not(target_os = "macos"))] -mod imp { - use super::*; - use crate::tree::AXElement; - - pub fn smart_activate(_el: &AXElement) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("smart_activate")) - } - pub fn smart_double_activate(_el: &AXElement) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("smart_double_activate")) - } - pub fn smart_right_activate(_el: &AXElement) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("smart_right_activate")) - } - pub fn smart_triple_activate(_el: &AXElement) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("smart_triple_activate")) - } - pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool { - false - } -} - -pub(crate) use imp::{ - is_attr_settable, smart_activate, smart_double_activate, smart_right_activate, - smart_triple_activate, -}; diff --git a/crates/macos/src/actions/ax_helpers.rs b/crates/macos/src/actions/ax_helpers.rs new file mode 100644 index 0000000..78f7a1a --- /dev/null +++ b/crates/macos/src/actions/ax_helpers.rs @@ -0,0 +1,227 @@ +use agent_desktop_core::error::AdapterError; + +#[cfg(target_os = "macos")] +mod imp { + use super::*; + use crate::tree::AXElement; + use accessibility_sys::{ + kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute, + AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, AXUIElementPerformAction, + AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout, + }; + use core_foundation::{ + array::CFArray, + base::{CFType, TCFType}, + boolean::CFBoolean, + string::CFString, + }; + use std::os::raw::c_uchar; + + pub fn try_ax_action(el: &AXElement, name: &str) -> bool { + let action = CFString::new(name); + let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + err == kAXErrorSuccess + } + + pub fn try_ax_action_retried(el: &AXElement, name: &str) -> bool { + let action = CFString::new(name); + let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + if err == kAXErrorSuccess { + return true; + } + if err == kAXErrorCannotComplete { + std::thread::sleep(std::time::Duration::from_millis(100)); + let retry = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + return retry == kAXErrorSuccess; + } + false + } + + pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool { + let cf_attr = CFString::new(attr); + let cf_val = if value { + CFBoolean::true_value() + } else { + CFBoolean::false_value() + }; + let err = unsafe { + AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) + }; + err == kAXErrorSuccess + } + + pub fn set_ax_string_or_err( + el: &AXElement, + attr: &str, + value: &str, + ) -> Result<(), AdapterError> { + let cf_attr = CFString::new(attr); + let cf_val = CFString::new(value); + let err = unsafe { + AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) + }; + if err != kAXErrorSuccess { + return Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::ActionFailed, + format!("AXSetAttributeValue({attr}) failed (err={err})"), + ) + .with_suggestion("Attribute may be read-only. Try 'click' or 'type' instead.")); + } + Ok(()) + } + + pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool { + let cf_attr = CFString::new(attr); + let mut settable: c_uchar = 0; + let err = unsafe { + AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable) + }; + err == kAXErrorSuccess && settable != 0 + } + + pub fn list_ax_actions(el: &AXElement) -> Vec { + let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null(); + let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) }; + if err != kAXErrorSuccess || actions_ref.is_null() { + return Vec::new(); + } + let actions: CFArray = unsafe { TCFType::wrap_under_create_rule(actions_ref) }; + let mut result = Vec::with_capacity(actions.len() as usize); + for i in 0..actions.len() { + if let Some(name) = actions.get(i).and_then(|v| v.downcast::()) { + result.push(name.to_string()); + } + } + result + } + + pub fn has_ax_action(el: &AXElement, target: &str) -> bool { + list_ax_actions(el).iter().any(|a| a == target) + } + + pub fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool { + for target in targets { + if actions.iter().any(|a| a == target) && try_ax_action(el, target) { + return true; + } + } + false + } + + pub fn try_each_child(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool { + let children = crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default(); + for child in children.iter().take(limit) { + if f(child) { + return true; + } + } + false + } + + pub fn try_each_ancestor(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool { + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..limit { + let ancestor = match ¤t { + Some(a) => a, + None => return false, + }; + if f(ancestor) { + return true; + } + current = crate::tree::copy_element_attr(ancestor, "AXParent"); + } + false + } + + pub fn ensure_visible(el: &AXElement) { + let action = CFString::new("AXScrollToVisible"); + unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; + } + + pub fn set_messaging_timeout(el: &AXElement, seconds: f32) { + unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; + } + + pub fn ax_focus(el: &AXElement) -> bool { + set_ax_bool(el, kAXFocusedAttribute, true) + } + + pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> { + set_ax_string_or_err(el, kAXValueAttribute, val) + } + + pub fn ax_press(el: &AXElement) -> bool { + try_ax_action(el, "AXPress") + } + + pub fn element_role(el: &AXElement) -> Option { + use accessibility_sys::kAXRoleAttribute; + crate::tree::copy_string_attr(el, kAXRoleAttribute) + .map(|r| crate::tree::roles::ax_role_to_str(&r).to_string()) + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::*; + use crate::tree::AXElement; + + pub fn try_ax_action(_el: &AXElement, _name: &str) -> bool { + false + } + pub fn try_ax_action_retried(_el: &AXElement, _name: &str) -> bool { + false + } + pub fn set_ax_bool(_el: &AXElement, _attr: &str, _value: bool) -> bool { + false + } + pub fn set_ax_string_or_err( + _el: &AXElement, + _attr: &str, + _value: &str, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("set_ax_string_or_err")) + } + pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool { + false + } + pub fn list_ax_actions(_el: &AXElement) -> Vec { + Vec::new() + } + pub fn has_ax_action(_el: &AXElement, _target: &str) -> bool { + false + } + pub fn try_action_from_list(_el: &AXElement, _actions: &[String], _targets: &[&str]) -> bool { + false + } + pub fn try_each_child(_el: &AXElement, _f: impl Fn(&AXElement) -> bool, _limit: usize) -> bool { + false + } + pub fn try_each_ancestor( + _el: &AXElement, + _f: impl Fn(&AXElement) -> bool, + _limit: usize, + ) -> bool { + false + } + pub fn ensure_visible(_el: &AXElement) {} + pub fn set_messaging_timeout(_el: &AXElement, _seconds: f32) {} + pub fn ax_focus(_el: &AXElement) -> bool { + false + } + pub fn ax_set_value(_el: &AXElement, _val: &str) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("ax_set_value")) + } + pub fn ax_press(_el: &AXElement) -> bool { + false + } + pub fn element_role(_el: &AXElement) -> Option { + None + } +} + +pub(crate) use imp::{ + ax_focus, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action, + is_attr_settable, list_ax_actions, set_ax_bool, set_ax_string_or_err, set_messaging_timeout, + try_action_from_list, try_ax_action, try_ax_action_retried, try_each_ancestor, try_each_child, +}; diff --git a/crates/macos/src/actions/chain.rs b/crates/macos/src/actions/chain.rs new file mode 100644 index 0000000..23e040e --- /dev/null +++ b/crates/macos/src/actions/chain.rs @@ -0,0 +1,209 @@ +use agent_desktop_core::action::MouseButton; +use agent_desktop_core::error::{AdapterError, ErrorCode}; + +use crate::actions::discovery::ElementCaps; +use crate::tree::AXElement; + +#[allow(dead_code)] +pub enum ChainStep { + Action(&'static str), + SetBool { + attr: &'static str, + value: bool, + }, + SetDynamic { + attr: &'static str, + }, + FocusThenAction(&'static str), + FocusThenSetDynamic { + attr: &'static str, + }, + FocusThenConfirmOrPress, + ChildActions { + actions: &'static [&'static str], + limit: usize, + }, + AncestorActions { + actions: &'static [&'static str], + limit: usize, + }, + Custom { + label: &'static str, + func: fn(&AXElement, &ElementCaps) -> bool, + }, + CGClick { + button: MouseButton, + count: u32, + }, +} + +pub struct ChainDef { + pub pre_scroll: bool, + pub steps: &'static [ChainStep], + pub suggestion: &'static str, +} + +pub struct ChainContext<'a> { + pub dynamic_value: Option<&'a str>, +} + +#[cfg(target_os = "macos")] +mod imp { + use super::*; + use crate::actions::ax_helpers; + use std::time::{Duration, Instant}; + + const CHAIN_TIMEOUT: Duration = Duration::from_secs(10); + + pub fn execute_chain( + el: &AXElement, + caps: &ElementCaps, + def: &ChainDef, + ctx: &ChainContext, + ) -> Result<(), AdapterError> { + let deadline = Instant::now() + CHAIN_TIMEOUT; + let total = def.steps.len(); + + ax_helpers::set_messaging_timeout(el, 3.0); + if def.pre_scroll { + tracing::debug!("chain: pre-scroll AXScrollToVisible"); + ax_helpers::ensure_visible(el); + } + + for (i, step) in def.steps.iter().enumerate() { + if Instant::now() > deadline { + tracing::debug!("chain: timeout after {i}/{total} steps"); + return Err(AdapterError::timeout("Chain execution exceeded 10s")); + } + let label = step_label(step); + if execute_step(el, caps, step, ctx) { + tracing::debug!("chain: [{}/{}] {} -> success", i + 1, total, label); + return Ok(()); + } + tracing::debug!("chain: [{}/{}] {} -> skip", i + 1, total, label); + } + + tracing::debug!("chain: all {total} steps exhausted"); + Err( + AdapterError::new(ErrorCode::ActionFailed, "All chain steps exhausted") + .with_suggestion(def.suggestion), + ) + } + + fn step_label(step: &ChainStep) -> &'static str { + match step { + ChainStep::Action(name) => name, + ChainStep::SetBool { attr, .. } => attr, + ChainStep::SetDynamic { attr } => attr, + ChainStep::FocusThenAction(name) => name, + ChainStep::FocusThenSetDynamic { attr } => attr, + ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress", + ChainStep::ChildActions { .. } => "ChildActions", + ChainStep::AncestorActions { .. } => "AncestorActions", + ChainStep::Custom { label, .. } => label, + ChainStep::CGClick { .. } => "CGClick", + } + } + + fn execute_step( + el: &AXElement, + caps: &ElementCaps, + step: &ChainStep, + ctx: &ChainContext, + ) -> bool { + match step { + ChainStep::Action(name) => ax_helpers::try_ax_action_retried(el, name), + + ChainStep::SetBool { attr, value } => { + let settable = match *attr { + "AXSelected" => caps.settable_selected, + "AXDisclosing" => caps.settable_disclosing, + "AXFocused" => caps.settable_focus, + _ => ax_helpers::is_attr_settable(el, attr), + }; + settable && ax_helpers::set_ax_bool(el, attr, *value) + } + + ChainStep::SetDynamic { attr } => { + let value = match ctx.dynamic_value { + Some(v) => v, + None => return false, + }; + ax_helpers::set_ax_string_or_err(el, attr, value).is_ok() + } + + ChainStep::FocusThenAction(name) => { + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + ax_helpers::try_ax_action_retried(el, name) + } + + ChainStep::FocusThenSetDynamic { attr } => { + let value = match ctx.dynamic_value { + Some(v) => v, + None => return false, + }; + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + ax_helpers::set_ax_string_or_err(el, attr, value).is_ok() + } + + ChainStep::FocusThenConfirmOrPress => { + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + ax_helpers::try_ax_action_retried(el, "AXConfirm") + || ax_helpers::try_ax_action_retried(el, "AXPress") + } + + ChainStep::ChildActions { actions, limit } => ax_helpers::try_each_child( + el, + |child| { + let child_actions = ax_helpers::list_ax_actions(child); + ax_helpers::try_action_from_list(child, &child_actions, actions) + }, + *limit, + ), + + ChainStep::AncestorActions { actions, limit } => ax_helpers::try_each_ancestor( + el, + |ancestor| { + let al = ax_helpers::list_ax_actions(ancestor); + ax_helpers::try_action_from_list(ancestor, &al, actions) + }, + *limit, + ), + + ChainStep::Custom { label: _, func } => func(el, caps), + + ChainStep::CGClick { button, count } => { + crate::actions::dispatch::click_via_bounds(el, button.clone(), *count).is_ok() + } + } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::*; + + pub fn execute_chain( + _el: &AXElement, + _caps: &ElementCaps, + def: &ChainDef, + _ctx: &ChainContext, + ) -> Result<(), AdapterError> { + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Chain execution not supported on this platform", + ) + .with_suggestion(def.suggestion)) + } +} + +pub(crate) use imp::execute_chain; diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs new file mode 100644 index 0000000..9c73b3c --- /dev/null +++ b/crates/macos/src/actions/chain_defs.rs @@ -0,0 +1,361 @@ +use agent_desktop_core::error::AdapterError; + +#[cfg(target_os = "macos")] +mod imp { + use super::*; + use crate::actions::{ + ax_helpers, + chain::{execute_chain, ChainContext, ChainDef, ChainStep}, + discovery::ElementCaps, + }; + use crate::tree::AXElement; + use agent_desktop_core::action::MouseButton; + + pub static CLICK_CHAIN: ChainDef = ChainDef { + pre_scroll: true, + steps: &[ + ChainStep::Action("AXPress"), + ChainStep::Action("AXConfirm"), + ChainStep::Action("AXOpen"), + ChainStep::Action("AXPick"), + ChainStep::Custom { + label: "show_alternate_ui", + func: try_show_alternate_ui, + }, + ChainStep::ChildActions { + actions: &["AXPress", "AXConfirm", "AXOpen"], + limit: 3, + }, + ChainStep::SetBool { + attr: "AXSelected", + value: true, + }, + ChainStep::Custom { + label: "select_via_parent", + func: try_select_via_parent, + }, + ChainStep::Custom { + label: "custom_actions", + func: try_custom_actions, + }, + ChainStep::FocusThenConfirmOrPress, + ChainStep::Custom { + label: "keyboard_activate", + func: try_keyboard_activate, + }, + ChainStep::AncestorActions { + actions: &["AXPress", "AXConfirm"], + limit: 2, + }, + ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }, + ], + suggestion: "Element may not be interactable. Try 'mouse-click --xy X,Y'.", + }; + + pub static RIGHT_CLICK_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::Action("AXShowMenu"), + ChainStep::Custom { + label: "focus_app_show_menu", + func: focus_app_then_show_menu, + }, + ChainStep::Custom { + label: "select_then_show_menu", + func: select_then_show_menu, + }, + ChainStep::FocusThenAction("AXShowMenu"), + ChainStep::AncestorActions { + actions: &["AXShowMenu"], + limit: 3, + }, + ChainStep::ChildActions { + actions: &["AXShowMenu"], + limit: 5, + }, + ChainStep::CGClick { + button: MouseButton::Right, + count: 1, + }, + ], + suggestion: "Try 'mouse-click --button right --xy X,Y'.", + }; + + pub static EXPAND_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::Action("AXExpand"), + ChainStep::SetBool { + attr: "AXDisclosing", + value: true, + }, + ], + suggestion: "Try 'click' to open it instead.", + }; + + pub static COLLAPSE_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::Action("AXCollapse"), + ChainStep::SetBool { + attr: "AXDisclosing", + value: false, + }, + ], + suggestion: "Try 'click' to close it instead.", + }; + + pub static SET_VALUE_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::FocusThenSetDynamic { attr: "AXValue" }, + ], + suggestion: "Try 'clear' then 'type', or check element is a text field.", + }; + + pub static CLEAR_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::FocusThenSetDynamic { attr: "AXValue" }, + ChainStep::Custom { + label: "select_all_delete", + func: select_all_then_delete, + }, + ], + suggestion: "Try 'press cmd+a' then 'press delete'.", + }; + + pub static FOCUS_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetBool { + attr: "AXFocused", + value: true, + }, + ChainStep::Action("AXRaise"), + ChainStep::Action("AXPress"), + ChainStep::SetBool { + attr: "AXSelected", + value: true, + }, + ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }, + ], + suggestion: "Try 'click' to focus the element instead.", + }; + + pub static SCROLL_TO_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::Action("AXScrollToVisible"), + ChainStep::Custom { + label: "walk_parents_scroll", + func: walk_parents_and_scroll, + }, + ], + suggestion: "Element may not be in a scrollable container.", + }; + + fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent; + + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let pid = match crate::system::app_ops::pid_from_element(el) { + Some(p) => p, + None => return false, + }; + let app = crate::tree::element_for_pid(pid); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 55, true); + AXUIElementPostKeyboardEvent(app.0, 0, 0, true); + AXUIElementPostKeyboardEvent(app.0, 0, 0, false); + AXUIElementPostKeyboardEvent(app.0, 0, 55, false); + }; + std::thread::sleep(std::time::Duration::from_millis(30)); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 51, true); + AXUIElementPostKeyboardEvent(app.0, 0, 51, false); + }; + true + } + + fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::kAXRoleAttribute; + + let bounds = match crate::tree::read_bounds(el) { + Some(b) => b, + None => return false, + }; + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..8 { + let parent = match ¤t { + Some(p) => p, + None => return false, + }; + let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute); + if role.as_deref() == Some("AXScrollArea") { + let parent_bounds = match crate::tree::read_bounds(parent) { + Some(b) => b, + None => return false, + }; + let target_y = bounds.y + bounds.height / 2.0; + let visible_mid = parent_bounds.y + parent_bounds.height / 2.0; + if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height { + let dy = if target_y > visible_mid { -5 } else { 5 }; + let cx = parent_bounds.x + parent_bounds.width / 2.0; + let cy = parent_bounds.y + parent_bounds.height / 2.0; + for _ in 0..20 { + let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0); + std::thread::sleep(std::time::Duration::from_millis(16)); + } + } + return true; + } + current = crate::tree::copy_element_attr(parent, "AXParent"); + } + false + } + + fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool { + if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { + return false; + } + ax_helpers::try_ax_action(el, "AXShowAlternateUI"); + std::thread::sleep(std::time::Duration::from_millis(100)); + ax_helpers::try_each_child( + el, + |child| { + let ca = ax_helpers::list_ax_actions(child); + ax_helpers::try_action_from_list(child, &ca, &["AXPress"]) + }, + 5, + ) + } + + fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + + let parent = match crate::tree::copy_element_attr(el, "AXParent") { + Some(p) => p, + None => return false, + }; + let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) { + Some(r) => r, + None => return false, + }; + if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") { + return false; + } + if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") { + return false; + } + unsafe { CFRetain(el.0 as CFTypeRef) }; + let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; + let arr = CFArray::from_CFTypes(&[el_cf]); + let cf_attr = CFString::new("AXSelectedRows"); + let err = unsafe { + AXUIElementSetAttributeValue( + parent.0, + cf_attr.as_concrete_TypeRef(), + arr.as_CFTypeRef(), + ) + }; + err == kAXErrorSuccess + } + + fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool { + let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default(); + if custom.is_empty() { + return false; + } + ax_helpers::try_ax_action(el, "AXPerformCustomAction") + } + + fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent; + + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let pid = match crate::system::app_ops::pid_from_element(el) { + Some(p) => p, + None => return false, + }; + let app = crate::tree::element_for_pid(pid); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 49, true); + AXUIElementPostKeyboardEvent(app.0, 0, 49, false); + }; + true + } + + fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { + let pid = match crate::system::app_ops::pid_from_element(el) { + Some(p) => p, + None => return false, + }; + let _ = crate::system::app_ops::ensure_app_focused(pid); + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_helpers::try_ax_action(el, "AXShowMenu") + } + + fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { + if !ax_helpers::is_attr_settable(el, "AXSelected") { + return false; + } + if !ax_helpers::set_ax_bool(el, "AXSelected", true) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_helpers::try_ax_action(el, "AXShowMenu") + } + + pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> { + if ax_helpers::try_ax_action(el, "AXOpen") { + return Ok(()); + } + let ctx = ChainContext { + dynamic_value: None, + }; + let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); + std::thread::sleep(std::time::Duration::from_millis(50)); + let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); + crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2) + } + + pub fn triple_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> { + let ctx = ChainContext { + dynamic_value: None, + }; + for _ in 0..3 { + let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); + std::thread::sleep(std::time::Duration::from_millis(30)); + } + crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3) + } +} + +#[cfg(not(target_os = "macos"))] +mod imp {} + +#[cfg(target_os = "macos")] +pub(crate) use imp::{ + double_click, triple_click, CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, + FOCUS_CHAIN, RIGHT_CLICK_CHAIN, SCROLL_TO_CHAIN, SET_VALUE_CHAIN, +}; diff --git a/crates/macos/src/actions/discovery.rs b/crates/macos/src/actions/discovery.rs new file mode 100644 index 0000000..d574662 --- /dev/null +++ b/crates/macos/src/actions/discovery.rs @@ -0,0 +1,26 @@ +use crate::tree::AXElement; + +pub struct ElementCaps { + pub settable_focus: bool, + pub settable_selected: bool, + pub settable_disclosing: bool, +} + +#[cfg(target_os = "macos")] +pub fn discover(el: &AXElement) -> ElementCaps { + use crate::actions::ax_helpers; + ElementCaps { + settable_focus: ax_helpers::is_attr_settable(el, "AXFocused"), + settable_selected: ax_helpers::is_attr_settable(el, "AXSelected"), + settable_disclosing: ax_helpers::is_attr_settable(el, "AXDisclosing"), + } +} + +#[cfg(not(target_os = "macos"))] +pub fn discover(_el: &AXElement) -> ElementCaps { + ElementCaps { + settable_focus: false, + settable_selected: false, + settable_disclosing: false, + } +} diff --git a/crates/macos/src/actions/dispatch.rs b/crates/macos/src/actions/dispatch.rs index 9b31503..ae39c6d 100644 --- a/crates/macos/src/actions/dispatch.rs +++ b/crates/macos/src/actions/dispatch.rs @@ -6,23 +6,12 @@ use agent_desktop_core::{ #[cfg(target_os = "macos")] mod imp { use super::*; + use crate::actions::{ + ax_helpers, + chain::{execute_chain, ChainContext}, + chain_defs, discovery, + }; use crate::tree::AXElement; - use accessibility_sys::{ - kAXErrorSuccess, kAXFocusedAttribute, kAXPressAction, kAXValueAttribute, - AXUIElementCopyActionNames, AXUIElementPerformAction, AXUIElementSetAttributeValue, - }; - use core_foundation::{ - array::CFArray, - base::{CFType, TCFType}, - boolean::CFBoolean, - string::CFString, - }; - - pub fn try_ax_action(el: &AXElement, name: &str) -> bool { - let action = CFString::new(name); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - err == kAXErrorSuccess - } pub fn click_via_bounds( el: &AXElement, @@ -37,10 +26,10 @@ mod imp { .with_suggestion("AX action failed and CGEvent fallback unavailable") })?; if bounds.width <= 0.0 || bounds.height <= 0.0 { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - "Element has zero-size bounds", - )); + return Err( + AdapterError::new(ErrorCode::ActionFailed, "Element has zero-size bounds") + .with_suggestion("Element may be hidden or off-screen. Try 'scroll-to' first."), + ); } let center = Point { x: bounds.x + bounds.width / 2.0, @@ -71,21 +60,31 @@ mod imp { pub fn perform_action(el: &AXElement, action: &Action) -> Result { let label = action_label(action); + tracing::debug!("action: perform {label}"); match action { Action::Click => { - crate::actions::activate::smart_activate(el)?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?; } Action::DoubleClick => { - crate::actions::activate::smart_double_activate(el)?; + let caps = discovery::discover(el); + chain_defs::double_click(el, &caps)?; } Action::RightClick => { - crate::actions::activate::smart_right_activate(el)?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::RIGHT_CLICK_CHAIN, &ctx)?; } Action::Toggle => { - let role = element_role(el); + let role = ax_helpers::element_role(el); if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) { return Err(AdapterError::new( ErrorCode::ActionNotSupported, @@ -98,40 +97,31 @@ mod imp { "Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.", )); } - crate::actions::activate::smart_activate(el)?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?; } Action::SetValue(val) => { - ax_set_value(el, val)?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: Some(val), + }; + execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?; } Action::SetFocus => { - let cf_attr = CFString::new(kAXFocusedAttribute); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("SetFocus failed (err={err})"), - )); - } + execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?; } Action::TypeText(text) => { - let cf_attr = CFString::new(kAXFocusedAttribute); - unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - crate::input::keyboard::synthesize_text(text)?; + execute_type(el, text)?; } Action::PressKey(combo) => { @@ -139,57 +129,19 @@ mod imp { } Action::Expand => { - if !try_ax_action(el, "AXExpand") { - if crate::actions::activate::is_attr_settable(el, "AXDisclosing") { - let cf_attr = CFString::new("AXDisclosing"); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("AXDisclosing set to true failed (err={err})"), - )); - } - } else { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - "AXExpand failed and AXDisclosing not settable", - ) - .with_suggestion("Try 'click' to open it instead.")); - } - } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::EXPAND_CHAIN, &ctx)?; } Action::Collapse => { - if !try_ax_action(el, "AXCollapse") { - if crate::actions::activate::is_attr_settable(el, "AXDisclosing") { - let cf_attr = CFString::new("AXDisclosing"); - let err = unsafe { - AXUIElementSetAttributeValue( - el.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::false_value().as_CFTypeRef(), - ) - }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("AXDisclosing set to false failed (err={err})"), - )); - } - } else { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - "AXCollapse failed and AXDisclosing not settable", - ) - .with_suggestion("Try 'click' to close it instead.")); - } - } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::COLLAPSE_CHAIN, &ctx)?; } Action::Select(value) => { @@ -209,24 +161,24 @@ mod imp { } Action::TripleClick => { - crate::actions::activate::smart_triple_activate(el)?; + let caps = discovery::discover(el); + chain_defs::triple_click(el, &caps)?; } Action::ScrollTo => { - let ax_action = CFString::new("AXScrollToVisible"); - let err = - unsafe { AXUIElementPerformAction(el.0, ax_action.as_concrete_TypeRef()) }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("AXScrollToVisible failed (err={err})"), - ) - .with_suggestion("Element may not be inside a scrollable area")); - } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?; } Action::Clear => { - ax_set_value(el, "")?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: Some(""), + }; + execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?; } Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => { @@ -236,7 +188,8 @@ mod imp { "{} requires adapter-level handling, not element action", label ), - )); + ) + .with_suggestion("Use the top-level command (e.g. 'hover', 'drag', 'key-down') instead of targeting an element.")); } _ => { @@ -244,62 +197,91 @@ mod imp { } } - Ok(ActionResult::new(label)) + let mut result = ActionResult::new(label); + if let Some(state) = read_post_state(el, action) { + result = result.with_state(state); + } + Ok(result) + } + + fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + ax_helpers::ax_focus(el); + std::thread::sleep(std::time::Duration::from_millis(50)); + let has_non_ascii = !text.is_ascii(); + if has_non_ascii { + type_via_clipboard_paste(el, text) + } else { + crate::input::keyboard::synthesize_text(text) + } + } + + fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> { + let saved = crate::input::clipboard::get().ok(); + crate::input::clipboard::set(text)?; + std::thread::sleep(std::time::Duration::from_millis(50)); + + crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo { + key: "v".into(), + modifiers: vec![agent_desktop_core::action::Modifier::Cmd], + })?; + std::thread::sleep(std::time::Duration::from_millis(100)); + + if let Some(prev) = saved { + let _ = crate::input::clipboard::set(&prev); + } + Ok(()) + } + + fn read_post_state( + el: &AXElement, + action: &Action, + ) -> Option { + let delay_ms = match action { + Action::Click + | Action::Toggle + | Action::Check + | Action::Uncheck + | Action::TypeText(_) => 50, + Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0, + _ => return None, + }; + if delay_ms > 0 { + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + } + let value = crate::tree::copy_value_typed(el); + let role = ax_helpers::element_role(el).unwrap_or_default(); + let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false); + let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true); + let mut states = Vec::new(); + if focused { + states.push("focused".into()); + } + if !enabled { + states.push("disabled".into()); + } + Some(agent_desktop_core::action::ElementState { + role, + states, + value, + }) } pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> { - let action = CFString::new(kAXPressAction); - let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - if err != kAXErrorSuccess { + if !ax_helpers::ax_press(el) { return Err(AdapterError::new( ErrorCode::ActionFailed, - format!("{context}: AXPress failed (err={err})"), - )); + format!("{context}: AXPress failed"), + ) + .with_suggestion("Element may not be pressable. Try 'click' instead.")); } Ok(()) } - pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> { - let cf_attr = CFString::new(kAXValueAttribute); - let cf_val = CFString::new(val); - let err = unsafe { - AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) - }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("SetValue failed (err={err})"), - )); - } - Ok(()) - } - - pub fn element_role(el: &AXElement) -> Option { - use accessibility_sys::kAXRoleAttribute; - crate::tree::copy_string_attr(el, kAXRoleAttribute) - .map(|r| crate::tree::roles::ax_role_to_str(&r).to_string()) - } - - pub fn has_ax_action(el: &AXElement, action_name: &str) -> bool { - let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null(); - let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) }; - if err != kAXErrorSuccess || actions_ref.is_null() { - return false; - } - let actions: CFArray = unsafe { TCFType::wrap_under_create_rule(actions_ref) }; - let target = CFString::new(action_name); - for i in 0..actions.len() { - if let Some(name) = actions.get(i).and_then(|v| v.downcast::()) { - if name == target { - return true; - } - } - } - false - } - fn check_uncheck(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> { - let role = element_role(el); + let role = ax_helpers::element_role(el); if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) { return Err(AdapterError::new( ErrorCode::ActionNotSupported, @@ -315,8 +297,11 @@ mod imp { if is_checked == want_checked { return Ok(()); } - crate::actions::activate::smart_activate(el)?; - Ok(()) + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx) } } @@ -333,9 +318,7 @@ mod imp { pub use imp::perform_action; #[cfg(target_os = "macos")] -pub(crate) use imp::{ - ax_press_or_fail, ax_set_value, click_via_bounds, element_role, has_ax_action, -}; +pub(crate) use imp::{ax_press_or_fail, click_via_bounds}; fn action_label(action: &Action) -> String { match action { diff --git a/crates/macos/src/actions/extras.rs b/crates/macos/src/actions/extras.rs index 9cfdfc7..e6d4274 100644 --- a/crates/macos/src/actions/extras.rs +++ b/crates/macos/src/actions/extras.rs @@ -6,12 +6,13 @@ use crate::tree::AXElement; #[cfg(target_os = "macos")] pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> { - use crate::actions::dispatch::{ax_press_or_fail, ax_set_value, element_role}; + use crate::actions::ax_helpers; + use crate::actions::dispatch::ax_press_or_fail; - let role = element_role(el); + let role = ax_helpers::element_role(el); match role.as_deref() { Some("combobox") => { - ax_set_value(el, value)?; + ax_helpers::ax_set_value(el, value)?; } Some("popupbutton") | Some("menubutton") => { ax_press_or_fail(el, "select (open popup)")?; @@ -37,7 +38,7 @@ pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterErr } } _ => { - if ax_set_value(el, value).is_err() { + if ax_helpers::ax_set_value(el, value).is_err() { return Err(AdapterError::new( ErrorCode::ActionNotSupported, format!( @@ -155,7 +156,7 @@ pub(crate) fn ax_scroll( Direction::Right => "AXScrollRightByPage", Direction::Left => "AXScrollLeftByPage", }; - if crate::actions::dispatch::has_ax_action(target, page_action) { + if crate::actions::ax_helpers::has_ax_action(target, page_action) { let ax = CFString::new(page_action); for _ in 0..amount { unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) }; @@ -240,7 +241,7 @@ fn try_scroll_bar_value_shift( use agent_desktop_core::action::Direction; use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; - if !crate::actions::activate::is_attr_settable(bar, "AXValue") { + if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") { return false; } let current = read_scroll_bar_value(bar).unwrap_or(0.0); @@ -347,7 +348,7 @@ fn try_select_row_in_direction( if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) { continue; } - if !crate::actions::activate::is_attr_settable(child, "AXSelectedRows") { + if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") { continue; } let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default(); diff --git a/crates/macos/src/actions/mod.rs b/crates/macos/src/actions/mod.rs index c833ceb..3d9ba09 100644 --- a/crates/macos/src/actions/mod.rs +++ b/crates/macos/src/actions/mod.rs @@ -1,4 +1,7 @@ -pub mod activate; +pub mod ax_helpers; +pub mod chain; +pub mod chain_defs; +pub mod discovery; pub mod dispatch; pub mod extras; diff --git a/crates/macos/src/input/clipboard.rs b/crates/macos/src/input/clipboard.rs index 1fbaf47..a1805f2 100644 --- a/crates/macos/src/input/clipboard.rs +++ b/crates/macos/src/input/clipboard.rs @@ -3,35 +3,92 @@ use agent_desktop_core::error::AdapterError; #[cfg(target_os = "macos")] mod imp { use super::*; - use std::process::Command; + use core_foundation::base::TCFType; + use std::ffi::c_void; + + type Id = *mut c_void; + type Class = *mut c_void; + type Sel = *mut c_void; + + extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> Class; + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; + static NSPasteboardTypeString: Id; + } + + fn pasteboard() -> Result { + unsafe { + let cls = objc_getClass(c"NSPasteboard".as_ptr()); + if cls.is_null() { + return Err(AdapterError::internal("NSPasteboard class not found")); + } + let sel = sel_registerName(c"generalPasteboard".as_ptr()); + let send: unsafe extern "C" fn(Class, Sel) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let pb = send(cls, sel); + if pb.is_null() { + return Err(AdapterError::internal("generalPasteboard returned null")); + } + Ok(pb) + } + } pub fn get() -> Result { - let output = Command::new("pbpaste") - .output() - .map_err(|e| AdapterError::internal(format!("pbpaste failed: {e}")))?; - String::from_utf8(output.stdout) - .map_err(|_| AdapterError::internal("Clipboard contains non-UTF8 data")) + tracing::debug!("clipboard: get"); + unsafe { + let pb = pasteboard()?; + let sel = sel_registerName(c"stringForType:".as_ptr()); + let send: unsafe extern "C" fn(Id, Sel, Id) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let ns_string = send(pb, sel, NSPasteboardTypeString); + if ns_string.is_null() { + tracing::debug!("clipboard: get -> empty"); + return Ok(String::new()); + } + let cf_str = core_foundation::string::CFString::wrap_under_get_rule( + ns_string as core_foundation_sys::string::CFStringRef, + ); + let result = cf_str.to_string(); + tracing::debug!("clipboard: get -> {} chars", result.len()); + Ok(result) + } } pub fn set(text: &str) -> Result<(), AdapterError> { - use std::io::Write; - let mut child = Command::new("pbcopy") - .stdin(std::process::Stdio::piped()) - .spawn() - .map_err(|e| AdapterError::internal(format!("pbcopy failed: {e}")))?; - if let Some(stdin) = child.stdin.as_mut() { - stdin - .write_all(text.as_bytes()) - .map_err(|e| AdapterError::internal(format!("Write to pbcopy failed: {e}")))?; + tracing::debug!("clipboard: set {} chars", text.len()); + unsafe { + let pb = pasteboard()?; + let clear_sel = sel_registerName(c"clearContents".as_ptr()); + let send_void: unsafe extern "C" fn(Id, Sel) = + std::mem::transmute(objc_msgSend as *const c_void); + send_void(pb, clear_sel); + + let cf_text = core_foundation::string::CFString::new(text); + let ns_text = cf_text.as_concrete_TypeRef() as Id; + let set_sel = sel_registerName(c"setString:forType:".as_ptr()); + let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool = + std::mem::transmute(objc_msgSend as *const c_void); + let ok = send_two(pb, set_sel, ns_text, NSPasteboardTypeString); + if !ok { + return Err(AdapterError::internal( + "NSPasteboard setString:forType: failed", + )); + } + Ok(()) } - child - .wait() - .map_err(|e| AdapterError::internal(format!("pbcopy wait failed: {e}")))?; - Ok(()) } pub fn clear() -> Result<(), AdapterError> { - set("") + tracing::debug!("clipboard: clear"); + unsafe { + let pb = pasteboard()?; + let sel = sel_registerName(c"clearContents".as_ptr()); + let send: unsafe extern "C" fn(Id, Sel) = + std::mem::transmute(objc_msgSend as *const c_void); + send(pb, sel); + Ok(()) + } } } diff --git a/crates/macos/src/input/keyboard.rs b/crates/macos/src/input/keyboard.rs index 4f86c17..afde81a 100644 --- a/crates/macos/src/input/keyboard.rs +++ b/crates/macos/src/input/keyboard.rs @@ -8,6 +8,23 @@ mod imp { }; pub fn synthesize_key(combo: &KeyCombo) -> Result<(), AdapterError> { + tracing::debug!( + "keyboard: synthesize_key {}{}", + if combo.modifiers.is_empty() { + String::new() + } else { + format!( + "{}+", + combo + .modifiers + .iter() + .map(|m| format!("{m:?}")) + .collect::>() + .join("+") + ) + }, + combo.key + ); let key_code = key_name_to_code(&combo.key)?; let sys_wide = unsafe { AXUIElementCreateSystemWide() }; @@ -54,6 +71,7 @@ mod imp { } pub fn synthesize_text(text: &str) -> Result<(), AdapterError> { + tracing::debug!("keyboard: synthesize_text {} chars", text.len()); let sys_wide = unsafe { AXUIElementCreateSystemWide() }; if sys_wide.is_null() { return Err(AdapterError::internal( @@ -258,7 +276,8 @@ mod imp { return Err(AdapterError::new( agent_desktop_core::error::ErrorCode::InvalidArgs, format!("Unknown key: '{other}'"), - )) + ) + .with_suggestion("Valid keys: a-z, 0-9, return, escape, tab, space, delete, left, right, up, down, f1-f12")) } }; Ok(code) diff --git a/crates/macos/src/input/mouse.rs b/crates/macos/src/input/mouse.rs index 658fb0b..7f618e4 100644 --- a/crates/macos/src/input/mouse.rs +++ b/crates/macos/src/input/mouse.rs @@ -11,6 +11,13 @@ mod imp { use core_graphics::geometry::CGPoint; pub fn synthesize_mouse(event: MouseEvent) -> Result<(), AdapterError> { + tracing::debug!( + "mouse: {:?} {:?} at ({:.0}, {:.0})", + event.kind, + event.button, + event.point.x, + event.point.y + ); let point = CGPoint::new(event.point.x, event.point.y); let cg_button = to_cg_button(&event.button); match event.kind { @@ -24,6 +31,14 @@ mod imp { } pub fn synthesize_drag(params: DragParams) -> Result<(), AdapterError> { + tracing::debug!( + "mouse: drag ({:.0},{:.0}) -> ({:.0},{:.0}) duration={}ms", + params.from.x, + params.from.y, + params.to.x, + params.to.y, + params.duration_ms.unwrap_or(300) + ); let from = CGPoint::new(params.from.x, params.from.y); let to = CGPoint::new(params.to.x, params.to.y); let duration_ms = params.duration_ms.unwrap_or(300); @@ -31,7 +46,7 @@ mod imp { let step_delay = std::time::Duration::from_millis(duration_ms / steps as u64); post_event(CGEventType::LeftMouseDown, from, CGMouseButton::Left)?; - std::thread::sleep(std::time::Duration::from_millis(50)); + std::thread::sleep(std::time::Duration::from_millis(200)); for i in 1..=steps { let t = i as f64 / steps as f64; @@ -45,6 +60,7 @@ mod imp { std::thread::sleep(step_delay); } + std::thread::sleep(std::time::Duration::from_millis(500)); post_event(CGEventType::LeftMouseUp, to, CGMouseButton::Left) } @@ -131,6 +147,7 @@ mod imp { } pub fn synthesize_scroll_at(x: f64, y: f64, dy: i32, dx: i32) -> Result<(), AdapterError> { + tracing::debug!("mouse: scroll at ({x:.0},{y:.0}) dy={dy} dx={dx}"); use core_graphics::geometry::CGPoint; extern "C" { diff --git a/crates/macos/src/system/app_ops.rs b/crates/macos/src/system/app_ops.rs index d54b19e..2bee50e 100644 --- a/crates/macos/src/system/app_ops.rs +++ b/crates/macos/src/system/app_ops.rs @@ -13,6 +13,7 @@ pub fn pid_from_element(el: &crate::tree::AXElement) -> Option { #[cfg(target_os = "macos")] pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> { + tracing::debug!("system: ensure_app_focused pid={pid}"); use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; @@ -36,6 +37,11 @@ pub fn ensure_app_focused(pid: i32) -> Result<(), AdapterError> { #[cfg(target_os = "macos")] pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> { + tracing::debug!( + "system: focus_window app={:?} title={:?}", + win.app, + win.title + ); use accessibility_sys::{ kAXErrorSuccess, AXUIElementCreateApplication, AXUIElementPerformAction, AXUIElementSetAttributeValue, @@ -85,6 +91,7 @@ pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> { #[cfg(target_os = "macos")] pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result { + tracing::debug!("system: launch app={id:?} timeout={timeout_ms}ms"); use crate::adapter::list_windows_impl; use std::process::Command; use std::time::{Duration, Instant}; @@ -93,7 +100,8 @@ pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result Result Result<(), AdapterError> { + tracing::debug!("system: close app={id:?} force={force}"); use std::process::Command; if force { Command::new("pkill") @@ -167,7 +176,8 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> { return Err(AdapterError::new( agent_desktop_core::error::ErrorCode::InvalidArgs, format!("Invalid app name '{id}'"), - )); + ) + .with_suggestion("App name should only contain letters, numbers, spaces, hyphens, dots, or underscores.")); } let script = format!( r#"tell application "System Events" diff --git a/crates/macos/src/system/key_dispatch.rs b/crates/macos/src/system/key_dispatch.rs index 002c5a7..55717eb 100644 --- a/crates/macos/src/system/key_dispatch.rs +++ b/crates/macos/src/system/key_dispatch.rs @@ -11,6 +11,7 @@ pub fn press_for_app_impl(app_name: &str, combo: &KeyCombo) -> Result Result { agent_desktop_core::error::ErrorCode::AppNotFound, format!("App '{app_name}' not found"), ) + .with_suggestion("Verify the app is running. Use 'list-apps' to see running applications.") }) } diff --git a/crates/macos/src/system/screenshot.rs b/crates/macos/src/system/screenshot.rs index c62ec7f..fc4215e 100644 --- a/crates/macos/src/system/screenshot.rs +++ b/crates/macos/src/system/screenshot.rs @@ -6,6 +6,7 @@ mod imp { use std::process::Command; pub fn capture_app(pid: i32) -> Result { + tracing::debug!("system: screenshot app pid={pid}"); let temp = temp_path(); let cg_id = find_cg_window_id_for_pid(pid); @@ -31,6 +32,7 @@ mod imp { } pub fn capture_screen(_idx: usize) -> Result { + tracing::debug!("system: screenshot screen"); let temp = temp_path(); let status = Command::new("screencapture") .args(["-x", "-t", "png"]) diff --git a/crates/macos/src/system/window_ops.rs b/crates/macos/src/system/window_ops.rs index 174090e..5014e18 100644 --- a/crates/macos/src/system/window_ops.rs +++ b/crates/macos/src/system/window_ops.rs @@ -17,6 +17,12 @@ mod imp { } pub fn execute(win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { + tracing::debug!( + "system: window_op {:?} app={:?} title={:?}", + op, + win.app, + win.title + ); let win_el = crate::tree::window_element_for(win.pid, &win.title); match op { WindowOp::Resize { width, height } => set_size(&win_el, width, height), @@ -43,7 +49,8 @@ mod imp { return Err(AdapterError::new( ErrorCode::ActionFailed, format!("Resize failed (err={err})"), - )); + ) + .with_suggestion("Window may not support resizing. Try a different size.")); } Ok(()) } @@ -66,6 +73,9 @@ mod imp { return Err(AdapterError::new( ErrorCode::ActionFailed, format!("Move failed (err={err})"), + ) + .with_suggestion( + "Window may not support repositioning. Verify coordinates are on-screen.", )); } Ok(()) @@ -86,7 +96,8 @@ mod imp { return Err(AdapterError::new( ErrorCode::ActionFailed, format!("{op} failed (err={err})"), - )); + ) + .with_suggestion("Window may not support this operation. Try 'focus-window' first.")); } Ok(()) } @@ -101,7 +112,8 @@ mod imp { return Err(AdapterError::new( ErrorCode::ActionFailed, format!("Zoom button press failed (err={err})"), - )); + ) + .with_suggestion("Try 'resize-window' with explicit dimensions instead.")); } Ok(()) } diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index 93a607c..fd57870 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -161,7 +161,7 @@ mod imp { cf_type.downcast::().map(|s| s.to_string()) } - fn copy_value_typed(el: &AXElement) -> Option { + pub fn copy_value_typed(el: &AXElement) -> Option { let cf_attr = CFString::new(kAXValueAttribute); let mut val_ref: CFTypeRef = std::ptr::null_mut(); let err = unsafe { @@ -335,6 +335,9 @@ mod imp { pub fn resolve_element_name(_el: &AXElement) -> Option { None } + pub fn copy_value_typed(_el: &AXElement) -> Option { + None + } pub fn fetch_node_attrs( _el: &AXElement, ) -> ( @@ -350,6 +353,6 @@ mod imp { } pub use imp::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid, - fetch_node_attrs, read_bounds, resolve_element_name, AXElement, + copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed, + element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement, }; diff --git a/crates/macos/src/tree/mod.rs b/crates/macos/src/tree/mod.rs index c1daf45..c26c51c 100644 --- a/crates/macos/src/tree/mod.rs +++ b/crates/macos/src/tree/mod.rs @@ -6,8 +6,8 @@ pub mod surfaces; pub use builder::{build_subtree, window_element_for}; pub use element::{ - copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, read_bounds, - resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH, + copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid, + read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH, }; pub use resolve::{find_element_recursive, resolve_element_impl}; pub use roles::{ax_role_to_str, is_interactive_role}; diff --git a/crates/macos/src/tree/resolve.rs b/crates/macos/src/tree/resolve.rs index e7a1b43..dc4de06 100644 --- a/crates/macos/src/tree/resolve.rs +++ b/crates/macos/src/tree/resolve.rs @@ -7,9 +7,41 @@ use super::element::{ #[cfg(target_os = "macos")] pub fn resolve_element_impl(entry: &RefEntry) -> Result { + tracing::debug!( + "resolve: searching pid={} role={} name={:?} bounds_hash={:?}", + entry.pid, + entry.role, + entry.name.as_deref().unwrap_or("(none)"), + entry.bounds_hash + ); let root = element_for_pid(entry.pid); let mut visited = FxHashSet::default(); - find_element_recursive(&root, entry, 0, 20, &mut visited) + if let Ok(handle) = find_element_recursive(&root, entry, 0, 20, &mut visited) { + tracing::debug!("resolve: found exact match"); + return Ok(handle); + } + if entry.bounds_hash.is_some() && entry.name.is_some() { + tracing::debug!("resolve: exact match failed, trying relaxed (name-only)"); + let relaxed = RefEntry { + bounds_hash: None, + ..entry.clone() + }; + visited.clear(); + if let Ok(handle) = find_element_recursive(&root, &relaxed, 0, 20, &mut visited) { + tracing::debug!("resolve: found via relaxed match (bounds changed)"); + return Ok(handle); + } + } + tracing::debug!("resolve: element not found"); + Err(AdapterError::new( + agent_desktop_core::error::ErrorCode::StaleRef, + format!( + "Element not found: role={}, name={:?}", + entry.role, + entry.name.as_deref().unwrap_or("(none)") + ), + ) + .with_suggestion("Run 'snapshot' to refresh, then retry with the updated ref.")) } #[cfg(target_os = "macos")] diff --git a/src/cli_args.rs b/src/cli_args.rs index e5d011f..2048a73 100644 --- a/src/cli_args.rs +++ b/src/cli_args.rs @@ -12,6 +12,21 @@ pub enum Surface { Alert, } +impl Surface { + pub fn to_core(&self) -> agent_desktop_core::adapter::SnapshotSurface { + use agent_desktop_core::adapter::SnapshotSurface; + match self { + Self::Window => SnapshotSurface::Window, + Self::Focused => SnapshotSurface::Focused, + Self::Menu => SnapshotSurface::Menu, + Self::Menubar => SnapshotSurface::Menubar, + Self::Sheet => SnapshotSurface::Sheet, + Self::Popover => SnapshotSurface::Popover, + Self::Alert => SnapshotSurface::Alert, + } + } +} + #[derive(Parser, Debug)] pub struct SnapshotArgs { #[arg(long, help = "Filter to application by name")] diff --git a/src/dispatch.rs b/src/dispatch.rs index 10fc0cc..c04cd5d 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -16,6 +16,7 @@ use serde_json::Value; use crate::cli::Commands; pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result { + tracing::debug!("dispatch: {}", cmd.name()); match cmd { Commands::Snapshot(a) => snapshot::execute( snapshot::SnapshotArgs { @@ -25,7 +26,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result) -> Result, AppError> { None => Ok(None), } } - -fn cli_surface_to_core(s: &crate::cli::Surface) -> agent_desktop_core::adapter::SnapshotSurface { - use crate::cli::Surface; - use agent_desktop_core::adapter::SnapshotSurface; - match s { - Surface::Window => SnapshotSurface::Window, - Surface::Focused => SnapshotSurface::Focused, - Surface::Menu => SnapshotSurface::Menu, - Surface::Menubar => SnapshotSurface::Menubar, - Surface::Sheet => SnapshotSurface::Sheet, - Surface::Popover => SnapshotSurface::Popover, - Surface::Alert => SnapshotSurface::Alert, - } -}