diff --git a/crates/core/src/commands/helpers.rs b/crates/core/src/commands/helpers.rs index a13e8a0..e30fb56 100644 --- a/crates/core/src/commands/helpers.rs +++ b/crates/core/src/commands/helpers.rs @@ -13,7 +13,10 @@ pub fn resolve_ref( adapter: &dyn PlatformAdapter, ) -> Result<(RefEntry, NativeHandle), AppError> { validate_ref_id(ref_id)?; - let refmap = RefMap::load().map_err(|_| AppError::stale_ref(ref_id))?; + let refmap = RefMap::load().map_err(|e| { + tracing::debug!("refmap load failed: {e}"); + AppError::stale_ref(ref_id) + })?; let entry = refmap .get(ref_id) .ok_or_else(|| AppError::stale_ref(ref_id))? diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index 3b038b9..1b321a2 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -1,4 +1,4 @@ -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccessibilityNode { @@ -31,12 +31,20 @@ pub struct AccessibilityNode { #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct Rect { + #[serde(default, deserialize_with = "f64_or_zero")] pub x: f64, + #[serde(default, deserialize_with = "f64_or_zero")] pub y: f64, + #[serde(default, deserialize_with = "f64_or_zero")] pub width: f64, + #[serde(default, deserialize_with = "f64_or_zero")] pub height: f64, } +fn f64_or_zero<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Option::::deserialize(deserializer).map(|opt| opt.unwrap_or(0.0)) +} + impl Rect { pub fn bounds_hash(&self) -> u64 { use rustc_hash::FxHasher; @@ -82,3 +90,39 @@ pub struct SurfaceInfo { #[serde(skip_serializing_if = "Option::is_none")] pub item_count: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rect_null_fields_deserialize() { + let json = r#"{"x": null, "y": null, "width": 0.0, "height": 0.0}"#; + let rect: Rect = serde_json::from_str(json).unwrap(); + assert_eq!(rect.x, 0.0); + assert_eq!(rect.y, 0.0); + } + + #[test] + fn test_rect_missing_fields_deserialize() { + let json = r#"{"width": 100.0, "height": 50.0}"#; + let rect: Rect = serde_json::from_str(json).unwrap(); + assert_eq!(rect.x, 0.0); + assert_eq!(rect.y, 0.0); + assert_eq!(rect.width, 100.0); + } + + #[test] + fn test_rect_normal_roundtrip() { + let rect = Rect { + x: 10.5, + y: 20.3, + width: 100.0, + height: 50.0, + }; + let json = serde_json::to_string(&rect).unwrap(); + let back: Rect = serde_json::from_str(&json).unwrap(); + assert_eq!(back.x, 10.5); + assert_eq!(back.width, 100.0); + } +} diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs index 9c73b3c..820abc1 100644 --- a/crates/macos/src/actions/chain_defs.rs +++ b/crates/macos/src/actions/chain_defs.rs @@ -6,6 +6,7 @@ mod imp { use crate::actions::{ ax_helpers, chain::{execute_chain, ChainContext, ChainDef, ChainStep}, + chain_steps, discovery::ElementCaps, }; use crate::tree::AXElement; @@ -14,34 +15,48 @@ mod imp { pub static CLICK_CHAIN: ChainDef = ChainDef { pre_scroll: true, steps: &[ - ChainStep::Action("AXPress"), + ChainStep::Custom { + label: "verified_press", + func: chain_steps::do_verified_press, + }, ChainStep::Action("AXConfirm"), ChainStep::Action("AXOpen"), ChainStep::Action("AXPick"), ChainStep::Custom { label: "show_alternate_ui", - func: try_show_alternate_ui, + func: chain_steps::try_show_alternate_ui, }, ChainStep::ChildActions { actions: &["AXPress", "AXConfirm", "AXOpen"], limit: 3, }, + ChainStep::Custom { + label: "value_relay", + func: chain_steps::try_value_relay, + }, ChainStep::SetBool { attr: "AXSelected", value: true, }, + ChainStep::Custom { + label: "parent_row_select", + func: chain_steps::try_parent_row_select, + }, ChainStep::Custom { label: "select_via_parent", - func: try_select_via_parent, + func: chain_steps::try_select_via_parent, }, ChainStep::Custom { label: "custom_actions", - func: try_custom_actions, + func: chain_steps::try_custom_actions, + }, + ChainStep::Custom { + label: "focus_verified_confirm_or_press", + func: chain_steps::try_focus_then_verified_confirm_or_press, }, - ChainStep::FocusThenConfirmOrPress, ChainStep::Custom { label: "keyboard_activate", - func: try_keyboard_activate, + func: chain_steps::try_keyboard_activate, }, ChainStep::AncestorActions { actions: &["AXPress", "AXConfirm"], @@ -61,11 +76,11 @@ mod imp { ChainStep::Action("AXShowMenu"), ChainStep::Custom { label: "focus_app_show_menu", - func: focus_app_then_show_menu, + func: chain_steps::focus_app_then_show_menu, }, ChainStep::Custom { label: "select_then_show_menu", - func: select_then_show_menu, + func: chain_steps::select_then_show_menu, }, ChainStep::FocusThenAction("AXShowMenu"), ChainStep::AncestorActions { @@ -124,7 +139,7 @@ mod imp { ChainStep::FocusThenSetDynamic { attr: "AXValue" }, ChainStep::Custom { label: "select_all_delete", - func: select_all_then_delete, + func: chain_steps::select_all_then_delete, }, ], suggestion: "Try 'press cmd+a' then 'press delete'.", @@ -157,175 +172,12 @@ mod imp { ChainStep::Action("AXScrollToVisible"), ChainStep::Custom { label: "walk_parents_scroll", - func: walk_parents_and_scroll, + func: chain_steps::walk_parents_and_scroll, }, ], suggestion: "Element may not be in a scrollable container.", }; - fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::AXUIElementPostKeyboardEvent; - - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let pid = match crate::system::app_ops::pid_from_element(el) { - Some(p) => p, - None => return false, - }; - let app = crate::tree::element_for_pid(pid); - unsafe { - AXUIElementPostKeyboardEvent(app.0, 0, 55, true); - AXUIElementPostKeyboardEvent(app.0, 0, 0, true); - AXUIElementPostKeyboardEvent(app.0, 0, 0, false); - AXUIElementPostKeyboardEvent(app.0, 0, 55, false); - }; - std::thread::sleep(std::time::Duration::from_millis(30)); - unsafe { - AXUIElementPostKeyboardEvent(app.0, 0, 51, true); - AXUIElementPostKeyboardEvent(app.0, 0, 51, false); - }; - true - } - - fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::kAXRoleAttribute; - - let bounds = match crate::tree::read_bounds(el) { - Some(b) => b, - None => return false, - }; - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..8 { - let parent = match ¤t { - Some(p) => p, - None => return false, - }; - let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute); - if role.as_deref() == Some("AXScrollArea") { - let parent_bounds = match crate::tree::read_bounds(parent) { - Some(b) => b, - None => return false, - }; - let target_y = bounds.y + bounds.height / 2.0; - let visible_mid = parent_bounds.y + parent_bounds.height / 2.0; - if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height { - let dy = if target_y > visible_mid { -5 } else { 5 }; - let cx = parent_bounds.x + parent_bounds.width / 2.0; - let cy = parent_bounds.y + parent_bounds.height / 2.0; - for _ in 0..20 { - let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0); - std::thread::sleep(std::time::Duration::from_millis(16)); - } - } - return true; - } - current = crate::tree::copy_element_attr(parent, "AXParent"); - } - false - } - - fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool { - if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { - return false; - } - ax_helpers::try_ax_action(el, "AXShowAlternateUI"); - std::thread::sleep(std::time::Duration::from_millis(100)); - ax_helpers::try_each_child( - el, - |child| { - let ca = ax_helpers::list_ax_actions(child); - ax_helpers::try_action_from_list(child, &ca, &["AXPress"]) - }, - 5, - ) - } - - fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; - use core_foundation::{ - array::CFArray, - base::{CFRetain, CFType, CFTypeRef, TCFType}, - string::CFString, - }; - - let parent = match crate::tree::copy_element_attr(el, "AXParent") { - Some(p) => p, - None => return false, - }; - let role = match crate::tree::copy_string_attr(&parent, kAXRoleAttribute) { - Some(r) => r, - None => return false, - }; - if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") { - return false; - } - if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") { - return false; - } - unsafe { CFRetain(el.0 as CFTypeRef) }; - let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; - let arr = CFArray::from_CFTypes(&[el_cf]); - let cf_attr = CFString::new("AXSelectedRows"); - let err = unsafe { - AXUIElementSetAttributeValue( - parent.0, - cf_attr.as_concrete_TypeRef(), - arr.as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess - } - - fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool { - let custom = crate::tree::copy_ax_array(el, "AXCustomActions").unwrap_or_default(); - if custom.is_empty() { - return false; - } - ax_helpers::try_ax_action(el, "AXPerformCustomAction") - } - - fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::AXUIElementPostKeyboardEvent; - - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let pid = match crate::system::app_ops::pid_from_element(el) { - Some(p) => p, - None => return false, - }; - let app = crate::tree::element_for_pid(pid); - unsafe { - AXUIElementPostKeyboardEvent(app.0, 0, 49, true); - AXUIElementPostKeyboardEvent(app.0, 0, 49, false); - }; - true - } - - fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { - let pid = match crate::system::app_ops::pid_from_element(el) { - Some(p) => p, - None => return false, - }; - let _ = crate::system::app_ops::ensure_app_focused(pid); - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_helpers::try_ax_action(el, "AXShowMenu") - } - - fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { - if !ax_helpers::is_attr_settable(el, "AXSelected") { - return false; - } - if !ax_helpers::set_ax_bool(el, "AXSelected", true) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_helpers::try_ax_action(el, "AXShowMenu") - } - pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> { if ax_helpers::try_ax_action(el, "AXOpen") { return Ok(()); diff --git a/crates/macos/src/actions/chain_steps.rs b/crates/macos/src/actions/chain_steps.rs new file mode 100644 index 0000000..3f97afe --- /dev/null +++ b/crates/macos/src/actions/chain_steps.rs @@ -0,0 +1,267 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::actions::{ax_helpers, discovery::ElementCaps}; + use crate::tree::AXElement; + + pub fn do_verified_press(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::kAXRoleAttribute; + let parent = crate::tree::copy_element_attr(el, "AXParent"); + let in_container = parent.as_ref().is_some_and(|p| { + matches!( + crate::tree::copy_string_attr(p, kAXRoleAttribute).as_deref(), + Some("AXOutline" | "AXList" | "AXTable") + ) + }); + if !in_container { + return ax_helpers::try_ax_action_retried(el, "AXPress"); + } + let selected_before = crate::tree::element::copy_bool_attr(el, "AXSelected"); + if !ax_helpers::try_ax_action_retried(el, "AXPress") { + return false; + } + if selected_before == Some(true) { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let selected_after = crate::tree::element::copy_bool_attr(el, "AXSelected"); + if selected_after == Some(true) { + return true; + } + if crate::tree::copy_string_attr(el, kAXRoleAttribute).is_none() { + return true; + } + tracing::debug!("verified_press: AXPress ok but no state change in selection container"); + false + } + + pub fn try_focus_then_verified_confirm_or_press(el: &AXElement, caps: &ElementCaps) -> bool { + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + if ax_helpers::try_ax_action_retried(el, "AXConfirm") { + return true; + } + do_verified_press(el, caps) + } + + pub fn try_value_relay(el: &AXElement, _caps: &ElementCaps) -> bool { + if !ax_helpers::list_ax_actions(el).is_empty() { + return false; + } + let win = crate::tree::copy_element_attr(el, "AXWindow"); + let is_dialog = win.as_ref().is_some_and(|w| { + crate::tree::copy_string_attr(w, "AXSubrole").as_deref() == Some("AXDialog") + }); + if !is_dialog { + return false; + } + let label = std::cell::RefCell::new(None::); + ax_helpers::try_each_child( + el, + |child| { + let d = crate::tree::copy_string_attr(child, "AXDescription").unwrap_or_default(); + if d.is_empty() { + return false; + } + *label.borrow_mut() = Some(d.split(',').next().unwrap_or(&d).trim().to_owned()); + true + }, + 5, + ); + let Some(label) = label.into_inner() else { + return false; + }; + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return false; + }; + let app = crate::tree::element_for_pid(pid); + let Some(owner) = crate::tree::copy_element_attr(&app, "AXFocusedUIElement") else { + return false; + }; + if !ax_helpers::is_attr_settable(&owner, "AXValue") { + return false; + } + let orig = crate::tree::copy_string_attr(&owner, "AXValue"); + if ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label).is_err() { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(150)); + if !ax_helpers::try_ax_action(&owner, "AXConfirm") { + if let Some(o) = &orig { + let _ = ax_helpers::set_ax_string_or_err(&owner, "AXValue", o); + } + return false; + } + std::thread::sleep(std::time::Duration::from_millis(150)); + let final_val = crate::tree::copy_string_attr(&owner, "AXValue"); + if final_val.as_deref() != Some(label.as_str()) { + tracing::debug!("value_relay: reverted to {final_val:?}, expected {label:?}"); + } + final_val.as_deref() == Some(label.as_str()) + } + + pub fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey; + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return false; + }; + let a = crate::tree::element_for_pid(pid); + unsafe { + PostKey(a.0, 0, 55, true); + PostKey(a.0, 0, 0, true); + PostKey(a.0, 0, 0, false); + PostKey(a.0, 0, 55, false); + } + std::thread::sleep(std::time::Duration::from_millis(30)); + unsafe { + PostKey(a.0, 0, 51, true); + PostKey(a.0, 0, 51, false); + } + true + } + + pub fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::kAXRoleAttribute; + let Some(bounds) = crate::tree::read_bounds(el) else { + return false; + }; + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..8 { + let Some(parent) = ¤t else { return false }; + if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() + == Some("AXScrollArea") + { + let Some(pb) = crate::tree::read_bounds(parent) else { + return false; + }; + let ty = bounds.y + bounds.height / 2.0; + if ty < pb.y || ty > pb.y + pb.height { + let dy = if ty > pb.y + pb.height / 2.0 { -5 } else { 5 }; + let (cx, cy) = (pb.x + pb.width / 2.0, pb.y + pb.height / 2.0); + for _ in 0..20 { + let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0); + std::thread::sleep(std::time::Duration::from_millis(16)); + } + } + return true; + } + current = crate::tree::copy_element_attr(parent, "AXParent"); + } + false + } + + pub fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool { + if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { + return false; + } + ax_helpers::try_ax_action(el, "AXShowAlternateUI"); + std::thread::sleep(std::time::Duration::from_millis(100)); + ax_helpers::try_each_child( + el, + |child| { + let ca = ax_helpers::list_ax_actions(child); + ax_helpers::try_action_from_list(child, &ca, &["AXPress"]) + }, + 5, + ) + } + + pub fn try_parent_row_select(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::kAXRoleAttribute; + let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { + return false; + }; + let role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute).unwrap_or_default(); + if !matches!(role.as_str(), "AXRow" | "AXOutlineRow") { + return false; + } + if !ax_helpers::is_attr_settable(&parent, "AXSelected") { + return false; + } + ax_helpers::set_ax_bool(&parent, "AXSelected", true) + } + + pub fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { + return false; + }; + let Some(role) = crate::tree::copy_string_attr(&parent, kAXRoleAttribute) else { + return false; + }; + if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") { + return false; + } + if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") { + return false; + } + unsafe { CFRetain(el.0 as CFTypeRef) }; + let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; + let arr = CFArray::from_CFTypes(&[el_cf]); + let cf_attr = CFString::new("AXSelectedRows"); + let err = unsafe { + AXUIElementSetAttributeValue( + parent.0, + cf_attr.as_concrete_TypeRef(), + arr.as_CFTypeRef(), + ) + }; + err == kAXErrorSuccess + } + + pub fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool { + let has = !crate::tree::copy_ax_array(el, "AXCustomActions") + .unwrap_or_default() + .is_empty(); + has && ax_helpers::try_ax_action(el, "AXPerformCustomAction") + } + + pub fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey; + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return false; + }; + let a = crate::tree::element_for_pid(pid); + unsafe { + PostKey(a.0, 0, 49, true); + PostKey(a.0, 0, 49, false); + } + true + } + + pub fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return false; + }; + let _ = crate::system::app_ops::ensure_app_focused(pid); + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_helpers::try_ax_action(el, "AXShowMenu") + } + + pub fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { + if !ax_helpers::is_attr_settable(el, "AXSelected") + || !ax_helpers::set_ax_bool(el, "AXSelected", true) + { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + ax_helpers::try_ax_action(el, "AXShowMenu") + } +} + +#[cfg(target_os = "macos")] +pub(crate) use imp::*; diff --git a/crates/macos/src/actions/mod.rs b/crates/macos/src/actions/mod.rs index 3d9ba09..6fde1b3 100644 --- a/crates/macos/src/actions/mod.rs +++ b/crates/macos/src/actions/mod.rs @@ -1,6 +1,7 @@ pub mod ax_helpers; pub mod chain; pub mod chain_defs; +pub mod chain_steps; pub mod discovery; pub mod dispatch; pub mod extras; diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index fd57870..7e1f417 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -290,6 +290,14 @@ mod imp { return None; } + if !point.x.is_finite() + || !point.y.is_finite() + || !size.width.is_finite() + || !size.height.is_finite() + { + return None; + } + Some(Rect { x: point.x, y: point.y,