From 11f8da06e84ed67b0e26dbc1946f7a7542e89dcd Mon Sep 17 00:00:00 2001 From: Lahfir Date: Mon, 23 Feb 2026 04:41:12 -0800 Subject: [PATCH] feat: add fallback chains for set-value, clear, focus, scroll-to, type and post-action state hints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage B of the centralized chain executor plan: - set-value chain: direct AXSetValue → focus+AXSetValue - clear chain: AXSetValue("") → focus+AXSetValue("") → Cmd+A+Delete - focus chain: AXFocused → AXRaise → AXPress → AXSelected → CGClick - scroll-to chain: AXScrollToVisible → walk-parents-scroll - type: app-focus before typing, non-ASCII via clipboard paste (Cmd+V) - post-action state hints: read element role/value/states after stateful actions (click, toggle, check, set-value, clear, type, expand, collapse) - FocusThenSetDynamic chain step variant for focus-before-set patterns - export copy_value_typed for reading numeric AXValues (checkboxes) --- crates/macos/src/actions/chain.rs | 16 ++++ crates/macos/src/actions/chain_defs.rs | 121 ++++++++++++++++++++++++- crates/macos/src/actions/dispatch.rs | 109 ++++++++++++++++++---- crates/macos/src/tree/element.rs | 9 +- crates/macos/src/tree/mod.rs | 4 +- 5 files changed, 235 insertions(+), 24 deletions(-) diff --git a/crates/macos/src/actions/chain.rs b/crates/macos/src/actions/chain.rs index baa8d5d..23e040e 100644 --- a/crates/macos/src/actions/chain.rs +++ b/crates/macos/src/actions/chain.rs @@ -15,6 +15,9 @@ pub enum ChainStep { attr: &'static str, }, FocusThenAction(&'static str), + FocusThenSetDynamic { + attr: &'static str, + }, FocusThenConfirmOrPress, ChildActions { actions: &'static [&'static str], @@ -93,6 +96,7 @@ mod imp { ChainStep::SetBool { attr, .. } => attr, ChainStep::SetDynamic { attr } => attr, ChainStep::FocusThenAction(name) => name, + ChainStep::FocusThenSetDynamic { attr } => attr, ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress", ChainStep::ChildActions { .. } => "ChildActions", ChainStep::AncestorActions { .. } => "AncestorActions", @@ -136,6 +140,18 @@ mod imp { ax_helpers::try_ax_action_retried(el, name) } + ChainStep::FocusThenSetDynamic { attr } => { + let value = match ctx.dynamic_value { + Some(v) => v, + None => return false, + }; + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(Duration::from_millis(50)); + ax_helpers::set_ax_string_or_err(el, attr, value).is_ok() + } + ChainStep::FocusThenConfirmOrPress => { if !ax_helpers::ax_focus(el) { return false; diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs index 9ad6f4a..9c73b3c 100644 --- a/crates/macos/src/actions/chain_defs.rs +++ b/crates/macos/src/actions/chain_defs.rs @@ -108,6 +108,124 @@ mod imp { suggestion: "Try 'click' to close it instead.", }; + pub static SET_VALUE_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::FocusThenSetDynamic { attr: "AXValue" }, + ], + suggestion: "Try 'clear' then 'type', or check element is a text field.", + }; + + pub static CLEAR_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::FocusThenSetDynamic { attr: "AXValue" }, + ChainStep::Custom { + label: "select_all_delete", + func: select_all_then_delete, + }, + ], + suggestion: "Try 'press cmd+a' then 'press delete'.", + }; + + pub static FOCUS_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::SetBool { + attr: "AXFocused", + value: true, + }, + ChainStep::Action("AXRaise"), + ChainStep::Action("AXPress"), + ChainStep::SetBool { + attr: "AXSelected", + value: true, + }, + ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }, + ], + suggestion: "Try 'click' to focus the element instead.", + }; + + pub static SCROLL_TO_CHAIN: ChainDef = ChainDef { + pre_scroll: false, + steps: &[ + ChainStep::Action("AXScrollToVisible"), + ChainStep::Custom { + label: "walk_parents_scroll", + func: walk_parents_and_scroll, + }, + ], + suggestion: "Element may not be in a scrollable container.", + }; + + fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::AXUIElementPostKeyboardEvent; + + if !ax_helpers::ax_focus(el) { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + let pid = match crate::system::app_ops::pid_from_element(el) { + Some(p) => p, + None => return false, + }; + let app = crate::tree::element_for_pid(pid); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 55, true); + AXUIElementPostKeyboardEvent(app.0, 0, 0, true); + AXUIElementPostKeyboardEvent(app.0, 0, 0, false); + AXUIElementPostKeyboardEvent(app.0, 0, 55, false); + }; + std::thread::sleep(std::time::Duration::from_millis(30)); + unsafe { + AXUIElementPostKeyboardEvent(app.0, 0, 51, true); + AXUIElementPostKeyboardEvent(app.0, 0, 51, false); + }; + true + } + + fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool { + use accessibility_sys::kAXRoleAttribute; + + let bounds = match crate::tree::read_bounds(el) { + Some(b) => b, + None => return false, + }; + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..8 { + let parent = match ¤t { + Some(p) => p, + None => return false, + }; + let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute); + if role.as_deref() == Some("AXScrollArea") { + let parent_bounds = match crate::tree::read_bounds(parent) { + Some(b) => b, + None => return false, + }; + let target_y = bounds.y + bounds.height / 2.0; + let visible_mid = parent_bounds.y + parent_bounds.height / 2.0; + if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height { + let dy = if target_y > visible_mid { -5 } else { 5 }; + let cx = parent_bounds.x + parent_bounds.width / 2.0; + let cy = parent_bounds.y + parent_bounds.height / 2.0; + for _ in 0..20 { + let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0); + std::thread::sleep(std::time::Duration::from_millis(16)); + } + } + return true; + } + current = crate::tree::copy_element_attr(parent, "AXParent"); + } + false + } + fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool { if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { return false; @@ -238,5 +356,6 @@ mod imp {} #[cfg(target_os = "macos")] pub(crate) use imp::{ - double_click, triple_click, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, RIGHT_CLICK_CHAIN, + double_click, triple_click, CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, + FOCUS_CHAIN, RIGHT_CLICK_CHAIN, SCROLL_TO_CHAIN, SET_VALUE_CHAIN, }; diff --git a/crates/macos/src/actions/dispatch.rs b/crates/macos/src/actions/dispatch.rs index afa966f..ae39c6d 100644 --- a/crates/macos/src/actions/dispatch.rs +++ b/crates/macos/src/actions/dispatch.rs @@ -105,21 +105,23 @@ mod imp { } Action::SetValue(val) => { - ax_helpers::ax_set_value(el, val)?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: Some(val), + }; + execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?; } Action::SetFocus => { - if !ax_helpers::ax_focus(el) { - return Err( - AdapterError::new(ErrorCode::ActionFailed, "SetFocus failed") - .with_suggestion("Element may not support focus. Try 'click' instead."), - ); - } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?; } Action::TypeText(text) => { - ax_helpers::ax_focus(el); - crate::input::keyboard::synthesize_text(text)?; + execute_type(el, text)?; } Action::PressKey(combo) => { @@ -164,17 +166,19 @@ mod imp { } Action::ScrollTo => { - if !ax_helpers::try_ax_action(el, "AXScrollToVisible") { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - "AXScrollToVisible failed", - ) - .with_suggestion("Element may not be inside a scrollable area")); - } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + }; + execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?; } Action::Clear => { - ax_helpers::ax_set_value(el, "")?; + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: Some(""), + }; + execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?; } Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => { @@ -193,7 +197,76 @@ mod imp { } } - Ok(ActionResult::new(label)) + let mut result = ActionResult::new(label); + if let Some(state) = read_post_state(el, action) { + result = result.with_state(state); + } + Ok(result) + } + + fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + ax_helpers::ax_focus(el); + std::thread::sleep(std::time::Duration::from_millis(50)); + let has_non_ascii = !text.is_ascii(); + if has_non_ascii { + type_via_clipboard_paste(el, text) + } else { + crate::input::keyboard::synthesize_text(text) + } + } + + fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> { + let saved = crate::input::clipboard::get().ok(); + crate::input::clipboard::set(text)?; + std::thread::sleep(std::time::Duration::from_millis(50)); + + crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo { + key: "v".into(), + modifiers: vec![agent_desktop_core::action::Modifier::Cmd], + })?; + std::thread::sleep(std::time::Duration::from_millis(100)); + + if let Some(prev) = saved { + let _ = crate::input::clipboard::set(&prev); + } + Ok(()) + } + + fn read_post_state( + el: &AXElement, + action: &Action, + ) -> Option { + let delay_ms = match action { + Action::Click + | Action::Toggle + | Action::Check + | Action::Uncheck + | Action::TypeText(_) => 50, + Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0, + _ => return None, + }; + if delay_ms > 0 { + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + } + let value = crate::tree::copy_value_typed(el); + let role = ax_helpers::element_role(el).unwrap_or_default(); + let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false); + let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true); + let mut states = Vec::new(); + if focused { + states.push("focused".into()); + } + if !enabled { + states.push("disabled".into()); + } + Some(agent_desktop_core::action::ElementState { + role, + states, + value, + }) } pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> { diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index 93a607c..fd57870 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -161,7 +161,7 @@ mod imp { cf_type.downcast::().map(|s| s.to_string()) } - fn copy_value_typed(el: &AXElement) -> Option { + pub fn copy_value_typed(el: &AXElement) -> Option { let cf_attr = CFString::new(kAXValueAttribute); let mut val_ref: CFTypeRef = std::ptr::null_mut(); let err = unsafe { @@ -335,6 +335,9 @@ mod imp { pub fn resolve_element_name(_el: &AXElement) -> Option { None } + pub fn copy_value_typed(_el: &AXElement) -> Option { + None + } pub fn fetch_node_attrs( _el: &AXElement, ) -> ( @@ -350,6 +353,6 @@ mod imp { } pub use imp::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid, - fetch_node_attrs, read_bounds, resolve_element_name, AXElement, + copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed, + element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement, }; diff --git a/crates/macos/src/tree/mod.rs b/crates/macos/src/tree/mod.rs index c1daf45..c26c51c 100644 --- a/crates/macos/src/tree/mod.rs +++ b/crates/macos/src/tree/mod.rs @@ -6,8 +6,8 @@ pub mod surfaces; pub use builder::{build_subtree, window_element_for}; pub use element::{ - copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, read_bounds, - resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH, + copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid, + read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH, }; pub use resolve::{find_element_recursive, resolve_element_impl}; pub use roles::{ax_role_to_str, is_interactive_role};