feat: add fallback chains for set-value, clear, focus, scroll-to, type and post-action state hints

Stage B of the centralized chain executor plan:
- set-value chain: direct AXSetValue → focus+AXSetValue
- clear chain: AXSetValue("") → focus+AXSetValue("") → Cmd+A+Delete
- focus chain: AXFocused → AXRaise → AXPress → AXSelected → CGClick
- scroll-to chain: AXScrollToVisible → walk-parents-scroll
- type: app-focus before typing, non-ASCII via clipboard paste (Cmd+V)
- post-action state hints: read element role/value/states after stateful
  actions (click, toggle, check, set-value, clear, type, expand, collapse)
- FocusThenSetDynamic chain step variant for focus-before-set patterns
- export copy_value_typed for reading numeric AXValues (checkboxes)
This commit is contained in:
Lahfir 2026-02-23 04:41:12 -08:00
parent 2a52d62106
commit 11f8da06e8
5 changed files with 235 additions and 24 deletions

View file

@ -15,6 +15,9 @@ pub enum ChainStep {
attr: &'static str,
},
FocusThenAction(&'static str),
FocusThenSetDynamic {
attr: &'static str,
},
FocusThenConfirmOrPress,
ChildActions {
actions: &'static [&'static str],
@ -93,6 +96,7 @@ mod imp {
ChainStep::SetBool { attr, .. } => attr,
ChainStep::SetDynamic { attr } => attr,
ChainStep::FocusThenAction(name) => name,
ChainStep::FocusThenSetDynamic { attr } => attr,
ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress",
ChainStep::ChildActions { .. } => "ChildActions",
ChainStep::AncestorActions { .. } => "AncestorActions",
@ -136,6 +140,18 @@ mod imp {
ax_helpers::try_ax_action_retried(el, name)
}
ChainStep::FocusThenSetDynamic { attr } => {
let value = match ctx.dynamic_value {
Some(v) => v,
None => return false,
};
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(Duration::from_millis(50));
ax_helpers::set_ax_string_or_err(el, attr, value).is_ok()
}
ChainStep::FocusThenConfirmOrPress => {
if !ax_helpers::ax_focus(el) {
return false;

View file

@ -108,6 +108,124 @@ mod imp {
suggestion: "Try 'click' to close it instead.",
};
pub static SET_VALUE_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetDynamic { attr: "AXValue" },
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
],
suggestion: "Try 'clear' then 'type', or check element is a text field.",
};
pub static CLEAR_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetDynamic { attr: "AXValue" },
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
ChainStep::Custom {
label: "select_all_delete",
func: select_all_then_delete,
},
],
suggestion: "Try 'press cmd+a' then 'press delete'.",
};
pub static FOCUS_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::SetBool {
attr: "AXFocused",
value: true,
},
ChainStep::Action("AXRaise"),
ChainStep::Action("AXPress"),
ChainStep::SetBool {
attr: "AXSelected",
value: true,
},
ChainStep::CGClick {
button: MouseButton::Left,
count: 1,
},
],
suggestion: "Try 'click' to focus the element instead.",
};
pub static SCROLL_TO_CHAIN: ChainDef = ChainDef {
pre_scroll: false,
steps: &[
ChainStep::Action("AXScrollToVisible"),
ChainStep::Custom {
label: "walk_parents_scroll",
func: walk_parents_and_scroll,
},
],
suggestion: "Element may not be in a scrollable container.",
};
fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::AXUIElementPostKeyboardEvent;
if !ax_helpers::ax_focus(el) {
return false;
}
std::thread::sleep(std::time::Duration::from_millis(50));
let pid = match crate::system::app_ops::pid_from_element(el) {
Some(p) => p,
None => return false,
};
let app = crate::tree::element_for_pid(pid);
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 55, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, true);
AXUIElementPostKeyboardEvent(app.0, 0, 0, false);
AXUIElementPostKeyboardEvent(app.0, 0, 55, false);
};
std::thread::sleep(std::time::Duration::from_millis(30));
unsafe {
AXUIElementPostKeyboardEvent(app.0, 0, 51, true);
AXUIElementPostKeyboardEvent(app.0, 0, 51, false);
};
true
}
fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
use accessibility_sys::kAXRoleAttribute;
let bounds = match crate::tree::read_bounds(el) {
Some(b) => b,
None => return false,
};
let mut current = crate::tree::copy_element_attr(el, "AXParent");
for _ in 0..8 {
let parent = match &current {
Some(p) => p,
None => return false,
};
let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute);
if role.as_deref() == Some("AXScrollArea") {
let parent_bounds = match crate::tree::read_bounds(parent) {
Some(b) => b,
None => return false,
};
let target_y = bounds.y + bounds.height / 2.0;
let visible_mid = parent_bounds.y + parent_bounds.height / 2.0;
if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height {
let dy = if target_y > visible_mid { -5 } else { 5 };
let cx = parent_bounds.x + parent_bounds.width / 2.0;
let cy = parent_bounds.y + parent_bounds.height / 2.0;
for _ in 0..20 {
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
std::thread::sleep(std::time::Duration::from_millis(16));
}
}
return true;
}
current = crate::tree::copy_element_attr(parent, "AXParent");
}
false
}
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
return false;
@ -238,5 +356,6 @@ mod imp {}
#[cfg(target_os = "macos")]
pub(crate) use imp::{
double_click, triple_click, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, RIGHT_CLICK_CHAIN,
double_click, triple_click, CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN,
FOCUS_CHAIN, RIGHT_CLICK_CHAIN, SCROLL_TO_CHAIN, SET_VALUE_CHAIN,
};

View file

@ -105,21 +105,23 @@ mod imp {
}
Action::SetValue(val) => {
ax_helpers::ax_set_value(el, val)?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: Some(val),
};
execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?;
}
Action::SetFocus => {
if !ax_helpers::ax_focus(el) {
return Err(
AdapterError::new(ErrorCode::ActionFailed, "SetFocus failed")
.with_suggestion("Element may not support focus. Try 'click' instead."),
);
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?;
}
Action::TypeText(text) => {
ax_helpers::ax_focus(el);
crate::input::keyboard::synthesize_text(text)?;
execute_type(el, text)?;
}
Action::PressKey(combo) => {
@ -164,17 +166,19 @@ mod imp {
}
Action::ScrollTo => {
if !ax_helpers::try_ax_action(el, "AXScrollToVisible") {
return Err(AdapterError::new(
ErrorCode::ActionFailed,
"AXScrollToVisible failed",
)
.with_suggestion("Element may not be inside a scrollable area"));
}
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: None,
};
execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?;
}
Action::Clear => {
ax_helpers::ax_set_value(el, "")?;
let caps = discovery::discover(el);
let ctx = ChainContext {
dynamic_value: Some(""),
};
execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?;
}
Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => {
@ -193,7 +197,76 @@ mod imp {
}
}
Ok(ActionResult::new(label))
let mut result = ActionResult::new(label);
if let Some(state) = read_post_state(el, action) {
result = result.with_state(state);
}
Ok(result)
}
fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> {
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
let _ = crate::system::app_ops::ensure_app_focused(pid);
}
ax_helpers::ax_focus(el);
std::thread::sleep(std::time::Duration::from_millis(50));
let has_non_ascii = !text.is_ascii();
if has_non_ascii {
type_via_clipboard_paste(el, text)
} else {
crate::input::keyboard::synthesize_text(text)
}
}
fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> {
let saved = crate::input::clipboard::get().ok();
crate::input::clipboard::set(text)?;
std::thread::sleep(std::time::Duration::from_millis(50));
crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo {
key: "v".into(),
modifiers: vec![agent_desktop_core::action::Modifier::Cmd],
})?;
std::thread::sleep(std::time::Duration::from_millis(100));
if let Some(prev) = saved {
let _ = crate::input::clipboard::set(&prev);
}
Ok(())
}
fn read_post_state(
el: &AXElement,
action: &Action,
) -> Option<agent_desktop_core::action::ElementState> {
let delay_ms = match action {
Action::Click
| Action::Toggle
| Action::Check
| Action::Uncheck
| Action::TypeText(_) => 50,
Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0,
_ => return None,
};
if delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
}
let value = crate::tree::copy_value_typed(el);
let role = ax_helpers::element_role(el).unwrap_or_default();
let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false);
let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true);
let mut states = Vec::new();
if focused {
states.push("focused".into());
}
if !enabled {
states.push("disabled".into());
}
Some(agent_desktop_core::action::ElementState {
role,
states,
value,
})
}
pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> {

View file

@ -161,7 +161,7 @@ mod imp {
cf_type.downcast::<CFString>().map(|s| s.to_string())
}
fn copy_value_typed(el: &AXElement) -> Option<String> {
pub fn copy_value_typed(el: &AXElement) -> Option<String> {
let cf_attr = CFString::new(kAXValueAttribute);
let mut val_ref: CFTypeRef = std::ptr::null_mut();
let err = unsafe {
@ -335,6 +335,9 @@ mod imp {
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
None
}
pub fn copy_value_typed(_el: &AXElement) -> Option<String> {
None
}
pub fn fetch_node_attrs(
_el: &AXElement,
) -> (
@ -350,6 +353,6 @@ mod imp {
}
pub use imp::{
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid,
fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed,
element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
};

View file

@ -6,8 +6,8 @@ pub mod surfaces;
pub use builder::{build_subtree, window_element_for};
pub use element::{
copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, read_bounds,
resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid,
read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
};
pub use resolve::{find_element_recursive, resolve_element_impl};
pub use roles::{ax_role_to_str, is_interactive_role};