mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 17:42:13 +00:00
feat: add fallback chains for set-value, clear, focus, scroll-to, type and post-action state hints
Stage B of the centralized chain executor plan:
- set-value chain: direct AXSetValue → focus+AXSetValue
- clear chain: AXSetValue("") → focus+AXSetValue("") → Cmd+A+Delete
- focus chain: AXFocused → AXRaise → AXPress → AXSelected → CGClick
- scroll-to chain: AXScrollToVisible → walk-parents-scroll
- type: app-focus before typing, non-ASCII via clipboard paste (Cmd+V)
- post-action state hints: read element role/value/states after stateful
actions (click, toggle, check, set-value, clear, type, expand, collapse)
- FocusThenSetDynamic chain step variant for focus-before-set patterns
- export copy_value_typed for reading numeric AXValues (checkboxes)
This commit is contained in:
parent
2a52d62106
commit
11f8da06e8
5 changed files with 235 additions and 24 deletions
|
|
@ -15,6 +15,9 @@ pub enum ChainStep {
|
|||
attr: &'static str,
|
||||
},
|
||||
FocusThenAction(&'static str),
|
||||
FocusThenSetDynamic {
|
||||
attr: &'static str,
|
||||
},
|
||||
FocusThenConfirmOrPress,
|
||||
ChildActions {
|
||||
actions: &'static [&'static str],
|
||||
|
|
@ -93,6 +96,7 @@ mod imp {
|
|||
ChainStep::SetBool { attr, .. } => attr,
|
||||
ChainStep::SetDynamic { attr } => attr,
|
||||
ChainStep::FocusThenAction(name) => name,
|
||||
ChainStep::FocusThenSetDynamic { attr } => attr,
|
||||
ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress",
|
||||
ChainStep::ChildActions { .. } => "ChildActions",
|
||||
ChainStep::AncestorActions { .. } => "AncestorActions",
|
||||
|
|
@ -136,6 +140,18 @@ mod imp {
|
|||
ax_helpers::try_ax_action_retried(el, name)
|
||||
}
|
||||
|
||||
ChainStep::FocusThenSetDynamic { attr } => {
|
||||
let value = match ctx.dynamic_value {
|
||||
Some(v) => v,
|
||||
None => return false,
|
||||
};
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
ax_helpers::set_ax_string_or_err(el, attr, value).is_ok()
|
||||
}
|
||||
|
||||
ChainStep::FocusThenConfirmOrPress => {
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -108,6 +108,124 @@ mod imp {
|
|||
suggestion: "Try 'click' to close it instead.",
|
||||
};
|
||||
|
||||
pub static SET_VALUE_CHAIN: ChainDef = ChainDef {
|
||||
pre_scroll: false,
|
||||
steps: &[
|
||||
ChainStep::SetDynamic { attr: "AXValue" },
|
||||
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
|
||||
],
|
||||
suggestion: "Try 'clear' then 'type', or check element is a text field.",
|
||||
};
|
||||
|
||||
pub static CLEAR_CHAIN: ChainDef = ChainDef {
|
||||
pre_scroll: false,
|
||||
steps: &[
|
||||
ChainStep::SetDynamic { attr: "AXValue" },
|
||||
ChainStep::FocusThenSetDynamic { attr: "AXValue" },
|
||||
ChainStep::Custom {
|
||||
label: "select_all_delete",
|
||||
func: select_all_then_delete,
|
||||
},
|
||||
],
|
||||
suggestion: "Try 'press cmd+a' then 'press delete'.",
|
||||
};
|
||||
|
||||
pub static FOCUS_CHAIN: ChainDef = ChainDef {
|
||||
pre_scroll: false,
|
||||
steps: &[
|
||||
ChainStep::SetBool {
|
||||
attr: "AXFocused",
|
||||
value: true,
|
||||
},
|
||||
ChainStep::Action("AXRaise"),
|
||||
ChainStep::Action("AXPress"),
|
||||
ChainStep::SetBool {
|
||||
attr: "AXSelected",
|
||||
value: true,
|
||||
},
|
||||
ChainStep::CGClick {
|
||||
button: MouseButton::Left,
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
suggestion: "Try 'click' to focus the element instead.",
|
||||
};
|
||||
|
||||
pub static SCROLL_TO_CHAIN: ChainDef = ChainDef {
|
||||
pre_scroll: false,
|
||||
steps: &[
|
||||
ChainStep::Action("AXScrollToVisible"),
|
||||
ChainStep::Custom {
|
||||
label: "walk_parents_scroll",
|
||||
func: walk_parents_and_scroll,
|
||||
},
|
||||
],
|
||||
suggestion: "Element may not be in a scrollable container.",
|
||||
};
|
||||
|
||||
fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::AXUIElementPostKeyboardEvent;
|
||||
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return false;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let pid = match crate::system::app_ops::pid_from_element(el) {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let app = crate::tree::element_for_pid(pid);
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 55, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 0, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 0, false);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 55, false);
|
||||
};
|
||||
std::thread::sleep(std::time::Duration::from_millis(30));
|
||||
unsafe {
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 51, true);
|
||||
AXUIElementPostKeyboardEvent(app.0, 0, 51, false);
|
||||
};
|
||||
true
|
||||
}
|
||||
|
||||
fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
use accessibility_sys::kAXRoleAttribute;
|
||||
|
||||
let bounds = match crate::tree::read_bounds(el) {
|
||||
Some(b) => b,
|
||||
None => return false,
|
||||
};
|
||||
let mut current = crate::tree::copy_element_attr(el, "AXParent");
|
||||
for _ in 0..8 {
|
||||
let parent = match ¤t {
|
||||
Some(p) => p,
|
||||
None => return false,
|
||||
};
|
||||
let role = crate::tree::copy_string_attr(parent, kAXRoleAttribute);
|
||||
if role.as_deref() == Some("AXScrollArea") {
|
||||
let parent_bounds = match crate::tree::read_bounds(parent) {
|
||||
Some(b) => b,
|
||||
None => return false,
|
||||
};
|
||||
let target_y = bounds.y + bounds.height / 2.0;
|
||||
let visible_mid = parent_bounds.y + parent_bounds.height / 2.0;
|
||||
if target_y < parent_bounds.y || target_y > parent_bounds.y + parent_bounds.height {
|
||||
let dy = if target_y > visible_mid { -5 } else { 5 };
|
||||
let cx = parent_bounds.x + parent_bounds.width / 2.0;
|
||||
let cy = parent_bounds.y + parent_bounds.height / 2.0;
|
||||
for _ in 0..20 {
|
||||
let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0);
|
||||
std::thread::sleep(std::time::Duration::from_millis(16));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
current = crate::tree::copy_element_attr(parent, "AXParent");
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool {
|
||||
if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") {
|
||||
return false;
|
||||
|
|
@ -238,5 +356,6 @@ mod imp {}
|
|||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) use imp::{
|
||||
double_click, triple_click, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, RIGHT_CLICK_CHAIN,
|
||||
double_click, triple_click, CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN,
|
||||
FOCUS_CHAIN, RIGHT_CLICK_CHAIN, SCROLL_TO_CHAIN, SET_VALUE_CHAIN,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -105,21 +105,23 @@ mod imp {
|
|||
}
|
||||
|
||||
Action::SetValue(val) => {
|
||||
ax_helpers::ax_set_value(el, val)?;
|
||||
let caps = discovery::discover(el);
|
||||
let ctx = ChainContext {
|
||||
dynamic_value: Some(val),
|
||||
};
|
||||
execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?;
|
||||
}
|
||||
|
||||
Action::SetFocus => {
|
||||
if !ax_helpers::ax_focus(el) {
|
||||
return Err(
|
||||
AdapterError::new(ErrorCode::ActionFailed, "SetFocus failed")
|
||||
.with_suggestion("Element may not support focus. Try 'click' instead."),
|
||||
);
|
||||
}
|
||||
let caps = discovery::discover(el);
|
||||
let ctx = ChainContext {
|
||||
dynamic_value: None,
|
||||
};
|
||||
execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?;
|
||||
}
|
||||
|
||||
Action::TypeText(text) => {
|
||||
ax_helpers::ax_focus(el);
|
||||
crate::input::keyboard::synthesize_text(text)?;
|
||||
execute_type(el, text)?;
|
||||
}
|
||||
|
||||
Action::PressKey(combo) => {
|
||||
|
|
@ -164,17 +166,19 @@ mod imp {
|
|||
}
|
||||
|
||||
Action::ScrollTo => {
|
||||
if !ax_helpers::try_ax_action(el, "AXScrollToVisible") {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::ActionFailed,
|
||||
"AXScrollToVisible failed",
|
||||
)
|
||||
.with_suggestion("Element may not be inside a scrollable area"));
|
||||
}
|
||||
let caps = discovery::discover(el);
|
||||
let ctx = ChainContext {
|
||||
dynamic_value: None,
|
||||
};
|
||||
execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?;
|
||||
}
|
||||
|
||||
Action::Clear => {
|
||||
ax_helpers::ax_set_value(el, "")?;
|
||||
let caps = discovery::discover(el);
|
||||
let ctx = ChainContext {
|
||||
dynamic_value: Some(""),
|
||||
};
|
||||
execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?;
|
||||
}
|
||||
|
||||
Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => {
|
||||
|
|
@ -193,7 +197,76 @@ mod imp {
|
|||
}
|
||||
}
|
||||
|
||||
Ok(ActionResult::new(label))
|
||||
let mut result = ActionResult::new(label);
|
||||
if let Some(state) = read_post_state(el, action) {
|
||||
result = result.with_state(state);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> {
|
||||
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {
|
||||
let _ = crate::system::app_ops::ensure_app_focused(pid);
|
||||
}
|
||||
ax_helpers::ax_focus(el);
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
let has_non_ascii = !text.is_ascii();
|
||||
if has_non_ascii {
|
||||
type_via_clipboard_paste(el, text)
|
||||
} else {
|
||||
crate::input::keyboard::synthesize_text(text)
|
||||
}
|
||||
}
|
||||
|
||||
fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> {
|
||||
let saved = crate::input::clipboard::get().ok();
|
||||
crate::input::clipboard::set(text)?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
|
||||
crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo {
|
||||
key: "v".into(),
|
||||
modifiers: vec![agent_desktop_core::action::Modifier::Cmd],
|
||||
})?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
|
||||
if let Some(prev) = saved {
|
||||
let _ = crate::input::clipboard::set(&prev);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_post_state(
|
||||
el: &AXElement,
|
||||
action: &Action,
|
||||
) -> Option<agent_desktop_core::action::ElementState> {
|
||||
let delay_ms = match action {
|
||||
Action::Click
|
||||
| Action::Toggle
|
||||
| Action::Check
|
||||
| Action::Uncheck
|
||||
| Action::TypeText(_) => 50,
|
||||
Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0,
|
||||
_ => return None,
|
||||
};
|
||||
if delay_ms > 0 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
|
||||
}
|
||||
let value = crate::tree::copy_value_typed(el);
|
||||
let role = ax_helpers::element_role(el).unwrap_or_default();
|
||||
let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false);
|
||||
let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true);
|
||||
let mut states = Vec::new();
|
||||
if focused {
|
||||
states.push("focused".into());
|
||||
}
|
||||
if !enabled {
|
||||
states.push("disabled".into());
|
||||
}
|
||||
Some(agent_desktop_core::action::ElementState {
|
||||
role,
|
||||
states,
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> {
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ mod imp {
|
|||
cf_type.downcast::<CFString>().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
fn copy_value_typed(el: &AXElement) -> Option<String> {
|
||||
pub fn copy_value_typed(el: &AXElement) -> Option<String> {
|
||||
let cf_attr = CFString::new(kAXValueAttribute);
|
||||
let mut val_ref: CFTypeRef = std::ptr::null_mut();
|
||||
let err = unsafe {
|
||||
|
|
@ -335,6 +335,9 @@ mod imp {
|
|||
pub fn resolve_element_name(_el: &AXElement) -> Option<String> {
|
||||
None
|
||||
}
|
||||
pub fn copy_value_typed(_el: &AXElement) -> Option<String> {
|
||||
None
|
||||
}
|
||||
pub fn fetch_node_attrs(
|
||||
_el: &AXElement,
|
||||
) -> (
|
||||
|
|
@ -350,6 +353,6 @@ mod imp {
|
|||
}
|
||||
|
||||
pub use imp::{
|
||||
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid,
|
||||
fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
|
||||
copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed,
|
||||
element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, AXElement,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ pub mod surfaces;
|
|||
|
||||
pub use builder::{build_subtree, window_element_for};
|
||||
pub use element::{
|
||||
copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, read_bounds,
|
||||
resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
|
||||
copy_ax_array, copy_element_attr, copy_string_attr, copy_value_typed, element_for_pid,
|
||||
read_bounds, resolve_element_name, AXElement, ABSOLUTE_MAX_DEPTH,
|
||||
};
|
||||
pub use resolve::{find_element_recursive, resolve_element_impl};
|
||||
pub use roles::{ax_role_to_str, is_interactive_role};
|
||||
|
|
|
|||
Loading…
Reference in a new issue