diff --git a/crates/macos/src/actions/post_state.rs b/crates/macos/src/actions/post_state.rs index 4ef3781..00e01a1 100644 --- a/crates/macos/src/actions/post_state.rs +++ b/crates/macos/src/actions/post_state.rs @@ -1,31 +1,31 @@ -use agent_desktop_core::{action::Action, adapter::LiveElement, element_state::ElementState}; +use agent_desktop_core::{adapter::LiveElement, element_state::ElementState}; -#[cfg(target_os = "macos")] pub(crate) fn read_post_state( el: &crate::tree::AXElement, - action: &Action, + action: &agent_desktop_core::action::Action, ) -> Option { let delay_ms = match action { - Action::Click | Action::TypeText(_) => 50, - Action::Toggle - | Action::Check - | Action::Uncheck - | Action::SetValue(_) - | Action::Clear - | Action::Expand - | Action::Collapse => 0, - Action::DoubleClick - | Action::RightClick - | Action::TripleClick - | Action::SetFocus - | Action::Select(_) - | Action::Scroll(_, _) - | Action::ScrollTo - | Action::PressKey(_) - | Action::KeyDown(_) - | Action::KeyUp(_) - | Action::Hover - | Action::Drag(_) => return None, + agent_desktop_core::action::Action::Click + | agent_desktop_core::action::Action::TypeText(_) => 50, + agent_desktop_core::action::Action::Toggle + | agent_desktop_core::action::Action::Check + | agent_desktop_core::action::Action::Uncheck + | agent_desktop_core::action::Action::SetValue(_) + | agent_desktop_core::action::Action::Clear + | agent_desktop_core::action::Action::Expand + | agent_desktop_core::action::Action::Collapse => 0, + agent_desktop_core::action::Action::DoubleClick + | agent_desktop_core::action::Action::RightClick + | agent_desktop_core::action::Action::TripleClick + | agent_desktop_core::action::Action::SetFocus + | agent_desktop_core::action::Action::Select(_) + | agent_desktop_core::action::Action::Scroll(_, _) + | agent_desktop_core::action::Action::ScrollTo + | agent_desktop_core::action::Action::PressKey(_) + | agent_desktop_core::action::Action::KeyDown(_) + | agent_desktop_core::action::Action::KeyUp(_) + | agent_desktop_core::action::Action::Hover + | agent_desktop_core::action::Action::Drag(_) => return None, }; if delay_ms > 0 { std::thread::sleep(std::time::Duration::from_millis(delay_ms)); @@ -36,7 +36,7 @@ pub(crate) fn read_post_state( pub(crate) fn read_element_state(el: &crate::tree::AXElement) -> ElementState { let attrs = crate::tree::element::fetch_node_attrs(el); let role = normalized_role(attrs.role.as_deref()); - element_state_from_attrs(attrs, role) + element_state_from_attrs(el, attrs, role) } pub(crate) fn read_live_element(el: &crate::tree::AXElement) -> LiveElement { @@ -44,7 +44,7 @@ pub(crate) fn read_live_element(el: &crate::tree::AXElement) -> LiveElement { let role = normalized_role(attrs.role.as_deref()); let bounds = attrs.bounds; let has_scrollbars = attrs.has_scrollbars; - let state = element_state_from_attrs(attrs, role.clone()); + let state = element_state_from_attrs(el, attrs, role.clone()); LiveElement { state: Some(state), bounds, @@ -62,31 +62,22 @@ pub(crate) fn read_live_actions(el: &crate::tree::AXElement) -> Vec { crate::tree::action_list::platform_available_actions(el, &role, attrs.has_scrollbars) } -fn element_state_from_attrs(attrs: crate::tree::NodeAttrs, role: String) -> ElementState { - let value = attrs.value; - let focused = attrs.states.focused.unwrap_or(false); - let expanded = attrs - .states - .expanded - .or(attrs.states.disclosing) - .unwrap_or(false); - let mut states = Vec::new(); - if focused { - states.push("focused".into()); - } - if !attrs.states.enabled { - states.push("disabled".into()); - } - if expanded { - states.push("expanded".into()); - } - if crate::tree::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { - states.push("checked".into()); - } +fn element_state_from_attrs( + el: &crate::tree::AXElement, + attrs: crate::tree::NodeAttrs, + role: String, +) -> ElementState { + let is_secure = attrs.role.as_deref() == Some("AXSecureTextField"); + let ctx = crate::tree::state_reader::StateReaderContext { + focused: None, + window_bounds: None, + is_secure_text: is_secure, + }; + let states = crate::tree::state_reader::states_from_element(el, &attrs, &role, &ctx); ElementState { role, states, - value, + value: attrs.value, } } @@ -96,7 +87,3 @@ fn normalized_role(ax_role: Option<&str>) -> String { .unwrap_or("unknown") .to_string() } - -fn value_is_checked(value: Option<&str>) -> bool { - matches!(value, Some("1" | "true")) -} diff --git a/crates/macos/src/tree/build_context.rs b/crates/macos/src/tree/build_context.rs index 7640260..58cf56a 100644 --- a/crates/macos/src/tree/build_context.rs +++ b/crates/macos/src/tree/build_context.rs @@ -1,7 +1,10 @@ +use agent_desktop_core::node::Rect; + use super::AXElement; pub struct TreeBuildContext { pub(crate) focused: Option, + pub(crate) window_bounds: Option, include_bounds: bool, } @@ -10,6 +13,7 @@ impl TreeBuildContext { let app = super::element_for_pid(pid); Self { focused: super::copy_element_attr(&app, "AXFocusedUIElement"), + window_bounds: None, include_bounds, } } @@ -17,10 +21,19 @@ impl TreeBuildContext { pub fn empty(include_bounds: bool) -> Self { Self { focused: None, + window_bounds: None, include_bounds, } } + pub(crate) fn child_context(&self, window_bounds: Option) -> Self { + Self { + focused: self.focused.clone(), + window_bounds: window_bounds.or(self.window_bounds), + include_bounds: self.include_bounds, + } + } + pub(crate) fn bounds_for( &self, bounds: Option, diff --git a/crates/macos/src/tree/builder.rs b/crates/macos/src/tree/builder.rs index 90f1733..9de3f04 100644 --- a/crates/macos/src/tree/builder.rs +++ b/crates/macos/src/tree/builder.rs @@ -4,9 +4,8 @@ use rustc_hash::FxHashSet; use super::AXElement; use super::action_list::platform_available_actions; -use super::attributes::{copy_ax_array, copy_ax_array_prefix, copy_bool_attr, copy_string_attr}; +use super::attributes::{copy_ax_array, copy_ax_array_prefix, copy_string_attr}; use super::build_context::TreeBuildContext; -use super::capabilities::same_element; use super::element::{ ABSOLUTE_MAX_DEPTH, child_attributes, count_children, element_for_pid, fetch_node_attrs, }; @@ -128,7 +127,7 @@ pub fn build_subtree( let (role, promoted_label) = crate::tree::roles::normalized_role_and_label(el, attrs.role.as_deref()); let is_secure_text = is_secure_text_role(attrs.role.as_deref()); - let value = redact_secure_value(attrs.role.as_deref(), attrs.value); + let value = redact_secure_value(attrs.role.as_deref(), attrs.value.clone()); let is_promoted_item = promoted_label.is_some(); let available_actions = if is_promoted_item { vec![capability::CLICK.into(), capability::RIGHT_CLICK.into()] @@ -138,7 +137,7 @@ pub fn build_subtree( let name = promoted_label.or_else(|| attrs.title.clone().or_else(|| attrs.description.clone())); let description = if attrs.title.is_some() { - attrs.description + attrs.description.clone() } else { None }; @@ -149,31 +148,12 @@ pub fn build_subtree( name }; - let mut states = Vec::new(); - if context - .focused - .as_ref() - .is_some_and(|focused| same_element(el, focused)) - { - states.push("focused".into()); - } - if !attrs.states.enabled { - states.push("disabled".into()); - } - if is_secure_text { - states.push("secure".into()); - } - if attrs - .states - .expanded - .or(attrs.states.disclosing) - .unwrap_or_else(|| element_is_expanded(el)) - { - states.push("expanded".into()); - } - if super::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { - states.push("checked".into()); - } + let state_ctx = super::state_reader::StateReaderContext { + focused: context.focused.as_ref(), + window_bounds: context.window_bounds, + is_secure_text, + }; + let states = super::state_reader::states_from_element(el, &attrs, &role, &state_ctx); let bounds = context.bounds_for(attrs.bounds); @@ -216,6 +196,13 @@ pub fn build_subtree( let children_raw = copy_children(el, attrs.role.as_deref()).unwrap_or_default(); let name = name.or_else(|| label_from_children(&children_raw)); + let child_window_bounds = if attrs.role.as_deref() == Some("AXWindow") { + attrs.bounds.or(context.window_bounds) + } else { + context.window_bounds + }; + let child_context = context.child_context(child_window_bounds); + let children = if is_promoted_item { Vec::new() } else { @@ -229,7 +216,7 @@ pub fn build_subtree( max_depth, ancestors, skeleton, - context, + &child_context, ) }) .collect() @@ -264,16 +251,6 @@ fn redact_secure_value(ax_role: Option<&str>, value: Option) -> Option bool { - copy_bool_attr(el, "AXExpanded") - .or_else(|| copy_bool_attr(el, "AXDisclosing")) - .unwrap_or(false) -} - -fn value_is_checked(value: Option<&str>) -> bool { - matches!(value, Some("1" | "true")) -} - pub fn label_from_children(children: &[AXElement]) -> Option { #[cfg(target_os = "macos")] { diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index c6e3ec1..4f8a6f5 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -63,6 +63,11 @@ mod imp { "AXFocused", "AXExpanded", "AXDisclosing", + "AXSelected", + "AXHidden", + "AXElementBusy", + "AXModal", + "AXRequired", kAXPositionAttribute, kAXSizeAttribute, SCROLLBAR_ATTRS[0], @@ -90,16 +95,16 @@ mod imp { return fetch_node_attrs_slow(el); }; - let mut texts: [Option; 8] = Default::default(); + let mut texts: [Option; 13] = Default::default(); let mut position: Option = None; let mut size: Option = None; let mut has_scrollbars = false; for (idx, item) in arr.into_iter().enumerate() { match idx { - 0..=7 => texts[idx] = decode_text_attr(idx, &item), - 8 => position = decode_ax_point(&item), - 9 => size = decode_ax_size(&item), - 10 | 11 => { + 0..=12 => texts[idx] = decode_text_attr(idx, &item), + 13 => position = decode_ax_point(&item), + 14 => size = decode_ax_size(&item), + 15 | 16 => { has_scrollbars = has_scrollbars || ax_value::retained_ax_element(&item).is_some(); } @@ -108,8 +113,11 @@ mod imp { } let get = |i: usize| texts.get(i).and_then(|v| v.clone()); + let role = get(0); + let readonly = editable_ax_role(role.as_deref()) + .then(|| !crate::tree::capabilities::is_attr_settable(el, kAXValueAttribute)); NodeAttrs { - role: get(0), + role, title: get(1), description: get(2), value: get(3), @@ -118,6 +126,12 @@ mod imp { focused: parse_bool_attr(get(5)), expanded: parse_bool_attr(get(6)), disclosing: parse_bool_attr(get(7)), + selected: parse_bool_attr(get(8)), + hidden: parse_bool_attr(get(9)), + busy: parse_bool_attr(get(10)), + modal: parse_bool_attr(get(11)), + required: parse_bool_attr(get(12)), + readonly, }, bounds: position.zip(size).and_then(|(p, s)| rect_from_parts(p, s)), has_scrollbars, @@ -143,7 +157,7 @@ mod imp { } None } - 4..=7 => item + 4..=12 => item .downcast::() .map(|b| bool::from(b).to_string()), _ => None, @@ -180,6 +194,8 @@ mod imp { let desc = copy_string_attr(el, kAXDescriptionAttribute); let val = copy_value_typed(el); let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true); + let readonly = editable_ax_role(role.as_deref()) + .then(|| !crate::tree::capabilities::is_attr_settable(el, kAXValueAttribute)); NodeAttrs { role, title, @@ -190,12 +206,35 @@ mod imp { focused: copy_bool_attr(el, "AXFocused"), expanded: copy_bool_attr(el, "AXExpanded"), disclosing: copy_bool_attr(el, "AXDisclosing"), + selected: copy_bool_attr(el, "AXSelected"), + hidden: copy_bool_attr(el, "AXHidden"), + busy: copy_bool_attr(el, "AXElementBusy"), + modal: copy_bool_attr(el, "AXModal"), + required: copy_bool_attr(el, "AXRequired"), + readonly, }, bounds: read_bounds(el), has_scrollbars: copy_first_element_attr(el, &SCROLLBAR_ATTRS).is_some(), } } + fn editable_ax_role(role: Option<&str>) -> bool { + matches!( + role, + Some( + "AXTextField" + | "AXTextArea" + | "AXSearchField" + | "AXComboBox" + | "AXPopUpButton" + | "AXIncrementor" + | "AXStepper" + | "AXSlider" + | "AXValueIndicator" + ) + ) + } + pub fn resolve_element_name(el: &AXElement) -> Option { let ax_role = copy_string_attr(el, kAXRoleAttribute); let title = copy_string_attr(el, kAXTitleAttribute); diff --git a/crates/macos/src/tree/mod.rs b/crates/macos/src/tree/mod.rs index a7cbce9..b0208c7 100644 --- a/crates/macos/src/tree/mod.rs +++ b/crates/macos/src/tree/mod.rs @@ -17,6 +17,7 @@ mod resolve_identity; mod resolve_roots; mod resolve_search; pub mod roles; +pub mod state_reader; pub mod surfaces; pub(crate) use attributes::{ diff --git a/crates/macos/src/tree/node_attrs.rs b/crates/macos/src/tree/node_attrs.rs index 92f9440..4064628 100644 --- a/crates/macos/src/tree/node_attrs.rs +++ b/crates/macos/src/tree/node_attrs.rs @@ -17,6 +17,12 @@ pub(crate) struct NodeAttrStates { pub(crate) focused: Option, pub(crate) expanded: Option, pub(crate) disclosing: Option, + pub(crate) selected: Option, + pub(crate) hidden: Option, + pub(crate) busy: Option, + pub(crate) modal: Option, + pub(crate) required: Option, + pub(crate) readonly: Option, } impl Default for NodeAttrStates { @@ -26,6 +32,12 @@ impl Default for NodeAttrStates { focused: None, expanded: None, disclosing: None, + selected: None, + hidden: None, + busy: None, + modal: None, + required: None, + readonly: None, } } } diff --git a/crates/macos/src/tree/state_reader.rs b/crates/macos/src/tree/state_reader.rs new file mode 100644 index 0000000..39352fa --- /dev/null +++ b/crates/macos/src/tree/state_reader.rs @@ -0,0 +1,199 @@ +use agent_desktop_core::node::Rect; +use agent_desktop_core::state; + +use super::attributes::copy_bool_attr; +use super::{AXElement, NodeAttrs}; + +pub(crate) struct StateReaderContext<'a> { + pub focused: Option<&'a AXElement>, + pub window_bounds: Option, + pub is_secure_text: bool, +} + +pub(crate) fn states_from_element( + el: &AXElement, + attrs: &NodeAttrs, + role: &str, + ctx: &StateReaderContext<'_>, +) -> Vec { + let mut states = Vec::new(); + if ctx + .focused + .is_some_and(|focused| super::capabilities::same_element(el, focused)) + || attrs.states.focused == Some(true) + { + states.push(state::FOCUSED.into()); + } + if !attrs.states.enabled { + states.push(state::DISABLED.into()); + } + if ctx.is_secure_text { + states.push(state::SECURE.into()); + } + if is_expanded(el, attrs) { + states.push(state::EXPANDED.into()); + } + if super::roles::is_toggleable_role(role) { + if value_is_checked(attrs.value.as_deref()) { + states.push(state::CHECKED.into()); + } else if value_is_indeterminate(attrs.value.as_deref()) { + states.push(state::INDETERMINATE.into()); + } + } + if attrs.states.selected == Some(true) { + states.push(state::SELECTED.into()); + } + if attrs.states.hidden == Some(true) { + states.push(state::HIDDEN.into()); + } + if attrs.states.busy == Some(true) { + states.push(state::BUSY.into()); + } + if attrs.states.modal == Some(true) { + states.push(state::MODAL.into()); + } + if attrs.states.required == Some(true) { + states.push(state::REQUIRED.into()); + } + if role == "button" && value_is_checked(attrs.value.as_deref()) { + states.push(state::PRESSED.into()); + } + if attrs.states.readonly == Some(true) { + states.push(state::READONLY.into()); + } + if is_offscreen(attrs.bounds, ctx.window_bounds) { + states.push(state::OFFSCREEN.into()); + } + states +} + +fn is_expanded(el: &AXElement, attrs: &NodeAttrs) -> bool { + if attrs + .states + .expanded + .or(attrs.states.disclosing) + .unwrap_or(false) + { + return true; + } + if attrs.states.expanded.is_some() || attrs.states.disclosing.is_some() { + return false; + } + copy_bool_attr(el, "AXExpanded") + .or_else(|| copy_bool_attr(el, "AXDisclosing")) + .unwrap_or(false) +} + +fn value_is_checked(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true")) +} + +fn value_is_indeterminate(value: Option<&str>) -> bool { + matches!(value, Some("2" | "mixed")) +} + +fn is_offscreen(bounds: Option, window_bounds: Option) -> bool { + let (Some(el), Some(win)) = (bounds, window_bounds) else { + return false; + }; + let el_right = el.x + el.width; + let el_bottom = el.y + el.height; + let win_right = win.x + win.width; + let win_bottom = win.y + win.height; + el_right <= win.x || el.x >= win_right || el_bottom <= win.y || el.y >= win_bottom +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::node_attrs::{NodeAttrStates, NodeAttrs}; + use agent_desktop_core::node::Rect; + + fn sample_attrs() -> NodeAttrs { + NodeAttrs { + role: Some("AXCheckBox".into()), + title: None, + description: None, + value: Some("2".into()), + states: NodeAttrStates { + enabled: true, + focused: None, + expanded: None, + disclosing: None, + selected: None, + hidden: None, + busy: None, + modal: None, + required: None, + readonly: None, + }, + bounds: Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }), + has_scrollbars: false, + } + } + + #[test] + fn hidden_and_offscreen_tokens_are_vocabulary_members() { + for token in [state::HIDDEN, state::OFFSCREEN, state::INDETERMINATE] { + state::assert_states_in_vocabulary(&[token.to_string()]); + } + } + + #[test] + fn mixed_checkbox_emits_indeterminate_not_checked() { + let attrs = sample_attrs(); + let ctx = StateReaderContext { + focused: None, + window_bounds: None, + is_secure_text: false, + }; + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "checkbox", &ctx); + assert!(states.contains(&state::INDETERMINATE.to_string())); + assert!(!states.contains(&state::CHECKED.to_string())); + } + + #[test] + fn hidden_attr_emits_hidden_token() { + let mut attrs = sample_attrs(); + attrs.states.hidden = Some(true); + let ctx = StateReaderContext { + focused: None, + window_bounds: None, + is_secure_text: false, + }; + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "button", &ctx); + assert!(states.contains(&state::HIDDEN.to_string())); + } + + #[test] + fn clipped_bounds_emit_offscreen() { + let mut attrs = sample_attrs(); + attrs.bounds = Some(Rect { + x: 100.0, + y: 0.0, + width: 10.0, + height: 10.0, + }); + let window = Rect { + x: 0.0, + y: 0.0, + width: 50.0, + height: 50.0, + }; + let ctx = StateReaderContext { + focused: None, + window_bounds: Some(window), + is_secure_text: false, + }; + let el = AXElement(std::ptr::null_mut()); + let states = states_from_element(&el, &attrs, "button", &ctx); + assert!(states.contains(&state::OFFSCREEN.to_string())); + } +}