Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-08-18 04:55:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: unify macOS state production onto canonical vocabulary

Browse source 
Add a shared state_reader for tree and live reads, expand AX batch
attributes, and emit hidden/offscreen/indeterminate tokens from evidence.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Lahfir 2026-07-02 21:01:18 -07:00
parent 3bed49dafb
commit e76a9cbc41
7 changed files with 326 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
crates/macos/src/actions/post_state.rs
View file

@ -1,31 +1,31 @@
use agent_desktop_core::{action::Action, adapter::LiveElement, element_state::ElementState};
use agent_desktop_core::{adapter::LiveElement, element_state::ElementState};
#[cfg(target_os = "macos")]
pub(crate) fn read_post_state(
el: &crate::tree::AXElement,
action: &Action,
action: &agent_desktop_core::action::Action,
) -> Option<ElementState> {
let delay_ms = match action {
Action::Click | Action::TypeText(_) => 50,
Action::Toggle
| Action::Check
| Action::Uncheck
| Action::SetValue(_)
| Action::Clear
| Action::Expand
| Action::Collapse => 0,
Action::DoubleClick
| Action::RightClick
| Action::TripleClick
| Action::SetFocus
| Action::Select(_)
| Action::Scroll(_, _)
| Action::ScrollTo
| Action::PressKey(_)
| Action::KeyDown(_)
| Action::KeyUp(_)
| Action::Hover
| Action::Drag(_) => return None,
agent_desktop_core::action::Action::Click
| agent_desktop_core::action::Action::TypeText(_) => 50,
agent_desktop_core::action::Action::Toggle
| agent_desktop_core::action::Action::Check
| agent_desktop_core::action::Action::Uncheck
| agent_desktop_core::action::Action::SetValue(_)
| agent_desktop_core::action::Action::Clear
| agent_desktop_core::action::Action::Expand
| agent_desktop_core::action::Action::Collapse => 0,
agent_desktop_core::action::Action::DoubleClick
| agent_desktop_core::action::Action::RightClick
| agent_desktop_core::action::Action::TripleClick
| agent_desktop_core::action::Action::SetFocus
| agent_desktop_core::action::Action::Select(_)
| agent_desktop_core::action::Action::Scroll(_, _)
| agent_desktop_core::action::Action::ScrollTo
| agent_desktop_core::action::Action::PressKey(_)
| agent_desktop_core::action::Action::KeyDown(_)
| agent_desktop_core::action::Action::KeyUp(_)
| agent_desktop_core::action::Action::Hover
| agent_desktop_core::action::Action::Drag(_) => return None,
};
if delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
@ -36,7 +36,7 @@ pub(crate) fn read_post_state(
pub(crate) fn read_element_state(el: &crate::tree::AXElement) -> ElementState {
let attrs = crate::tree::element::fetch_node_attrs(el);
let role = normalized_role(attrs.role.as_deref());
element_state_from_attrs(attrs, role)
element_state_from_attrs(el, attrs, role)
}
pub(crate) fn read_live_element(el: &crate::tree::AXElement) -> LiveElement {
@ -44,7 +44,7 @@ pub(crate) fn read_live_element(el: &crate::tree::AXElement) -> LiveElement {
let role = normalized_role(attrs.role.as_deref());
let bounds = attrs.bounds;
let has_scrollbars = attrs.has_scrollbars;
let state = element_state_from_attrs(attrs, role.clone());
let state = element_state_from_attrs(el, attrs, role.clone());
LiveElement {
state: Some(state),
bounds,
@ -62,31 +62,22 @@ pub(crate) fn read_live_actions(el: &crate::tree::AXElement) -> Vec<String> {
crate::tree::action_list::platform_available_actions(el, &role, attrs.has_scrollbars)
}
fn element_state_from_attrs(attrs: crate::tree::NodeAttrs, role: String) -> ElementState {
let value = attrs.value;
let focused = attrs.states.focused.unwrap_or(false);
let expanded = attrs
.states
.expanded
.or(attrs.states.disclosing)
.unwrap_or(false);
let mut states = Vec::new();
if focused {
states.push("focused".into());
}
if !attrs.states.enabled {
states.push("disabled".into());
}
if expanded {
states.push("expanded".into());
}
if crate::tree::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) {
states.push("checked".into());
}
fn element_state_from_attrs(
el: &crate::tree::AXElement,
attrs: crate::tree::NodeAttrs,
role: String,
) -> ElementState {
let is_secure = attrs.role.as_deref() == Some("AXSecureTextField");
let ctx = crate::tree::state_reader::StateReaderContext {
focused: None,
window_bounds: None,
is_secure_text: is_secure,
};
let states = crate::tree::state_reader::states_from_element(el, &attrs, &role, &ctx);
ElementState {
role,
states,
value,
value: attrs.value,
}
}
@ -96,7 +87,3 @@ fn normalized_role(ax_role: Option<&str>) -> String {
.unwrap_or("unknown")
.to_string()
}
fn value_is_checked(value: Option<&str>) -> bool {
matches!(value, Some("1" | "true"))
}

13
crates/macos/src/tree/build_context.rs
View file

@ -1,7 +1,10 @@
use agent_desktop_core::node::Rect;
use super::AXElement;
pub struct TreeBuildContext {
pub(crate) focused: Option<AXElement>,
pub(crate) window_bounds: Option<Rect>,
include_bounds: bool,
}
@ -10,6 +13,7 @@ impl TreeBuildContext {
let app = super::element_for_pid(pid);
Self {
focused: super::copy_element_attr(&app, "AXFocusedUIElement"),
window_bounds: None,
include_bounds,
}
}
@ -17,10 +21,19 @@ impl TreeBuildContext {
pub fn empty(include_bounds: bool) -> Self {
Self {
focused: None,
window_bounds: None,
include_bounds,
}
}
pub(crate) fn child_context(&self, window_bounds: Option<Rect>) -> Self {
Self {
focused: self.focused.clone(),
window_bounds: window_bounds.or(self.window_bounds),
include_bounds: self.include_bounds,
}
}
pub(crate) fn bounds_for(
&self,
bounds: Option<agent_desktop_core::node::Rect>,

57
crates/macos/src/tree/builder.rs
View file

@ -4,9 +4,8 @@ use rustc_hash::FxHashSet;
use super::AXElement;
use super::action_list::platform_available_actions;
use super::attributes::{copy_ax_array, copy_ax_array_prefix, copy_bool_attr, copy_string_attr};
use super::attributes::{copy_ax_array, copy_ax_array_prefix, copy_string_attr};
use super::build_context::TreeBuildContext;
use super::capabilities::same_element;
use super::element::{
ABSOLUTE_MAX_DEPTH, child_attributes, count_children, element_for_pid, fetch_node_attrs,
};
@ -128,7 +127,7 @@ pub fn build_subtree(
let (role, promoted_label) =
crate::tree::roles::normalized_role_and_label(el, attrs.role.as_deref());
let is_secure_text = is_secure_text_role(attrs.role.as_deref());
let value = redact_secure_value(attrs.role.as_deref(), attrs.value);
let value = redact_secure_value(attrs.role.as_deref(), attrs.value.clone());
let is_promoted_item = promoted_label.is_some();
let available_actions = if is_promoted_item {
vec![capability::CLICK.into(), capability::RIGHT_CLICK.into()]
@ -138,7 +137,7 @@ pub fn build_subtree(
let name = promoted_label.or_else(|| attrs.title.clone().or_else(|| attrs.description.clone()));
let description = if attrs.title.is_some() {
attrs.description
attrs.description.clone()
} else {
None
};
@ -149,31 +148,12 @@ pub fn build_subtree(
name
};
let mut states = Vec::new();
if context
.focused
.as_ref()
.is_some_and(|focused| same_element(el, focused))
{
states.push("focused".into());
}
if !attrs.states.enabled {
states.push("disabled".into());
}
if is_secure_text {
states.push("secure".into());
}
if attrs
.states
.expanded
.or(attrs.states.disclosing)
.unwrap_or_else(|| element_is_expanded(el))
{
states.push("expanded".into());
}
if super::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) {
states.push("checked".into());
}
let state_ctx = super::state_reader::StateReaderContext {
focused: context.focused.as_ref(),
window_bounds: context.window_bounds,
is_secure_text,
};
let states = super::state_reader::states_from_element(el, &attrs, &role, &state_ctx);
let bounds = context.bounds_for(attrs.bounds);
@ -216,6 +196,13 @@ pub fn build_subtree(
let children_raw = copy_children(el, attrs.role.as_deref()).unwrap_or_default();
let name = name.or_else(|| label_from_children(&children_raw));
let child_window_bounds = if attrs.role.as_deref() == Some("AXWindow") {
attrs.bounds.or(context.window_bounds)
} else {
context.window_bounds
};
let child_context = context.child_context(child_window_bounds);
let children = if is_promoted_item {
Vec::new()
} else {
@ -229,7 +216,7 @@ pub fn build_subtree(
max_depth,
ancestors,
skeleton,
context,
&child_context,
)
})
.collect()
@ -264,16 +251,6 @@ fn redact_secure_value(ax_role: Option<&str>, value: Option<String>) -> Option<S
}
}
fn element_is_expanded(el: &AXElement) -> bool {
copy_bool_attr(el, "AXExpanded")
.or_else(|| copy_bool_attr(el, "AXDisclosing"))
.unwrap_or(false)
}
fn value_is_checked(value: Option<&str>) -> bool {
matches!(value, Some("1" | "true"))
}
pub fn label_from_children(children: &[AXElement]) -> Option<String> {
#[cfg(target_os = "macos")]
{

53
crates/macos/src/tree/element.rs
View file

@ -63,6 +63,11 @@ mod imp {
"AXFocused",
"AXExpanded",
"AXDisclosing",
"AXSelected",
"AXHidden",
"AXElementBusy",
"AXModal",
"AXRequired",
kAXPositionAttribute,
kAXSizeAttribute,
SCROLLBAR_ATTRS[0],
@ -90,16 +95,16 @@ mod imp {
return fetch_node_attrs_slow(el);
};
let mut texts: [Option<String>; 8] = Default::default();
let mut texts: [Option<String>; 13] = Default::default();
let mut position: Option<CGPoint> = None;
let mut size: Option<CGSize> = None;
let mut has_scrollbars = false;
for (idx, item) in arr.into_iter().enumerate() {
match idx {
0..=7 => texts[idx] = decode_text_attr(idx, &item),
8 => position = decode_ax_point(&item),
9 => size = decode_ax_size(&item),
10 | 11 => {
0..=12 => texts[idx] = decode_text_attr(idx, &item),
13 => position = decode_ax_point(&item),
14 => size = decode_ax_size(&item),
15 | 16 => {
has_scrollbars =
has_scrollbars || ax_value::retained_ax_element(&item).is_some();
}
@ -108,8 +113,11 @@ mod imp {
}
let get = |i: usize| texts.get(i).and_then(|v| v.clone());
let role = get(0);
let readonly = editable_ax_role(role.as_deref())
.then(|| !crate::tree::capabilities::is_attr_settable(el, kAXValueAttribute));
NodeAttrs {
role: get(0),
role,
title: get(1),
description: get(2),
value: get(3),
@ -118,6 +126,12 @@ mod imp {
focused: parse_bool_attr(get(5)),
expanded: parse_bool_attr(get(6)),
disclosing: parse_bool_attr(get(7)),
selected: parse_bool_attr(get(8)),
hidden: parse_bool_attr(get(9)),
busy: parse_bool_attr(get(10)),
modal: parse_bool_attr(get(11)),
required: parse_bool_attr(get(12)),
readonly,
},
bounds: position.zip(size).and_then(|(p, s)| rect_from_parts(p, s)),
has_scrollbars,
@ -143,7 +157,7 @@ mod imp {
}
None
}
4..=7 => item
4..=12 => item
.downcast::<CFBoolean>()
.map(|b| bool::from(b).to_string()),
_ => None,
@ -180,6 +194,8 @@ mod imp {
let desc = copy_string_attr(el, kAXDescriptionAttribute);
let val = copy_value_typed(el);
let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true);
let readonly = editable_ax_role(role.as_deref())
.then(|| !crate::tree::capabilities::is_attr_settable(el, kAXValueAttribute));
NodeAttrs {
role,
title,
@ -190,12 +206,35 @@ mod imp {
focused: copy_bool_attr(el, "AXFocused"),
expanded: copy_bool_attr(el, "AXExpanded"),
disclosing: copy_bool_attr(el, "AXDisclosing"),
selected: copy_bool_attr(el, "AXSelected"),
hidden: copy_bool_attr(el, "AXHidden"),
busy: copy_bool_attr(el, "AXElementBusy"),
modal: copy_bool_attr(el, "AXModal"),
required: copy_bool_attr(el, "AXRequired"),
readonly,
},
bounds: read_bounds(el),
has_scrollbars: copy_first_element_attr(el, &SCROLLBAR_ATTRS).is_some(),
}
}
fn editable_ax_role(role: Option<&str>) -> bool {
matches!(
role,
Some(
"AXTextField"
| "AXTextArea"
| "AXSearchField"
| "AXComboBox"
| "AXPopUpButton"
| "AXIncrementor"
| "AXStepper"
| "AXSlider"
| "AXValueIndicator"
)
)
}
pub fn resolve_element_name(el: &AXElement) -> Option<String> {
let ax_role = copy_string_attr(el, kAXRoleAttribute);
let title = copy_string_attr(el, kAXTitleAttribute);

1
crates/macos/src/tree/mod.rs
View file

@ -17,6 +17,7 @@ mod resolve_identity;
mod resolve_roots;
mod resolve_search;
pub mod roles;
pub mod state_reader;
pub mod surfaces;
pub(crate) use attributes::{

12
crates/macos/src/tree/node_attrs.rs
View file

@ -17,6 +17,12 @@ pub(crate) struct NodeAttrStates {
pub(crate) focused: Option<bool>,
pub(crate) expanded: Option<bool>,
pub(crate) disclosing: Option<bool>,
pub(crate) selected: Option<bool>,
pub(crate) hidden: Option<bool>,
pub(crate) busy: Option<bool>,
pub(crate) modal: Option<bool>,
pub(crate) required: Option<bool>,
pub(crate) readonly: Option<bool>,
}
impl Default for NodeAttrStates {
@ -26,6 +32,12 @@ impl Default for NodeAttrStates {
focused: None,
expanded: None,
disclosing: None,
selected: None,
hidden: None,
busy: None,
modal: None,
required: None,
readonly: None,
}
}
}

199
crates/macos/src/tree/state_reader.rs Normal file
View file

@ -0,0 +1,199 @@
use agent_desktop_core::node::Rect;
use agent_desktop_core::state;
use super::attributes::copy_bool_attr;
use super::{AXElement, NodeAttrs};
pub(crate) struct StateReaderContext<'a> {
pub focused: Option<&'a AXElement>,
pub window_bounds: Option<Rect>,
pub is_secure_text: bool,
}
pub(crate) fn states_from_element(
el: &AXElement,
attrs: &NodeAttrs,
role: &str,
ctx: &StateReaderContext<'_>,
) -> Vec<String> {
let mut states = Vec::new();
if ctx
.focused
.is_some_and(|focused| super::capabilities::same_element(el, focused))
|| attrs.states.focused == Some(true)
{
states.push(state::FOCUSED.into());
}
if !attrs.states.enabled {
states.push(state::DISABLED.into());
}
if ctx.is_secure_text {
states.push(state::SECURE.into());
}
if is_expanded(el, attrs) {
states.push(state::EXPANDED.into());
}
if super::roles::is_toggleable_role(role) {
if value_is_checked(attrs.value.as_deref()) {
states.push(state::CHECKED.into());
} else if value_is_indeterminate(attrs.value.as_deref()) {
states.push(state::INDETERMINATE.into());
}
}
if attrs.states.selected == Some(true) {
states.push(state::SELECTED.into());
}
if attrs.states.hidden == Some(true) {
states.push(state::HIDDEN.into());
}
if attrs.states.busy == Some(true) {
states.push(state::BUSY.into());
}
if attrs.states.modal == Some(true) {
states.push(state::MODAL.into());
}
if attrs.states.required == Some(true) {
states.push(state::REQUIRED.into());
}
if role == "button" && value_is_checked(attrs.value.as_deref()) {
states.push(state::PRESSED.into());
}
if attrs.states.readonly == Some(true) {
states.push(state::READONLY.into());
}
if is_offscreen(attrs.bounds, ctx.window_bounds) {
states.push(state::OFFSCREEN.into());
}
states
}
fn is_expanded(el: &AXElement, attrs: &NodeAttrs) -> bool {
if attrs
.states
.expanded
.or(attrs.states.disclosing)
.unwrap_or(false)
{
return true;
}
if attrs.states.expanded.is_some() || attrs.states.disclosing.is_some() {
return false;
}
copy_bool_attr(el, "AXExpanded")
.or_else(|| copy_bool_attr(el, "AXDisclosing"))
.unwrap_or(false)
}
fn value_is_checked(value: Option<&str>) -> bool {
matches!(value, Some("1" | "true"))
}
fn value_is_indeterminate(value: Option<&str>) -> bool {
matches!(value, Some("2" | "mixed"))
}
fn is_offscreen(bounds: Option<Rect>, window_bounds: Option<Rect>) -> bool {
let (Some(el), Some(win)) = (bounds, window_bounds) else {
return false;
};
let el_right = el.x + el.width;
let el_bottom = el.y + el.height;
let win_right = win.x + win.width;
let win_bottom = win.y + win.height;
el_right <= win.x || el.x >= win_right || el_bottom <= win.y || el.y >= win_bottom
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tree::node_attrs::{NodeAttrStates, NodeAttrs};
use agent_desktop_core::node::Rect;
fn sample_attrs() -> NodeAttrs {
NodeAttrs {
role: Some("AXCheckBox".into()),
title: None,
description: None,
value: Some("2".into()),
states: NodeAttrStates {
enabled: true,
focused: None,
expanded: None,
disclosing: None,
selected: None,
hidden: None,
busy: None,
modal: None,
required: None,
readonly: None,
},
bounds: Some(Rect {
x: 0.0,
y: 0.0,
width: 10.0,
height: 10.0,
}),
has_scrollbars: false,
}
}
#[test]
fn hidden_and_offscreen_tokens_are_vocabulary_members() {
for token in [state::HIDDEN, state::OFFSCREEN, state::INDETERMINATE] {
state::assert_states_in_vocabulary(&[token.to_string()]);
}
}
#[test]
fn mixed_checkbox_emits_indeterminate_not_checked() {
let attrs = sample_attrs();
let ctx = StateReaderContext {
focused: None,
window_bounds: None,
is_secure_text: false,
};
let el = AXElement(std::ptr::null_mut());
let states = states_from_element(&el, &attrs, "checkbox", &ctx);
assert!(states.contains(&state::INDETERMINATE.to_string()));
assert!(!states.contains(&state::CHECKED.to_string()));
}
#[test]
fn hidden_attr_emits_hidden_token() {
let mut attrs = sample_attrs();
attrs.states.hidden = Some(true);
let ctx = StateReaderContext {
focused: None,
window_bounds: None,
is_secure_text: false,
};
let el = AXElement(std::ptr::null_mut());
let states = states_from_element(&el, &attrs, "button", &ctx);
assert!(states.contains(&state::HIDDEN.to_string()));
}
#[test]
fn clipped_bounds_emit_offscreen() {
let mut attrs = sample_attrs();
attrs.bounds = Some(Rect {
x: 100.0,
y: 0.0,
width: 10.0,
height: 10.0,
});
let window = Rect {
x: 0.0,
y: 0.0,
width: 50.0,
height: 50.0,
};
let ctx = StateReaderContext {
focused: None,
window_bounds: Some(window),
is_secure_text: false,
};
let el = AXElement(std::ptr::null_mut());
let states = states_from_element(&el, &attrs, "button", &ctx);
assert!(states.contains(&state::OFFSCREEN.to_string()));
}
}