Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-08-07 14:40:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: plumb native_id through ref identity and FFI

Browse source 
Capture AXIdentifier as native_id with auto-generated filtering, prioritize it
in identity_matches, and extend AdRefEntry via the size-pin sequence.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Lahfir 2026-07-02 21:10:45 -07:00
parent 2cf0919812
commit 74cd0c3584
55 changed files with 232 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
crates/core/src/actionability_live_tests.rs
View file

@ -140,6 +140,7 @@ fn entry() -> RefEntry {
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),

1
crates/core/src/actionability_tests.rs
View file

@ -21,6 +21,7 @@ fn entry() -> RefEntry {
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),

1
crates/core/src/commands/drag_tests.rs
View file

@ -111,6 +111,7 @@ fn ref_entry(pid: i32) -> RefEntry {
name: Some("Item".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

3
crates/core/src/commands/find_tests.rs
View file

@ -7,6 +7,7 @@ fn node(name: Option<&str>, value: Option<&str>, description: Option<&str>) -> A
name: name.map(String::from),
value: value.map(String::from),
description: description.map(String::from),
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -75,6 +76,7 @@ fn default_limit_caps_materialized_matches() {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -130,6 +132,7 @@ fn count_matches_does_not_build_result_json() {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

3
crates/core/src/commands/helpers_ref_action_tests.rs
View file

@ -239,6 +239,7 @@ impl ObservationOps for ScopedWaitAdapter {
name: Some("Saved!".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -342,6 +343,7 @@ impl ObservationOps for MultiWindowAdapter {
name: Some("Saved!".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -358,6 +360,7 @@ impl ObservationOps for MultiWindowAdapter {
name: Some(win.title.clone()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/commands/helpers_test_support.rs
View file

@ -7,6 +7,7 @@ pub(super) fn entry() -> RefEntry {
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

1
crates/core/src/commands/hover_tests.rs
View file

@ -65,6 +65,7 @@ fn ref_snapshot(pid: i32) -> String {
name: Some("Target".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

2
crates/core/src/commands/is_check_tests.rs
View file

@ -76,6 +76,7 @@ fn entry(states: Vec<String>, value: Option<&str>, actions: Vec<&str>) -> RefEnt
name: Some("Target".into()),
value: value.map(str::to_string),
description: None,
native_id: None,
states,
bounds: None,
bounds_hash: None,
@ -331,6 +332,7 @@ fn action_availability_makes_toggle_and_expand_applicable() {
name: Some("Disclosure".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

1
crates/core/src/commands/query_tests.rs
View file

@ -8,6 +8,7 @@ fn node(role: &str, name: Option<&str>, value: Option<&str>) -> AccessibilityNod
name: name.map(String::from),
value: value.map(String::from),
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/commands/ref_policy_tests.rs
View file

@ -84,6 +84,7 @@ fn snapshot_id() -> String {
name: Some("Target".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

2
crates/core/src/commands/right_click_tests.rs
View file

@ -57,6 +57,7 @@ impl ObservationOps for ProbeFailingAdapter {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: Vec::new(),
available_actions: Vec::new(),
@ -89,6 +90,7 @@ fn save_refmap(source_app: Option<String>) -> String {
name: Some("Open".into()),
value: None,
description: None,
native_id: None,
states: Vec::new(),
bounds: None,
bounds_hash: None,

2
crates/core/src/commands/snapshot_tests.rs
View file

@ -39,6 +39,7 @@ impl ObservationOps for WaitSnapshotAdapter {
name: Some("Doc".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -50,6 +51,7 @@ impl ObservationOps for WaitSnapshotAdapter {
name: Some("Submit".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/commands/wait_latest_ref_cache_tests.rs
View file

@ -21,6 +21,7 @@ fn refmap_with_ref(pid: i32, name: Option<&str>) -> RefMap {
name: name.map(String::from),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

1
crates/core/src/commands/wait_resolution_tests.rs
View file

@ -116,6 +116,7 @@ fn snapshot_with_one_ref() -> String {
name: Some("Run".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

2
crates/core/src/commands/wait_selector_tests.rs
View file

@ -17,6 +17,7 @@ fn window_node(children: Vec<AccessibilityNode>) -> AccessibilityNode {
name: Some("Doc".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -33,6 +34,7 @@ fn button_node(label: &str) -> AccessibilityNode {
name: Some(label.into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/commands/wait_test_support.rs
View file

@ -90,6 +90,7 @@ fn save_ref_in_store(store: RefStore, states: Vec<String>) -> String {
name: Some("Run".into()),
value: None,
description: None,
native_id: None,
states,
bounds: None,
bounds_hash: None,

1
crates/core/src/commands/wait_tests.rs
View file

@ -324,6 +324,7 @@ impl ObservationOps for TextlessTreeAdapter {
name: Some("Doc".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/commands/wait_text_match.rs
View file

@ -51,6 +51,7 @@ mod tests {
name: Some(name.into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

3
crates/core/src/node.rs
View file

@ -16,6 +16,9 @@ pub struct AccessibilityNode {
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub native_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hint: Option<String>,

4
crates/core/src/node_tests.rs
View file

@ -25,6 +25,7 @@ fn test_children_count_omitted_when_none() {
name: Some("Sidebar".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -44,6 +45,7 @@ fn test_children_count_present_when_set() {
name: Some("Sidebar".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -148,6 +150,7 @@ fn accessibility_node_bounds_none_omitted_from_json() {
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],
@ -172,6 +175,7 @@ fn accessibility_node_hint_none_omitted_from_json() {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/ref_action_tests.rs
View file

@ -70,6 +70,7 @@ fn entry() -> RefEntry {
name: Some("Run".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

1
crates/core/src/ref_alloc.rs
View file

@ -19,6 +19,7 @@ pub(crate) fn ref_entry_from_node(
name: meaningful_string(node.name.clone()),
value: meaningful_string(node.value.clone()),
description: meaningful_string(node.description.clone()),
native_id: meaningful_string(node.native_id.clone()),
states: node.states.clone(),
bounds: node.bounds,
bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()),

1
crates/core/src/ref_alloc_tests.rs
View file

@ -8,6 +8,7 @@ fn node(role: &str, name: Option<&str>) -> AccessibilityNode {
name: name.map(str::to_string),
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

23
crates/core/src/ref_identity.rs
View file

@ -2,12 +2,13 @@ use crate::{adapter::SnapshotSurface, refs::RefEntry, roles::is_mutable_value_ro
/// Returns true when a saved ref has stable text identity beyond role/path/bounds.
pub fn has_meaningful_identity(entry: &RefEntry) -> bool {
stable_name(
entry.role.as_str(),
entry.name.as_deref(),
entry.value.as_deref(),
)
.is_some()
meaningful_text(entry.native_id.as_deref()).is_some()
|| stable_name(
entry.role.as_str(),
entry.name.as_deref(),
entry.value.as_deref(),
)
.is_some()
|| stable_value(entry.role.as_str(), entry.value.as_deref()).is_some()
|| meaningful_text(entry.description.as_deref()).is_some()
}
@ -19,7 +20,17 @@ pub fn identity_matches(
actual_name: Option<&str>,
actual_value: Option<&str>,
actual_description: Option<&str>,
actual_native_id: Option<&str>,
) -> bool {
match (
meaningful_text(entry.native_id.as_deref()),
meaningful_text(actual_native_id),
) {
(Some(expected), Some(actual)) => return actual == expected,
(Some(_), None) => return false,
(None, _) => {}
}
let expected_name = stable_name(
entry.role.as_str(),
entry.name.as_deref(),

122
crates/core/src/ref_identity_tests.rs
View file

@ -7,6 +7,7 @@ fn entry() -> RefEntry {
name: None,
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,
@ -28,10 +29,16 @@ fn empty_identity_matches_missing_or_empty_ax_text() {
entry.name = Some(String::new());
assert!(!has_meaningful_identity(&entry));
assert!(identity_matches(&entry, None, None, None));
assert!(identity_matches(&entry, Some(""), None, None));
assert!(identity_matches(&entry, None, Some(""), None));
assert!(!identity_matches(&entry, Some("Insert Shape"), None, None));
assert!(identity_matches(&entry, None, None, None, None));
assert!(identity_matches(&entry, Some(""), None, None, None));
assert!(identity_matches(&entry, None, Some(""), None, None));
assert!(!identity_matches(
&entry,
Some("Insert Shape"),
None,
None,
None
));
}
#[test]
@ -44,20 +51,23 @@ fn description_identity_matches_blank_title_controls() {
&entry,
Some(""),
None,
Some("Insert Text Box")
Some("Insert Text Box"),
None,
));
assert!(identity_matches(
&entry,
Some("Insert Text Box"),
None,
None
None,
None,
));
assert!(!identity_matches(&entry, Some(""), None, None));
assert!(!identity_matches(&entry, Some(""), None, None, None));
assert!(!identity_matches(
&entry,
Some(""),
None,
Some("Insert Shape")
Some("Insert Shape"),
None,
));
}
@ -67,15 +77,16 @@ fn name_identity_cannot_be_rescued_by_matching_description() {
entry.name = Some("Primary".into());
entry.description = Some("Generic".into());
assert!(identity_matches(&entry, Some("Primary"), None, None));
assert!(identity_matches(&entry, None, Some("Primary"), None));
assert!(identity_matches(&entry, Some("Primary"), None, None, None));
assert!(identity_matches(&entry, None, Some("Primary"), None, None));
assert!(!identity_matches(
&entry,
Some("Other"),
None,
Some("Primary")
Some("Primary"),
None,
));
assert!(!identity_matches(&entry, Some("Generic"), None, None));
assert!(!identity_matches(&entry, Some("Generic"), None, None, None));
}
#[test]
@ -83,9 +94,15 @@ fn value_identity_cannot_be_rescued_by_matching_name_when_value_mismatches() {
let mut entry = entry();
entry.value = Some("On".into());
assert!(identity_matches(&entry, None, Some("On"), None));
assert!(identity_matches(&entry, Some("On"), None, None));
assert!(!identity_matches(&entry, Some("On"), Some("Off"), None));
assert!(identity_matches(&entry, None, Some("On"), None, None));
assert!(identity_matches(&entry, Some("On"), None, None, None));
assert!(!identity_matches(
&entry,
Some("On"),
Some("Off"),
None,
None
));
}
#[test]
@ -95,7 +112,7 @@ fn mutable_value_role_does_not_go_stale_when_value_changes() {
entry.value = Some("seed".into());
assert!(!has_meaningful_identity(&entry));
assert!(identity_matches(&entry, None, Some("changed"), None));
assert!(identity_matches(&entry, None, Some("changed"), None, None));
}
#[test]
@ -108,6 +125,7 @@ fn unnamed_mutable_value_role_does_not_go_stale_when_content_becomes_name() {
&entry,
Some("typed document text"),
Some("typed document text"),
None,
None
));
}
@ -120,7 +138,13 @@ fn mutable_value_text_promoted_to_name_is_not_stable_identity() {
entry.value = Some("00:01".into());
assert!(!has_meaningful_identity(&entry));
assert!(identity_matches(&entry, Some("00:06"), Some("00:06"), None));
assert!(identity_matches(
&entry,
Some("00:06"),
Some("00:06"),
None,
None
));
}
#[test]
@ -131,7 +155,13 @@ fn formatted_numeric_mutable_value_promoted_to_name_is_not_stable_identity() {
entry.value = Some("50.0".into());
assert!(!has_meaningful_identity(&entry));
assert!(identity_matches(&entry, Some("51"), Some("51.0"), None));
assert!(identity_matches(
&entry,
Some("51"),
Some("51.0"),
None,
None
));
}
#[test]
@ -146,12 +176,14 @@ fn named_mutable_value_role_still_uses_name_identity() {
&entry,
Some("Search"),
Some("new query"),
None,
None
));
assert!(!identity_matches(
&entry,
Some("Replace"),
Some("new query"),
None,
None
));
}
@ -164,8 +196,58 @@ fn mutable_role_label_different_from_value_remains_stable_identity() {
entry.value = Some("Helvetica".into());
assert!(has_meaningful_identity(&entry));
assert!(identity_matches(&entry, Some("Font"), Some("Arial"), None));
assert!(!identity_matches(&entry, Some("Size"), Some("Arial"), None));
assert!(identity_matches(
&entry,
Some("Font"),
Some("Arial"),
None,
None
));
assert!(!identity_matches(
&entry,
Some("Size"),
Some("Arial"),
None,
None
));
}
#[test]
fn native_id_is_strongest_identity_signal() {
let mut entry = entry();
entry.native_id = Some("submit-btn".into());
entry.name = Some("Old Label".into());
assert!(has_meaningful_identity(&entry));
assert!(identity_matches(
&entry,
Some("Renamed"),
None,
None,
Some("submit-btn"),
));
assert!(!identity_matches(
&entry,
Some("Renamed"),
None,
None,
Some("cancel-btn"),
));
}
#[test]
fn differing_native_ids_are_hard_non_match() {
let mut entry = entry();
entry.native_id = Some("field-a".into());
entry.name = Some("Same".into());
assert!(!identity_matches(
&entry,
Some("Same"),
None,
None,
Some("field-b"),
));
}
#[test]

2
crates/core/src/refs.rs
View file

@ -28,6 +28,8 @@ pub struct RefEntry {
pub value: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub native_id: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub states: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]

3
crates/core/src/refs_serde_tests.rs
View file

@ -8,6 +8,7 @@ fn minimal_entry(role: &str) -> RefEntry {
name: None,
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,
@ -141,6 +142,7 @@ fn ref_entry_full_roundtrip_preserves_all_fields() {
name: Some("Email".into()),
value: Some("user@example.com".into()),
description: Some("Enter email".into()),
native_id: Some("email-field".into()),
states: vec!["focused".into()],
bounds: None,
bounds_hash: Some(0xDEAD_BEEF),
@ -161,6 +163,7 @@ fn ref_entry_full_roundtrip_preserves_all_fields() {
assert_eq!(back.name, original.name);
assert_eq!(back.value, original.value);
assert_eq!(back.description, original.description);
assert_eq!(back.native_id, original.native_id);
assert_eq!(back.states, original.states);
assert_eq!(back.bounds_hash, original.bounds_hash);
assert_eq!(back.available_actions, original.available_actions);

1
crates/core/src/refs_store_tests.rs
View file

@ -12,6 +12,7 @@ fn entry(name: &str) -> RefEntry {
name: Some(name.into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: Some(42),

3
crates/core/src/refs_tests.rs
View file

@ -8,6 +8,7 @@ fn entry(role: &str, name: Option<&str>) -> RefEntry {
name: name.map(String::from),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,
@ -41,6 +42,7 @@ fn test_get_existing() {
name: None,
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: Some(12345),
@ -187,6 +189,7 @@ fn test_save_load_roundtrip_with_home_override() {
name: Some("Send".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: Some(42),

1
crates/core/src/search_text_tests.rs
View file

@ -8,6 +8,7 @@ fn node(name: Option<&str>, value: Option<&str>, description: Option<&str>) -> A
name: name.map(String::from),
value: value.map(String::from),
description: description.map(String::from),
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/snapshot_ref_alloc_tests.rs
View file

@ -9,6 +9,7 @@ fn node(role: &str) -> AccessibilityNode {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/snapshot_ref_tests.rs
View file

@ -15,6 +15,7 @@ fn node(role: &str) -> AccessibilityNode {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/snapshot_tests.rs
View file

@ -8,6 +8,7 @@ fn node(role: &str) -> AccessibilityNode {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/core/src/trace_artifacts_tests.rs
View file

@ -32,6 +32,7 @@ pub(super) fn entry(pid: i32) -> RefEntry {
name: Some("Run".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash: None,

3
crates/ffi/include/agent_desktop.h
View file

@ -74,7 +74,7 @@
#define AD_ELEMENT_STATE_SIZE 32
#define AD_REF_ENTRY_SIZE 192
#define AD_REF_ENTRY_SIZE 200
/**
* Per-field input caps enforced when converting an `AdRefEntry` at the C
@ -469,6 +469,7 @@ typedef struct AdRefEntry {
const char *name;
const char *value;
const char *description;
const char *native_id;
const char *const *states;
size_t state_count;
const char *const *available_actions;

2
crates/ffi/src/actions/resolve.rs
View file

@ -61,6 +61,7 @@ pub(crate) unsafe fn core_ref_entry_from_ffi(
let name = unsafe { optional_string(entry.name, "name") }?;
let value = unsafe { optional_string(entry.value, "value") }?;
let description = unsafe { optional_string(entry.description, "description") }?;
let native_id = unsafe { optional_string(entry.native_id, "native_id") }?;
let states = unsafe {
string_array(
entry.states,
@ -103,6 +104,7 @@ pub(crate) unsafe fn core_ref_entry_from_ffi(
name,
value,
description,
native_id,
states,
bounds,
bounds_hash,

1
crates/ffi/src/actions/resolve_tests.rs
View file

@ -9,6 +9,7 @@ fn test_ref_entry() -> AdRefEntry {
name: std::ptr::null(),
value: std::ptr::null(),
description: std::ptr::null(),
native_id: std::ptr::null(),
states: std::ptr::null(),
state_count: 0,
available_actions: std::ptr::null(),

1
crates/ffi/src/observation/find.rs
View file

@ -98,6 +98,7 @@ pub unsafe extern "C" fn ad_find(
name: matched.name.clone(),
value: matched.value.clone(),
description: matched.description.clone(),
native_id: matched.native_id.clone(),
states: matched.states.clone(),
bounds: matched.bounds,
bounds_hash,

1
crates/ffi/src/observation/is.rs
View file

@ -169,6 +169,7 @@ mod tests {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: states.iter().map(|s| s.to_string()).collect(),
available_actions: vec![],

1
crates/ffi/src/observation/walk.rs
View file

@ -63,6 +63,7 @@ mod tests {
name: name.map(str::to_string),
value: value.map(str::to_string),
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/ffi/src/tree/flatten.rs
View file

@ -149,6 +149,7 @@ mod tests {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

1
crates/ffi/src/tree/free.rs
View file

@ -173,6 +173,7 @@ mod tests {
name: None,
value: None,
description: None,
native_id: None,
hint: None,
states: vec![],
available_actions: vec![],

3
crates/ffi/src/types/ref_entry.rs
View file

@ -8,6 +8,7 @@ pub struct AdRefEntry {
pub name: *const c_char,
pub value: *const c_char,
pub description: *const c_char,
pub native_id: *const c_char,
pub states: *const *const c_char,
pub state_count: usize,
pub available_actions: *const *const c_char,
@ -26,7 +27,7 @@ pub struct AdRefEntry {
pub path_count: usize,
}
pub const AD_REF_ENTRY_SIZE: usize = 192;
pub const AD_REF_ENTRY_SIZE: usize = 200;
/// Per-field input caps enforced when converting an `AdRefEntry` at the C
/// boundary, sized from what real accessibility trees produce (a handful of

5
crates/ffi/tests/c_abi_layout.rs
View file

@ -131,12 +131,12 @@ fn ref_entry_input_caps_match_the_published_header_values() {
#[test]
fn ref_entry_layout_is_guarded_for_c_consumers() {
assert_eq!(agent_desktop_ffi::types::ref_entry::AD_REF_ENTRY_SIZE, 192);
assert_eq!(agent_desktop_ffi::types::ref_entry::AD_REF_ENTRY_SIZE, 200);
assert_eq!(
unsafe { common::ad_ref_entry_size() },
agent_desktop_ffi::types::ref_entry::AD_REF_ENTRY_SIZE
);
assert_eq!(size_of::<AdRefEntry>(), 192);
assert_eq!(size_of::<AdRefEntry>(), 200);
assert_eq!(align_of::<AdRefEntry>(), align_of::<usize>());
assert_eq!(offset_of!(AdRefEntry, pid), 0);
@ -146,6 +146,7 @@ fn ref_entry_layout_is_guarded_for_c_consumers() {
offset_of!(AdRefEntry, name),
offset_of!(AdRefEntry, value),
offset_of!(AdRefEntry, description),
offset_of!(AdRefEntry, native_id),
offset_of!(AdRefEntry, states),
offset_of!(AdRefEntry, state_count),
offset_of!(AdRefEntry, available_actions),

2
crates/ffi/tests/c_header_compile.rs
View file

@ -76,7 +76,7 @@ int main(void) {
(void)ad_ref_entry_size;
(void)ad_last_error_details;
_Static_assert(AD_REF_ENTRY_SIZE == sizeof(AdRefEntry), "AdRefEntry size macro drifted");
_Static_assert(AD_REF_ENTRY_SIZE == 192, "AdRefEntry ABI size changed");
_Static_assert(AD_REF_ENTRY_SIZE == 200, "AdRefEntry ABI size changed");
return 0;
}
"#;

1
crates/ffi/tests/common/mod.rs
View file

@ -137,6 +137,7 @@ pub fn default_ref_entry() -> AdRefEntry {
name: std::ptr::null(),
value: std::ptr::null(),
description: std::ptr::null(),
native_id: std::ptr::null(),
states: std::ptr::null(),
state_count: 0,
available_actions: std::ptr::null(),

3
crates/macos/src/tree/builder.rs
View file

@ -105,6 +105,7 @@ pub fn build_subtree(
name,
value,
description: None,
native_id: None,
hint: None,
states,
role,
@ -184,6 +185,7 @@ pub fn build_subtree(
name,
value,
description,
native_id: attrs.native_id.clone(),
hint: None,
states,
available_actions,
@ -230,6 +232,7 @@ pub fn build_subtree(
name,
value,
description,
native_id: attrs.native_id.clone(),
hint: None,
states,
available_actions,

15
crates/macos/src/tree/element.rs
View file

@ -68,6 +68,7 @@ mod imp {
"AXElementBusy",
"AXModal",
"AXRequired",
"AXIdentifier",
kAXPositionAttribute,
kAXSizeAttribute,
SCROLLBAR_ATTRS[0],
@ -95,16 +96,16 @@ mod imp {
return fetch_node_attrs_slow(el);
};
let mut texts: [Option<String>; 13] = Default::default();
let mut texts: [Option<String>; 14] = Default::default();
let mut position: Option<CGPoint> = None;
let mut size: Option<CGSize> = None;
let mut has_scrollbars = false;
for (idx, item) in arr.into_iter().enumerate() {
match idx {
0..=12 => texts[idx] = decode_text_attr(idx, &item),
13 => position = decode_ax_point(&item),
14 => size = decode_ax_size(&item),
15 | 16 => {
0..=13 => texts[idx] = decode_text_attr(idx, &item),
14 => position = decode_ax_point(&item),
15 => size = decode_ax_size(&item),
16 | 17 => {
has_scrollbars =
has_scrollbars || ax_value::retained_ax_element(&item).is_some();
}
@ -121,6 +122,7 @@ mod imp {
title: get(1),
description: get(2),
value: get(3),
native_id: crate::tree::native_id::meaningful_native_id(get(13)),
states: NodeAttrStates {
enabled: parse_enabled(get(4)),
focused: parse_bool_attr(get(5)),
@ -196,11 +198,14 @@ mod imp {
let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true);
let readonly = editable_ax_role(role.as_deref())
.then(|| !crate::tree::capabilities::is_attr_settable(el, kAXValueAttribute));
let native_id =
crate::tree::native_id::meaningful_native_id(copy_string_attr(el, "AXIdentifier"));
NodeAttrs {
role,
title,
description: desc,
value: val,
native_id,
states: NodeAttrStates {
enabled,
focused: copy_bool_attr(el, "AXFocused"),

1
crates/macos/src/tree/mod.rs
View file

@ -8,6 +8,7 @@ pub mod capabilities;
pub mod element;
pub mod element_bounds;
pub(crate) mod element_dedupe;
pub mod native_id;
pub(crate) mod node_attrs;
pub mod resolve;
mod resolve_bounds;

25
crates/macos/src/tree/native_id.rs Normal file
View file

@ -0,0 +1,25 @@
pub(crate) fn meaningful_native_id(raw: Option<String>) -> Option<String> {
raw.filter(|id| !id.is_empty() && !is_auto_generated(id))
}
fn is_auto_generated(id: &str) -> bool {
id.starts_with("_NS")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn auto_generated_ns_prefix_is_filtered() {
assert_eq!(meaningful_native_id(Some("_NS:42".into())), None);
}
#[test]
fn developer_assigned_id_is_kept() {
assert_eq!(
meaningful_native_id(Some("submit-btn".into())),
Some("submit-btn".into())
);
}
}

1
crates/macos/src/tree/node_attrs.rs
View file

@ -6,6 +6,7 @@ pub(crate) struct NodeAttrs {
pub(crate) title: Option<String>,
pub(crate) description: Option<String>,
pub(crate) value: Option<String>,
pub(crate) native_id: Option<String>,
pub(crate) states: NodeAttrStates,
pub(crate) bounds: Option<Rect>,
pub(crate) has_scrollbars: bool,

3
crates/macos/src/tree/resolve_search.rs
View file

@ -181,11 +181,14 @@ fn element_identity_matches(
let elem_name = promoted_label.or_else(|| resolve_element_name(el));
let elem_value = crate::tree::copy_value_typed(el);
let elem_description = copy_string_attr(el, accessibility_sys::kAXDescriptionAttribute);
let elem_native_id =
super::native_id::meaningful_native_id(copy_string_attr(el, "AXIdentifier"));
identity_matches(
entry,
elem_name.as_deref(),
elem_value.as_deref(),
elem_description.as_deref(),
elem_native_id.as_deref(),
)
}

1
crates/macos/src/tree/resolve_tests.rs
View file

@ -18,6 +18,7 @@ fn entry(
name: Some("Investors".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: None,
bounds_hash,

1
crates/macos/src/tree/state_reader.rs
View file

@ -115,6 +115,7 @@ mod tests {
title: None,
description: None,
value: Some("2".into()),
native_id: None,
states: NodeAttrStates {
enabled: true,
focused: None,

1
src/tests/conformance.rs
View file

@ -115,6 +115,7 @@ fn entry(bounds: Rect) -> RefEntry {
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),