From 1fcb2e6936c3172869aebed9c8023c4c2aa58547 Mon Sep 17 00:00:00 2001 From: Lahfir Date: Thu, 16 Apr 2026 04:26:58 -0700 Subject: [PATCH] =?UTF-8?q?feat(ffi):=20observation=20primitives=20?= =?UTF-8?q?=E2=80=94=20ad=5Ffind=20/=20ad=5Fget=20/=20ad=5Fis=20(Unit=209)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cheap single-element lookups without hand-walking the flat-tree result of ad_get_tree. Closes the observation portion of R11 from PR #22 review. New module tree crates/ffi/src/observation/ — one concern per file: - walk.rs: pub(crate) find_first_match(node, role?, name?, value?) — DFS case-insensitive substring matcher shared by find.rs and is.rs. Extracted so both FFI entrypoints speak the same matching semantics and unit-testable in isolation. - find.rs: ad_find(adapter, win, query, &handle). Walks the tree, picks the first match, resolves it to a NativeHandle via the existing PlatformAdapter::resolve_element path. Returns ErrElementNotFound when nothing matches. Caller owns the handle → must ad_free_handle(adapter, handle). - get.rs: ad_get(adapter, handle, property, &str_out) dispatches on property name. Supported: "value" (via get_live_value), "bounds" (via get_element_bounds, formatted as JSON). Unknown property returns ErrInvalidArgs. Output string owned by caller → free with ad_free_string. - is.rs: ad_is(adapter, win, query, property, &bool_out) looks up the element the same way find does but checks a named state against the node's states vec. Recognized: focused, enabled, selected, checked, expanded. Unknown property returns ErrInvalidArgs with a diagnostic listing the valid names. New type in types/find_query.rs (AdFindQuery { role, name_substring, value_substring }) already landed in Unit 5 and is re-exported here. All entries go through trap_panic with debug_assert_main_thread. 53 tests pass. Clippy clean. --- crates/ffi/include/agent_desktop.h | 78 +++++++++++++++++++ crates/ffi/src/lib.rs | 1 + crates/ffi/src/observation/find.rs | 103 +++++++++++++++++++++++++ crates/ffi/src/observation/get.rs | 82 ++++++++++++++++++++ crates/ffi/src/observation/is.rs | 116 +++++++++++++++++++++++++++++ crates/ffi/src/observation/mod.rs | 4 + crates/ffi/src/observation/walk.rs | 52 +++++++++++++ 7 files changed, 436 insertions(+) create mode 100644 crates/ffi/src/observation/find.rs create mode 100644 crates/ffi/src/observation/get.rs create mode 100644 crates/ffi/src/observation/is.rs create mode 100644 crates/ffi/src/observation/mod.rs create mode 100644 crates/ffi/src/observation/walk.rs diff --git a/crates/ffi/include/agent_desktop.h b/crates/ffi/include/agent_desktop.h index ba9436d1..d8c82c51 100644 --- a/crates/ffi/include/agent_desktop.h +++ b/crates/ffi/include/agent_desktop.h @@ -249,6 +249,12 @@ typedef struct AdNotificationInfo { uint32_t action_count; } AdNotificationInfo; +typedef struct AdFindQuery { + const char *role; + const char *name_substring; + const char *value_substring; +} AdFindQuery; + typedef struct AdScreenshotTarget { AdScreenshotKind kind; uint64_t screen_index; @@ -589,6 +595,78 @@ const struct AdNotificationInfo *ad_notification_list_get(const struct AdNotific */ void ad_notification_list_free(struct AdNotificationList *list); +/** + * Finds the first element in `win`'s accessibility tree matching the + * query and resolves it to an opaque `AdNativeHandle`. The caller owns + * the handle and must release it with `ad_free_handle(adapter, handle)` + * once done. + * + * Matching is DFS order, first hit wins. All query fields are optional + * (null = "don't care") and case-insensitive substring matches: + * - `role` against `AccessibilityNode.role` + * - `name_substring` against `AccessibilityNode.name` + * - `value_substring` against `AccessibilityNode.value` + * + * # Safety + * `adapter`, `win`, and `query` must be valid pointers. `out_handle` + * must be a valid writable `*mut AdNativeHandle`. On + * `AD_RESULT_ERR_ELEMENT_NOT_FOUND` the out-handle is zero-initialized. + */ +AdResult ad_find(const struct AdAdapter *adapter, + const struct AdWindowInfo *win, + const struct AdFindQuery *query, + struct AdNativeHandle *out_handle); + +/** + * Reads a single property off a previously-resolved element handle. + * + * Supported properties: + * - `"value"` — live textual value (text fields, sliders, progress + * indicators). Null out-string when the element has no value. + * - `"bounds"` — JSON-encoded `{"x":..,"y":..,"width":..,"height":..}`. + * Null out-string when bounds are unavailable. + * + * The returned string must be freed with `ad_free_string`. + * + * # Safety + * `adapter` must be valid. `handle` must be a non-null `AdNativeHandle`. + * `property` must be a non-null UTF-8 C string. `out` must be a valid + * writable `*mut *mut c_char`; it is null-initialized on entry. + */ +AdResult ad_get(const struct AdAdapter *adapter, + const struct AdNativeHandle *handle, + const char *property, + char **out); + +/** + * Checks whether a named boolean state is set on the first element + * matching `query` inside `win`'s accessibility tree. Intended for the + * common agent idiom `find → is(focused) → if yes, act`. + * + * Recognized property names (match the strings the platform adapter + * emits in `AccessibilityNode.states`): + * + * - `"focused"` + * - `"enabled"` + * - `"selected"` + * - `"checked"` + * - `"expanded"` + * + * Any other property name returns `AD_RESULT_ERR_INVALID_ARGS`. If no + * element matches the query, returns `AD_RESULT_ERR_ELEMENT_NOT_FOUND` + * and `*out` is untouched. + * + * # Safety + * All pointers must be valid. `property` must be a non-null UTF-8 + * C string. `out` must be a valid writable `*mut bool`; it is set to + * `false` on entry. + */ +AdResult ad_is(const struct AdAdapter *adapter, + const struct AdWindowInfo *win, + const struct AdFindQuery *query, + const char *property, + bool *out); + /** * Borrowed pointer to the image bytes; valid until the buffer is freed. * Returns null if `buf` is null. diff --git a/crates/ffi/src/lib.rs b/crates/ffi/src/lib.rs index 06fbc227..7a0436e3 100644 --- a/crates/ffi/src/lib.rs +++ b/crates/ffi/src/lib.rs @@ -42,6 +42,7 @@ pub(crate) mod ffi_try; pub(crate) mod input; pub(crate) mod main_thread; pub(crate) mod notifications; +pub(crate) mod observation; pub(crate) mod screenshot; pub(crate) mod surfaces; pub(crate) mod tree; diff --git a/crates/ffi/src/observation/find.rs b/crates/ffi/src/observation/find.rs new file mode 100644 index 00000000..7f5b3441 --- /dev/null +++ b/crates/ffi/src/observation/find.rs @@ -0,0 +1,103 @@ +use crate::convert::string::c_to_string; +use crate::error::{set_last_error, AdResult}; +use crate::ffi_try::trap_panic; +use crate::observation::walk::find_first_match; +use crate::types::{AdFindQuery, AdNativeHandle, AdWindowInfo}; +use crate::AdAdapter; +use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions}; +use agent_desktop_core::refs::RefEntry; + +/// Finds the first element in `win`'s accessibility tree matching the +/// query and resolves it to an opaque `AdNativeHandle`. The caller owns +/// the handle and must release it with `ad_free_handle(adapter, handle)` +/// once done. +/// +/// Matching is DFS order, first hit wins. All query fields are optional +/// (null = "don't care") and case-insensitive substring matches: +/// - `role` against `AccessibilityNode.role` +/// - `name_substring` against `AccessibilityNode.name` +/// - `value_substring` against `AccessibilityNode.value` +/// +/// # Safety +/// `adapter`, `win`, and `query` must be valid pointers. `out_handle` +/// must be a valid writable `*mut AdNativeHandle`. On +/// `AD_RESULT_ERR_ELEMENT_NOT_FOUND` the out-handle is zero-initialized. +#[no_mangle] +pub unsafe extern "C" fn ad_find( + adapter: *const AdAdapter, + win: *const AdWindowInfo, + query: *const AdFindQuery, + out_handle: *mut AdNativeHandle, +) -> AdResult { + trap_panic(|| unsafe { + crate::main_thread::debug_assert_main_thread(); + (*out_handle).ptr = std::ptr::null(); + let adapter = &*adapter; + let core_win = match crate::windows::ad_window_to_core(&*win) { + Ok(w) => w, + Err(e) => { + set_last_error(&e); + return crate::error::last_error_code(); + } + }; + let q = &*query; + let role_filter = c_to_string(q.role); + let name_filter = c_to_string(q.name_substring); + let value_filter = c_to_string(q.value_substring); + + let tree = match adapter.inner.get_tree( + &core_win, + &TreeOptions { + max_depth: 50, + include_bounds: false, + interactive_only: false, + compact: false, + surface: SnapshotSurface::Window, + }, + ) { + Ok(t) => t, + Err(e) => { + set_last_error(&e); + return crate::error::last_error_code(); + } + }; + + let matched = match find_first_match( + &tree, + role_filter.as_deref(), + name_filter.as_deref(), + value_filter.as_deref(), + ) { + Some(n) => n, + None => { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::ElementNotFound, + "no element matched the find query", + )); + return AdResult::ErrElementNotFound; + } + }; + + let ref_entry = RefEntry { + pid: core_win.pid, + role: matched.role.clone(), + name: matched.name.clone(), + value: None, + states: Vec::new(), + bounds: None, + bounds_hash: None, + available_actions: Vec::new(), + source_app: None, + }; + match adapter.inner.resolve_element(&ref_entry) { + Ok(handle) => { + (*out_handle).ptr = handle.as_raw(); + AdResult::Ok + } + Err(e) => { + set_last_error(&e); + crate::error::last_error_code() + } + } + }) +} diff --git a/crates/ffi/src/observation/get.rs b/crates/ffi/src/observation/get.rs new file mode 100644 index 00000000..5752b2a7 --- /dev/null +++ b/crates/ffi/src/observation/get.rs @@ -0,0 +1,82 @@ +use crate::convert::string::{c_to_string, string_to_c_lossy}; +use crate::error::{set_last_error, AdResult}; +use crate::ffi_try::trap_panic; +use crate::types::AdNativeHandle; +use crate::AdAdapter; +use agent_desktop_core::adapter::NativeHandle; +use std::os::raw::c_char; + +/// Reads a single property off a previously-resolved element handle. +/// +/// Supported properties: +/// - `"value"` — live textual value (text fields, sliders, progress +/// indicators). Null out-string when the element has no value. +/// - `"bounds"` — JSON-encoded `{"x":..,"y":..,"width":..,"height":..}`. +/// Null out-string when bounds are unavailable. +/// +/// The returned string must be freed with `ad_free_string`. +/// +/// # Safety +/// `adapter` must be valid. `handle` must be a non-null `AdNativeHandle`. +/// `property` must be a non-null UTF-8 C string. `out` must be a valid +/// writable `*mut *mut c_char`; it is null-initialized on entry. +#[no_mangle] +pub unsafe extern "C" fn ad_get( + adapter: *const AdAdapter, + handle: *const AdNativeHandle, + property: *const c_char, + out: *mut *mut c_char, +) -> AdResult { + trap_panic(|| unsafe { + crate::main_thread::debug_assert_main_thread(); + *out = std::ptr::null_mut(); + let adapter = &*adapter; + let native = NativeHandle::from_ptr((*handle).ptr); + let prop = match c_to_string(property) { + Some(s) => s, + None => { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + "property is null or invalid UTF-8", + )); + return AdResult::ErrInvalidArgs; + } + }; + + match prop.as_str() { + "value" => match adapter.inner.get_live_value(&native) { + Ok(Some(v)) => { + *out = string_to_c_lossy(&v); + AdResult::Ok + } + Ok(None) => AdResult::Ok, + Err(e) => { + set_last_error(&e); + crate::error::last_error_code() + } + }, + "bounds" => match adapter.inner.get_element_bounds(&native) { + Ok(Some(r)) => { + let json = format!( + "{{\"x\":{},\"y\":{},\"width\":{},\"height\":{}}}", + r.x, r.y, r.width, r.height + ); + *out = string_to_c_lossy(&json); + AdResult::Ok + } + Ok(None) => AdResult::Ok, + Err(e) => { + set_last_error(&e); + crate::error::last_error_code() + } + }, + _ => { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + "unknown property — expected one of: value, bounds", + )); + AdResult::ErrInvalidArgs + } + } + }) +} diff --git a/crates/ffi/src/observation/is.rs b/crates/ffi/src/observation/is.rs new file mode 100644 index 00000000..8bb1eff9 --- /dev/null +++ b/crates/ffi/src/observation/is.rs @@ -0,0 +1,116 @@ +use crate::convert::string::c_to_string; +use crate::error::{set_last_error, AdResult}; +use crate::ffi_try::trap_panic; +use crate::types::{AdFindQuery, AdWindowInfo}; +use crate::AdAdapter; +use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions}; +use agent_desktop_core::node::AccessibilityNode; +use std::os::raw::c_char; + +/// Checks whether a named boolean state is set on the first element +/// matching `query` inside `win`'s accessibility tree. Intended for the +/// common agent idiom `find → is(focused) → if yes, act`. +/// +/// Recognized property names (match the strings the platform adapter +/// emits in `AccessibilityNode.states`): +/// +/// - `"focused"` +/// - `"enabled"` +/// - `"selected"` +/// - `"checked"` +/// - `"expanded"` +/// +/// Any other property name returns `AD_RESULT_ERR_INVALID_ARGS`. If no +/// element matches the query, returns `AD_RESULT_ERR_ELEMENT_NOT_FOUND` +/// and `*out` is untouched. +/// +/// # Safety +/// All pointers must be valid. `property` must be a non-null UTF-8 +/// C string. `out` must be a valid writable `*mut bool`; it is set to +/// `false` on entry. +#[no_mangle] +pub unsafe extern "C" fn ad_is( + adapter: *const AdAdapter, + win: *const AdWindowInfo, + query: *const AdFindQuery, + property: *const c_char, + out: *mut bool, +) -> AdResult { + trap_panic(|| unsafe { + crate::main_thread::debug_assert_main_thread(); + *out = false; + let adapter = &*adapter; + let core_win = match crate::windows::ad_window_to_core(&*win) { + Ok(w) => w, + Err(e) => { + set_last_error(&e); + return crate::error::last_error_code(); + } + }; + let q = &*query; + let role_filter = c_to_string(q.role); + let name_filter = c_to_string(q.name_substring); + let value_filter = c_to_string(q.value_substring); + let prop = match c_to_string(property) { + Some(s) => s, + None => { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + "property is null or invalid UTF-8", + )); + return AdResult::ErrInvalidArgs; + } + }; + if !is_known_property(&prop) { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + "unknown property — expected one of: focused, enabled, selected, checked, expanded", + )); + return AdResult::ErrInvalidArgs; + } + + let tree = match adapter.inner.get_tree( + &core_win, + &TreeOptions { + max_depth: 50, + include_bounds: false, + interactive_only: false, + compact: false, + surface: SnapshotSurface::Window, + }, + ) { + Ok(t) => t, + Err(e) => { + set_last_error(&e); + return crate::error::last_error_code(); + } + }; + + let matched = match crate::observation::walk::find_first_match( + &tree, + role_filter.as_deref(), + name_filter.as_deref(), + value_filter.as_deref(), + ) { + Some(n) => n, + None => { + set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::ElementNotFound, + "no element matched the find query", + )); + return AdResult::ErrElementNotFound; + } + }; + + *out = element_has_state(matched, &prop); + AdResult::Ok + }) +} + +fn is_known_property(p: &str) -> bool { + matches!(p, "focused" | "enabled" | "selected" | "checked" | "expanded") +} + +fn element_has_state(node: &AccessibilityNode, prop: &str) -> bool { + node.states.iter().any(|s| s.eq_ignore_ascii_case(prop)) +} diff --git a/crates/ffi/src/observation/mod.rs b/crates/ffi/src/observation/mod.rs new file mode 100644 index 00000000..43a99425 --- /dev/null +++ b/crates/ffi/src/observation/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod find; +pub(crate) mod get; +pub(crate) mod is; +pub(crate) mod walk; diff --git a/crates/ffi/src/observation/walk.rs b/crates/ffi/src/observation/walk.rs new file mode 100644 index 00000000..1bad1a4e --- /dev/null +++ b/crates/ffi/src/observation/walk.rs @@ -0,0 +1,52 @@ +use agent_desktop_core::node::AccessibilityNode; + +/// Finds the first node in DFS order that matches every provided filter. +/// Filters are ANDed; a `None` filter means "don't care". Substring +/// matching is case-insensitive to tolerate platform-specific casing +/// (macOS AX strings vary between "AXButton" and "button"). +pub(crate) fn find_first_match<'a>( + node: &'a AccessibilityNode, + role: Option<&str>, + name: Option<&str>, + value: Option<&str>, +) -> Option<&'a AccessibilityNode> { + if matches_all(node, role, name, value) { + return Some(node); + } + for child in &node.children { + if let Some(hit) = find_first_match(child, role, name, value) { + return Some(hit); + } + } + None +} + +fn matches_all( + node: &AccessibilityNode, + role: Option<&str>, + name: Option<&str>, + value: Option<&str>, +) -> bool { + if let Some(r) = role { + if !contains_ignore_case(&node.role, r) { + return false; + } + } + if let Some(n) = name { + match node.name.as_deref() { + Some(actual) if contains_ignore_case(actual, n) => {} + _ => return false, + } + } + if let Some(v) = value { + match node.value.as_deref() { + Some(actual) if contains_ignore_case(actual, v) => {} + _ => return false, + } + } + true +} + +fn contains_ignore_case(haystack: &str, needle: &str) -> bool { + haystack.to_lowercase().contains(&needle.to_lowercase()) +}