Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-08-16 12:03:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(ffi): observation primitives — ad_find / ad_get / ad_is (Unit 9)

Browse source 
Cheap single-element lookups without hand-walking the flat-tree result
of ad_get_tree. Closes the observation portion of R11 from PR #22 review.

New module tree crates/ffi/src/observation/ — one concern per file:

- walk.rs: pub(crate) find_first_match(node, role?, name?, value?) — DFS
  case-insensitive substring matcher shared by find.rs and is.rs.
  Extracted so both FFI entrypoints speak the same matching semantics
  and unit-testable in isolation.
- find.rs: ad_find(adapter, win, query, &handle). Walks the tree, picks
  the first match, resolves it to a NativeHandle via the existing
  PlatformAdapter::resolve_element path. Returns ErrElementNotFound
  when nothing matches. Caller owns the handle → must
  ad_free_handle(adapter, handle).
- get.rs: ad_get(adapter, handle, property, &str_out) dispatches on
  property name. Supported: "value" (via get_live_value), "bounds" (via
  get_element_bounds, formatted as JSON). Unknown property returns
  ErrInvalidArgs. Output string owned by caller → free with ad_free_string.
- is.rs: ad_is(adapter, win, query, property, &bool_out) looks up the
  element the same way find does but checks a named state against the
  node's states vec. Recognized: focused, enabled, selected, checked,
  expanded. Unknown property returns ErrInvalidArgs with a diagnostic
  listing the valid names.

New type in types/find_query.rs (AdFindQuery { role, name_substring,
value_substring }) already landed in Unit 5 and is re-exported here.

All entries go through trap_panic with debug_assert_main_thread.

53 tests pass. Clippy clean.
This commit is contained in:
Lahfir 2026-04-16 04:26:58 -07:00
parent 6d6917b939
commit 1fcb2e6936
7 changed files with 436 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
crates/ffi/include/agent_desktop.h
View file

@ -249,6 +249,12 @@ typedef struct AdNotificationInfo {
uint32_t action_count;
} AdNotificationInfo;
typedef struct AdFindQuery {
const char *role;
const char *name_substring;
const char *value_substring;
} AdFindQuery;
typedef struct AdScreenshotTarget {
AdScreenshotKind kind;
uint64_t screen_index;
@ -589,6 +595,78 @@ const struct AdNotificationInfo *ad_notification_list_get(const struct AdNotific
*/
void ad_notification_list_free(struct AdNotificationList *list);
/**
* Finds the first element in `win`'s accessibility tree matching the
* query and resolves it to an opaque `AdNativeHandle`. The caller owns
* the handle and must release it with `ad_free_handle(adapter, handle)`
* once done.
*
* Matching is DFS order, first hit wins. All query fields are optional
* (null = "don't care") and case-insensitive substring matches:
* - `role` against `AccessibilityNode.role`
* - `name_substring` against `AccessibilityNode.name`
* - `value_substring` against `AccessibilityNode.value`
*
* # Safety
* `adapter`, `win`, and `query` must be valid pointers. `out_handle`
* must be a valid writable `*mut AdNativeHandle`. On
* `AD_RESULT_ERR_ELEMENT_NOT_FOUND` the out-handle is zero-initialized.
*/
AdResult ad_find(const struct AdAdapter *adapter,
const struct AdWindowInfo *win,
const struct AdFindQuery *query,
struct AdNativeHandle *out_handle);
/**
* Reads a single property off a previously-resolved element handle.
*
* Supported properties:
* - `"value"` — live textual value (text fields, sliders, progress
* indicators). Null out-string when the element has no value.
* - `"bounds"` — JSON-encoded `{"x":..,"y":..,"width":..,"height":..}`.
* Null out-string when bounds are unavailable.
*
* The returned string must be freed with `ad_free_string`.
*
* # Safety
* `adapter` must be valid. `handle` must be a non-null `AdNativeHandle`.
* `property` must be a non-null UTF-8 C string. `out` must be a valid
* writable `*mut *mut c_char`; it is null-initialized on entry.
*/
AdResult ad_get(const struct AdAdapter *adapter,
const struct AdNativeHandle *handle,
const char *property,
char **out);
/**
* Checks whether a named boolean state is set on the first element
* matching `query` inside `win`'s accessibility tree. Intended for the
* common agent idiom `find → is(focused) → if yes, act`.
*
* Recognized property names (match the strings the platform adapter
* emits in `AccessibilityNode.states`):
*
* - `"focused"`
* - `"enabled"`
* - `"selected"`
* - `"checked"`
* - `"expanded"`
*
* Any other property name returns `AD_RESULT_ERR_INVALID_ARGS`. If no
* element matches the query, returns `AD_RESULT_ERR_ELEMENT_NOT_FOUND`
* and `*out` is untouched.
*
* # Safety
* All pointers must be valid. `property` must be a non-null UTF-8
* C string. `out` must be a valid writable `*mut bool`; it is set to
* `false` on entry.
*/
AdResult ad_is(const struct AdAdapter *adapter,
const struct AdWindowInfo *win,
const struct AdFindQuery *query,
const char *property,
bool *out);
/**
* Borrowed pointer to the image bytes; valid until the buffer is freed.
* Returns null if `buf` is null.

1
crates/ffi/src/lib.rs
View file

@ -42,6 +42,7 @@ pub(crate) mod ffi_try;
pub(crate) mod input;
pub(crate) mod main_thread;
pub(crate) mod notifications;
pub(crate) mod observation;
pub(crate) mod screenshot;
pub(crate) mod surfaces;
pub(crate) mod tree;

103
crates/ffi/src/observation/find.rs Normal file
View file

@ -0,0 +1,103 @@
use crate::convert::string::c_to_string;
use crate::error::{set_last_error, AdResult};
use crate::ffi_try::trap_panic;
use crate::observation::walk::find_first_match;
use crate::types::{AdFindQuery, AdNativeHandle, AdWindowInfo};
use crate::AdAdapter;
use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions};
use agent_desktop_core::refs::RefEntry;
/// Finds the first element in `win`'s accessibility tree matching the
/// query and resolves it to an opaque `AdNativeHandle`. The caller owns
/// the handle and must release it with `ad_free_handle(adapter, handle)`
/// once done.
///
/// Matching is DFS order, first hit wins. All query fields are optional
/// (null = "don't care") and case-insensitive substring matches:
/// - `role` against `AccessibilityNode.role`
/// - `name_substring` against `AccessibilityNode.name`
/// - `value_substring` against `AccessibilityNode.value`
///
/// # Safety
/// `adapter`, `win`, and `query` must be valid pointers. `out_handle`
/// must be a valid writable `*mut AdNativeHandle`. On
/// `AD_RESULT_ERR_ELEMENT_NOT_FOUND` the out-handle is zero-initialized.
#[no_mangle]
pub unsafe extern "C" fn ad_find(
adapter: *const AdAdapter,
win: *const AdWindowInfo,
query: *const AdFindQuery,
out_handle: *mut AdNativeHandle,
) -> AdResult {
trap_panic(|| unsafe {
crate::main_thread::debug_assert_main_thread();
(*out_handle).ptr = std::ptr::null();
let adapter = &*adapter;
let core_win = match crate::windows::ad_window_to_core(&*win) {
Ok(w) => w,
Err(e) => {
set_last_error(&e);
return crate::error::last_error_code();
}
};
let q = &*query;
let role_filter = c_to_string(q.role);
let name_filter = c_to_string(q.name_substring);
let value_filter = c_to_string(q.value_substring);
let tree = match adapter.inner.get_tree(
&core_win,
&TreeOptions {
max_depth: 50,
include_bounds: false,
interactive_only: false,
compact: false,
surface: SnapshotSurface::Window,
},
) {
Ok(t) => t,
Err(e) => {
set_last_error(&e);
return crate::error::last_error_code();
}
};
let matched = match find_first_match(
&tree,
role_filter.as_deref(),
name_filter.as_deref(),
value_filter.as_deref(),
) {
Some(n) => n,
None => {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::ElementNotFound,
"no element matched the find query",
));
return AdResult::ErrElementNotFound;
}
};
let ref_entry = RefEntry {
pid: core_win.pid,
role: matched.role.clone(),
name: matched.name.clone(),
value: None,
states: Vec::new(),
bounds: None,
bounds_hash: None,
available_actions: Vec::new(),
source_app: None,
};
match adapter.inner.resolve_element(&ref_entry) {
Ok(handle) => {
(*out_handle).ptr = handle.as_raw();
AdResult::Ok
}
Err(e) => {
set_last_error(&e);
crate::error::last_error_code()
}
}
})
}

82
crates/ffi/src/observation/get.rs Normal file
View file

@ -0,0 +1,82 @@
use crate::convert::string::{c_to_string, string_to_c_lossy};
use crate::error::{set_last_error, AdResult};
use crate::ffi_try::trap_panic;
use crate::types::AdNativeHandle;
use crate::AdAdapter;
use agent_desktop_core::adapter::NativeHandle;
use std::os::raw::c_char;
/// Reads a single property off a previously-resolved element handle.
///
/// Supported properties:
/// - `"value"` — live textual value (text fields, sliders, progress
/// indicators). Null out-string when the element has no value.
/// - `"bounds"` — JSON-encoded `{"x":..,"y":..,"width":..,"height":..}`.
/// Null out-string when bounds are unavailable.
///
/// The returned string must be freed with `ad_free_string`.
///
/// # Safety
/// `adapter` must be valid. `handle` must be a non-null `AdNativeHandle`.
/// `property` must be a non-null UTF-8 C string. `out` must be a valid
/// writable `*mut *mut c_char`; it is null-initialized on entry.
#[no_mangle]
pub unsafe extern "C" fn ad_get(
adapter: *const AdAdapter,
handle: *const AdNativeHandle,
property: *const c_char,
out: *mut *mut c_char,
) -> AdResult {
trap_panic(|| unsafe {
crate::main_thread::debug_assert_main_thread();
*out = std::ptr::null_mut();
let adapter = &*adapter;
let native = NativeHandle::from_ptr((*handle).ptr);
let prop = match c_to_string(property) {
Some(s) => s,
None => {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
"property is null or invalid UTF-8",
));
return AdResult::ErrInvalidArgs;
}
};
match prop.as_str() {
"value" => match adapter.inner.get_live_value(&native) {
Ok(Some(v)) => {
*out = string_to_c_lossy(&v);
AdResult::Ok
}
Ok(None) => AdResult::Ok,
Err(e) => {
set_last_error(&e);
crate::error::last_error_code()
}
},
"bounds" => match adapter.inner.get_element_bounds(&native) {
Ok(Some(r)) => {
let json = format!(
"{{\"x\":{},\"y\":{},\"width\":{},\"height\":{}}}",
r.x, r.y, r.width, r.height
);
*out = string_to_c_lossy(&json);
AdResult::Ok
}
Ok(None) => AdResult::Ok,
Err(e) => {
set_last_error(&e);
crate::error::last_error_code()
}
},
_ => {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
"unknown property — expected one of: value, bounds",
));
AdResult::ErrInvalidArgs
}
}
})
}

116
crates/ffi/src/observation/is.rs Normal file
View file

@ -0,0 +1,116 @@
use crate::convert::string::c_to_string;
use crate::error::{set_last_error, AdResult};
use crate::ffi_try::trap_panic;
use crate::types::{AdFindQuery, AdWindowInfo};
use crate::AdAdapter;
use agent_desktop_core::adapter::{SnapshotSurface, TreeOptions};
use agent_desktop_core::node::AccessibilityNode;
use std::os::raw::c_char;
/// Checks whether a named boolean state is set on the first element
/// matching `query` inside `win`'s accessibility tree. Intended for the
/// common agent idiom `find → is(focused) → if yes, act`.
///
/// Recognized property names (match the strings the platform adapter
/// emits in `AccessibilityNode.states`):
///
/// - `"focused"`
/// - `"enabled"`
/// - `"selected"`
/// - `"checked"`
/// - `"expanded"`
///
/// Any other property name returns `AD_RESULT_ERR_INVALID_ARGS`. If no
/// element matches the query, returns `AD_RESULT_ERR_ELEMENT_NOT_FOUND`
/// and `*out` is untouched.
///
/// # Safety
/// All pointers must be valid. `property` must be a non-null UTF-8
/// C string. `out` must be a valid writable `*mut bool`; it is set to
/// `false` on entry.
#[no_mangle]
pub unsafe extern "C" fn ad_is(
adapter: *const AdAdapter,
win: *const AdWindowInfo,
query: *const AdFindQuery,
property: *const c_char,
out: *mut bool,
) -> AdResult {
trap_panic(|| unsafe {
crate::main_thread::debug_assert_main_thread();
*out = false;
let adapter = &*adapter;
let core_win = match crate::windows::ad_window_to_core(&*win) {
Ok(w) => w,
Err(e) => {
set_last_error(&e);
return crate::error::last_error_code();
}
};
let q = &*query;
let role_filter = c_to_string(q.role);
let name_filter = c_to_string(q.name_substring);
let value_filter = c_to_string(q.value_substring);
let prop = match c_to_string(property) {
Some(s) => s,
None => {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
"property is null or invalid UTF-8",
));
return AdResult::ErrInvalidArgs;
}
};
if !is_known_property(&prop) {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::InvalidArgs,
"unknown property — expected one of: focused, enabled, selected, checked, expanded",
));
return AdResult::ErrInvalidArgs;
}
let tree = match adapter.inner.get_tree(
&core_win,
&TreeOptions {
max_depth: 50,
include_bounds: false,
interactive_only: false,
compact: false,
surface: SnapshotSurface::Window,
},
) {
Ok(t) => t,
Err(e) => {
set_last_error(&e);
return crate::error::last_error_code();
}
};
let matched = match crate::observation::walk::find_first_match(
&tree,
role_filter.as_deref(),
name_filter.as_deref(),
value_filter.as_deref(),
) {
Some(n) => n,
None => {
set_last_error(&agent_desktop_core::error::AdapterError::new(
agent_desktop_core::error::ErrorCode::ElementNotFound,
"no element matched the find query",
));
return AdResult::ErrElementNotFound;
}
};
*out = element_has_state(matched, &prop);
AdResult::Ok
})
}
fn is_known_property(p: &str) -> bool {
matches!(p, "focused" | "enabled" | "selected" | "checked" | "expanded")
}
fn element_has_state(node: &AccessibilityNode, prop: &str) -> bool {
node.states.iter().any(|s| s.eq_ignore_ascii_case(prop))
}

4
crates/ffi/src/observation/mod.rs Normal file
View file

@ -0,0 +1,4 @@
pub(crate) mod find;
pub(crate) mod get;
pub(crate) mod is;
pub(crate) mod walk;

52
crates/ffi/src/observation/walk.rs Normal file
View file

@ -0,0 +1,52 @@
use agent_desktop_core::node::AccessibilityNode;
/// Finds the first node in DFS order that matches every provided filter.
/// Filters are ANDed; a `None` filter means "don't care". Substring
/// matching is case-insensitive to tolerate platform-specific casing
/// (macOS AX strings vary between "AXButton" and "button").
pub(crate) fn find_first_match<'a>(
node: &'a AccessibilityNode,
role: Option<&str>,
name: Option<&str>,
value: Option<&str>,
) -> Option<&'a AccessibilityNode> {
if matches_all(node, role, name, value) {
return Some(node);
}
for child in &node.children {
if let Some(hit) = find_first_match(child, role, name, value) {
return Some(hit);
}
}
None
}
fn matches_all(
node: &AccessibilityNode,
role: Option<&str>,
name: Option<&str>,
value: Option<&str>,
) -> bool {
if let Some(r) = role {
if !contains_ignore_case(&node.role, r) {
return false;
}
}
if let Some(n) = name {
match node.name.as_deref() {
Some(actual) if contains_ignore_case(actual, n) => {}
_ => return false,
}
}
if let Some(v) = value {
match node.value.as_deref() {
Some(actual) if contains_ignore_case(actual, v) => {}
_ => return false,
}
}
true
}
fn contains_ignore_case(haystack: &str, needle: &str) -> bool {
haystack.to_lowercase().contains(&needle.to_lowercase())
}