mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-07-27 01:22:16 +00:00
fix: close reliability review gaps
This commit is contained in:
parent
282700c038
commit
d319daa745
25 changed files with 320 additions and 135 deletions
|
|
@ -65,7 +65,7 @@ The side-effect contract attached to an action request, controlling whether the
|
|||
### Headless Ref Action
|
||||
A ref-based action that uses semantic accessibility operations without implicit focus stealing, cursor movement, synthetic keyboard input, or pasteboard use. This is the default mode.
|
||||
|
||||
Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation; they fail closed with `POLICY_DENIED` rather than silently substituting physical input. The broader **headed** policy must be selected explicitly with `--headed`.
|
||||
Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation; they fail closed with structured actionability or policy errors rather than silently substituting physical input. The broader **headed** policy must be selected explicitly with `--headed`.
|
||||
|
||||
### Action Chain
|
||||
The ordered ladder of strategies a ref action walks to perform one intent — semantic accessibility actions first, then settable attributes, then policy-gated physical input — with each step verified against the element's observed state before it counts as success.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[non_exhaustive]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum Action {
|
||||
Click,
|
||||
|
|
|
|||
|
|
@ -243,7 +243,6 @@ mod tests {
|
|||
let body = std::fs::read_to_string(&path).unwrap();
|
||||
let event: serde_json::Value = serde_json::from_str(body.trim()).unwrap();
|
||||
assert_eq!(event["text"]["redacted"], true);
|
||||
assert_eq!(event["text"]["chars_bucket"], "1-8");
|
||||
assert_eq!(event["value"]["redacted"], true);
|
||||
assert_eq!(event["name"]["redacted"], true);
|
||||
assert_eq!(event["description"]["redacted"], true);
|
||||
|
|
|
|||
|
|
@ -143,12 +143,12 @@ fn sanitize_trace_value(value: Value) -> Value {
|
|||
}
|
||||
|
||||
fn is_sensitive_trace_key(key: &str) -> bool {
|
||||
let key = key.to_ascii_lowercase();
|
||||
[
|
||||
const SENSITIVE_KEYS: &[&str] = &[
|
||||
"text",
|
||||
"value",
|
||||
"expected",
|
||||
"name",
|
||||
"username",
|
||||
"description",
|
||||
"label",
|
||||
"query",
|
||||
|
|
@ -159,34 +159,49 @@ fn is_sensitive_trace_key(key: &str) -> bool {
|
|||
"url",
|
||||
"help",
|
||||
"placeholder",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| key.contains(needle))
|
||||
];
|
||||
trace_key_tokens(key)
|
||||
.iter()
|
||||
.any(|part| SENSITIVE_KEYS.contains(&part.as_str()))
|
||||
}
|
||||
|
||||
fn trace_key_tokens(key: &str) -> Vec<String> {
|
||||
let mut tokens = Vec::new();
|
||||
let mut current = String::new();
|
||||
let mut previous_was_lower_or_digit = false;
|
||||
|
||||
for ch in key.chars() {
|
||||
if !ch.is_ascii_alphanumeric() {
|
||||
push_trace_key_token(&mut tokens, &mut current);
|
||||
previous_was_lower_or_digit = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ch.is_ascii_uppercase() && previous_was_lower_or_digit {
|
||||
push_trace_key_token(&mut tokens, &mut current);
|
||||
}
|
||||
|
||||
current.push(ch.to_ascii_lowercase());
|
||||
previous_was_lower_or_digit = ch.is_ascii_lowercase() || ch.is_ascii_digit();
|
||||
}
|
||||
|
||||
push_trace_key_token(&mut tokens, &mut current);
|
||||
tokens
|
||||
}
|
||||
|
||||
fn push_trace_key_token(tokens: &mut Vec<String>, current: &mut String) {
|
||||
if !current.is_empty() {
|
||||
tokens.push(std::mem::take(current));
|
||||
}
|
||||
}
|
||||
|
||||
fn redacted_value(value: Value) -> Value {
|
||||
match value {
|
||||
Value::String(text) => json!({
|
||||
"redacted": true,
|
||||
"chars_bucket": char_count_bucket(text.chars().count())
|
||||
}),
|
||||
Value::Array(items) => json!({ "redacted": true, "items": items.len() }),
|
||||
Value::Object(map) => json!({ "redacted": true, "keys": map.len() }),
|
||||
Value::Null => Value::Null,
|
||||
_ => json!({ "redacted": true }),
|
||||
}
|
||||
}
|
||||
|
||||
fn char_count_bucket(count: usize) -> &'static str {
|
||||
match count {
|
||||
0 => "0",
|
||||
1..=8 => "1-8",
|
||||
9..=32 => "9-32",
|
||||
33..=128 => "33-128",
|
||||
_ => "129+",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -237,20 +252,25 @@ mod tests {
|
|||
"action": {
|
||||
"typed_text": ["secret", "another"],
|
||||
"api_token": {"kind": "bearer"},
|
||||
"typedText": "secret",
|
||||
"apiToken": "secret",
|
||||
"targetLabel": "secret",
|
||||
"userName": "secret",
|
||||
"filename": "report.txt",
|
||||
"password": null,
|
||||
"counter": 3
|
||||
}
|
||||
}));
|
||||
|
||||
assert_eq!(value["action"]["typed_text"]["redacted"], true);
|
||||
assert_eq!(value["action"]["typed_text"]["items"], 2);
|
||||
assert_eq!(value["action"]["api_token"]["redacted"], true);
|
||||
assert_eq!(value["action"]["api_token"]["keys"], 1);
|
||||
assert_eq!(value["action"]["typedText"]["redacted"], true);
|
||||
assert_eq!(value["action"]["apiToken"]["redacted"], true);
|
||||
assert_eq!(value["action"]["targetLabel"]["redacted"], true);
|
||||
assert_eq!(value["action"]["userName"]["redacted"], true);
|
||||
assert_eq!(value["action"]["filename"], "report.txt");
|
||||
assert!(value["action"]["password"].is_null());
|
||||
assert_eq!(value["action"]["counter"], 3);
|
||||
assert_eq!(char_count_bucket(0), "0");
|
||||
assert_eq!(char_count_bucket(8), "1-8");
|
||||
assert_eq!(char_count_bucket(65), "33-128");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -206,6 +206,7 @@ typedef struct AdPoint {
|
|||
* do not re-report): 0.x grew this struct 40 -> 48 bytes by adding
|
||||
* `drop_delay_ms` (which also grew the embedding `AdAction` — see
|
||||
* `AD_ACTION_SIZE`), `AdRefEntry` grew to 192 bytes (`AD_REF_ENTRY_SIZE`),
|
||||
* `AdActionStep` was added as a public 16-byte struct (`AD_ACTION_STEP_SIZE`),
|
||||
* `AdActionResult` grew to 40 bytes (`AD_ACTION_RESULT_SIZE`) to expose action
|
||||
* steps, and `AD_POLICY_KIND_PHYSICAL` was renamed to `AD_POLICY_KIND_HEADED`
|
||||
* with a stable discriminant and intentionally no compatibility alias.
|
||||
|
|
@ -278,11 +279,23 @@ typedef struct AdActionStep {
|
|||
const char *outcome;
|
||||
} AdActionStep;
|
||||
|
||||
#define AD_ACTION_STEP_SIZE (sizeof(AdActionStep))
|
||||
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
_Static_assert(sizeof(AdActionStep) == 16, "AdActionStep ABI size changed");
|
||||
_Static_assert(_Alignof(AdActionStep) == 8, "AdActionStep ABI alignment changed");
|
||||
_Static_assert(offsetof(AdActionStep, label) == 0, "AdActionStep.label offset changed");
|
||||
_Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed");
|
||||
#endif
|
||||
|
||||
uintptr_t ad_action_step_size(void);
|
||||
|
||||
/*
|
||||
* Result for action calls. `steps` contains `step_count` activation-chain
|
||||
* entries (`label`, `outcome`) when the adapter recorded how the action was
|
||||
* attempted. The array and all nested strings are owned by the result and are
|
||||
* released by `ad_free_action_result`.
|
||||
* Result for action calls. `post_state->states` contains `state_count` strings
|
||||
* when present. `steps` contains `step_count` activation-chain entries
|
||||
* (`label`, `outcome`) when the adapter recorded how the action was attempted.
|
||||
* Non-empty owned arrays are additionally null-sentinel terminated by Rust; C
|
||||
* callers should use the counts for reads and release the unmodified result via
|
||||
* `ad_free_action_result`.
|
||||
*/
|
||||
typedef struct AdActionResult {
|
||||
const char *action;
|
||||
|
|
@ -294,10 +307,6 @@ typedef struct AdActionResult {
|
|||
|
||||
#define AD_ACTION_RESULT_SIZE (sizeof(AdActionResult))
|
||||
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
_Static_assert(sizeof(AdActionStep) == 16, "AdActionStep ABI size changed");
|
||||
_Static_assert(_Alignof(AdActionStep) == 8, "AdActionStep ABI alignment changed");
|
||||
_Static_assert(offsetof(AdActionStep, label) == 0, "AdActionStep.label offset changed");
|
||||
_Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed");
|
||||
_Static_assert(sizeof(AdActionResult) == 40, "AdActionResult ABI size changed");
|
||||
_Static_assert(_Alignof(AdActionResult) == 8, "AdActionResult ABI alignment changed");
|
||||
_Static_assert(offsetof(AdActionResult, action) == 0, "AdActionResult.action offset changed");
|
||||
|
|
|
|||
|
|
@ -4,9 +4,6 @@ use agent_desktop_core::action_result::ActionResult as CoreActionResult;
|
|||
use agent_desktop_core::action_step_outcome::ActionStepOutcome;
|
||||
use std::ptr;
|
||||
|
||||
const MAX_STATE_STRINGS_TO_FREE: usize = 1024;
|
||||
const MAX_STEPS_TO_FREE: usize = 1024;
|
||||
|
||||
pub(crate) fn action_result_to_c(r: &CoreActionResult) -> AdActionResult {
|
||||
let action = string_to_c_lossy(&r.action);
|
||||
let post_state = match &r.post_state {
|
||||
|
|
@ -92,10 +89,7 @@ fn action_steps_to_c(r: &CoreActionResult) -> *mut AdActionStep {
|
|||
outcome: string_to_c_lossy(step_outcome_name(&step.outcome)),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
steps.push(AdActionStep {
|
||||
label: ptr::null(),
|
||||
outcome: ptr::null(),
|
||||
});
|
||||
steps.push(step_sentinel());
|
||||
let mut boxed = steps.into_boxed_slice();
|
||||
let raw = boxed.as_mut_ptr();
|
||||
std::mem::forget(boxed);
|
||||
|
|
@ -110,13 +104,22 @@ fn step_outcome_name(outcome: &ActionStepOutcome) -> &'static str {
|
|||
}
|
||||
}
|
||||
|
||||
fn step_sentinel() -> AdActionStep {
|
||||
AdActionStep {
|
||||
label: ptr::null(),
|
||||
outcome: ptr::null(),
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn free_state_array(states: *mut *mut std::os::raw::c_char) {
|
||||
unsafe {
|
||||
let mut len = 0;
|
||||
while len < MAX_STATE_STRINGS_TO_FREE && !(*states.add(len)).is_null() {
|
||||
free_c_string(*states.add(len));
|
||||
while !(*states.add(len)).is_null() {
|
||||
len += 1;
|
||||
}
|
||||
for index in 0..len {
|
||||
free_c_string(*states.add(index));
|
||||
}
|
||||
drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut(
|
||||
states,
|
||||
len + 1,
|
||||
|
|
@ -127,14 +130,13 @@ unsafe fn free_state_array(states: *mut *mut std::os::raw::c_char) {
|
|||
unsafe fn free_step_array(steps: *mut AdActionStep) {
|
||||
unsafe {
|
||||
let mut len = 0;
|
||||
while len < MAX_STEPS_TO_FREE {
|
||||
let step = &mut *steps.add(len);
|
||||
if step.label.is_null() && step.outcome.is_null() {
|
||||
break;
|
||||
}
|
||||
while !step_is_sentinel(&*steps.add(len)) {
|
||||
len += 1;
|
||||
}
|
||||
for index in 0..len {
|
||||
let step = &mut *steps.add(index);
|
||||
free_c_string(step.label as *mut _);
|
||||
free_c_string(step.outcome as *mut _);
|
||||
len += 1;
|
||||
}
|
||||
drop(Box::from_raw(std::ptr::slice_from_raw_parts_mut(
|
||||
steps,
|
||||
|
|
@ -143,6 +145,10 @@ unsafe fn free_step_array(steps: *mut AdActionStep) {
|
|||
}
|
||||
}
|
||||
|
||||
fn step_is_sentinel(step: &AdActionStep) -> bool {
|
||||
step.label.is_null() && step.outcome.is_null()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -177,20 +183,29 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn free_action_result_ignores_mutated_state_count() {
|
||||
fn free_action_result_ignores_mutated_counts() {
|
||||
let post_state = Box::new(AdElementState {
|
||||
role: crate::convert::string::string_to_c_lossy("button"),
|
||||
states: state_array(&["focused"]),
|
||||
state_count: u32::MAX,
|
||||
value: ptr::null(),
|
||||
});
|
||||
let mut steps = vec![
|
||||
AdActionStep {
|
||||
label: crate::convert::string::string_to_c_lossy("AXPress"),
|
||||
outcome: crate::convert::string::string_to_c_lossy("succeeded"),
|
||||
},
|
||||
step_sentinel(),
|
||||
]
|
||||
.into_boxed_slice();
|
||||
let mut c_result = AdActionResult {
|
||||
action: crate::convert::string::string_to_c_lossy("click"),
|
||||
ref_id: ptr::null(),
|
||||
post_state: Box::into_raw(post_state),
|
||||
steps: ptr::null_mut(),
|
||||
steps: steps.as_mut_ptr(),
|
||||
step_count: u32::MAX,
|
||||
};
|
||||
std::mem::forget(steps);
|
||||
unsafe { ad_free_action_result(&mut c_result) };
|
||||
|
||||
assert!(c_result.post_state.is_null());
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
use crate::AdAdapter;
|
||||
use crate::convert::string::c_to_string;
|
||||
use crate::error::{AdResult, set_last_error};
|
||||
use crate::ffi_try::trap_panic;
|
||||
use std::os::raw::c_char;
|
||||
|
|
@ -26,17 +25,14 @@ pub unsafe extern "C" fn ad_close_app(
|
|||
return rc;
|
||||
}
|
||||
crate::pointer_guard::guard_non_null!(adapter, c"adapter is null");
|
||||
let adapter = &*adapter;
|
||||
let id_str = match c_to_string(id) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
set_last_error(&agent_desktop_core::error::AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
"app id is null or invalid UTF-8",
|
||||
));
|
||||
return AdResult::ErrInvalidArgs;
|
||||
let id_str = match super::decode_app_id(id) {
|
||||
Ok(id) => id,
|
||||
Err(err) => {
|
||||
set_last_error(&err);
|
||||
return crate::error::last_error_code();
|
||||
}
|
||||
};
|
||||
let adapter = &*adapter;
|
||||
|
||||
match adapter.inner.close_app(&id_str, force) {
|
||||
Ok(()) => AdResult::Ok,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
use crate::AdAdapter;
|
||||
use crate::convert::string::c_to_string;
|
||||
use crate::convert::window::window_info_to_c;
|
||||
use crate::error::{AdResult, set_last_error};
|
||||
use crate::ffi_try::trap_panic;
|
||||
|
|
@ -33,14 +32,11 @@ pub unsafe extern "C" fn ad_launch_app(
|
|||
return rc;
|
||||
}
|
||||
crate::pointer_guard::guard_non_null!(adapter, c"adapter is null");
|
||||
let id_str = match c_to_string(id) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
set_last_error(&agent_desktop_core::error::AdapterError::new(
|
||||
agent_desktop_core::error::ErrorCode::InvalidArgs,
|
||||
"app id is null or invalid UTF-8",
|
||||
));
|
||||
return AdResult::ErrInvalidArgs;
|
||||
let id_str = match super::decode_app_id(id) {
|
||||
Ok(id) => id,
|
||||
Err(err) => {
|
||||
set_last_error(&err);
|
||||
return crate::error::last_error_code();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,51 @@
|
|||
pub(crate) mod close;
|
||||
pub(crate) mod launch;
|
||||
pub(crate) mod list;
|
||||
|
||||
use agent_desktop_core::error::AdapterError;
|
||||
use std::os::raw::c_char;
|
||||
|
||||
fn decode_app_id(id: *const c_char) -> Result<String, AdapterError> {
|
||||
crate::convert::string::required_adapter_string(id, "app id")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::convert::string::{MAX_C_STRING_BYTES, string_to_c};
|
||||
|
||||
#[test]
|
||||
fn app_id_rejects_null() {
|
||||
let err = decode_app_id(std::ptr::null()).unwrap_err();
|
||||
|
||||
assert_eq!(err.message, "app id is null");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_id_rejects_invalid_utf8() {
|
||||
let bad = [0xC3_u8, 0];
|
||||
let err = decode_app_id(bad.as_ptr().cast()).unwrap_err();
|
||||
|
||||
assert_eq!(err.message, "app id is not valid UTF-8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_id_rejects_overlong_input() {
|
||||
let bytes = vec![b'a'; MAX_C_STRING_BYTES + 1];
|
||||
let err = decode_app_id(bytes.as_ptr().cast()).unwrap_err();
|
||||
|
||||
assert!(
|
||||
err.message
|
||||
.starts_with("app id exceeds AD_MAX_STRING_BYTES")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn app_id_accepts_valid_utf8() {
|
||||
let c = string_to_c("TextEdit");
|
||||
let decoded = decode_app_id(c).unwrap();
|
||||
|
||||
assert_eq!(decoded, "TextEdit");
|
||||
unsafe { crate::convert::string::free_c_string(c) };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,4 +6,11 @@ pub struct AdActionStep {
|
|||
pub outcome: *const c_char,
|
||||
}
|
||||
|
||||
const _: () = assert!(std::mem::size_of::<AdActionStep>() == 16);
|
||||
pub const AD_ACTION_STEP_SIZE: usize = 16;
|
||||
|
||||
const _: () = assert!(std::mem::size_of::<AdActionStep>() == AD_ACTION_STEP_SIZE);
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn ad_action_step_size() -> usize {
|
||||
std::mem::size_of::<AdActionStep>()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
pub mod action;
|
||||
pub mod action_kind;
|
||||
pub mod action_result;
|
||||
pub(crate) mod action_step;
|
||||
pub mod action_step;
|
||||
pub mod app_info;
|
||||
pub mod app_list;
|
||||
pub mod direction;
|
||||
|
|
@ -40,6 +40,7 @@ pub mod window_op_kind;
|
|||
pub use action::AdAction;
|
||||
pub use action_kind::AdActionKind;
|
||||
pub use action_result::AdActionResult;
|
||||
pub use action_step::AdActionStep;
|
||||
pub use app_info::AdAppInfo;
|
||||
pub use app_list::AdAppList;
|
||||
pub use direction::AdDirection;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
mod common;
|
||||
|
||||
use common::{
|
||||
AdActionResult, AdNativeHandle, AdPolicyKind, AdResult, ad_execute_action,
|
||||
ad_execute_action_with_policy, ad_execute_ref_action_with_policy, default_action,
|
||||
default_ref_entry, with_adapter,
|
||||
AdActionResult, AdActionStep, AdNativeHandle, AdPolicyKind, AdResult, ad_execute_action,
|
||||
ad_execute_action_with_policy, ad_execute_ref_action_with_policy, ad_free_action_result,
|
||||
default_action, default_ref_entry, with_adapter,
|
||||
};
|
||||
use std::ffi::CString;
|
||||
|
||||
#[test]
|
||||
fn enum_fuzz_invalid_discriminant_rejected() {
|
||||
|
|
@ -110,3 +111,36 @@ fn execute_action_policy_requires_main_thread_on_macos() {
|
|||
#[cfg(not(target_os = "macos"))]
|
||||
assert_eq!(rc, AdResult::ErrInvalidArgs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn free_action_result_releases_non_empty_steps_array() {
|
||||
let mut steps = vec![
|
||||
AdActionStep {
|
||||
label: CString::new("AXScrollToVisible").unwrap().into_raw(),
|
||||
outcome: CString::new("attempted").unwrap().into_raw(),
|
||||
},
|
||||
AdActionStep {
|
||||
label: CString::new("AXPress").unwrap().into_raw(),
|
||||
outcome: CString::new("succeeded").unwrap().into_raw(),
|
||||
},
|
||||
AdActionStep {
|
||||
label: std::ptr::null(),
|
||||
outcome: std::ptr::null(),
|
||||
},
|
||||
]
|
||||
.into_boxed_slice();
|
||||
let mut result = AdActionResult {
|
||||
action: CString::new("click").unwrap().into_raw(),
|
||||
ref_id: std::ptr::null(),
|
||||
post_state: std::ptr::null_mut(),
|
||||
steps: steps.as_mut_ptr(),
|
||||
step_count: 2,
|
||||
};
|
||||
std::mem::forget(steps);
|
||||
|
||||
unsafe { ad_free_action_result(&mut result) };
|
||||
|
||||
assert!(result.action.is_null());
|
||||
assert!(result.steps.is_null());
|
||||
assert_eq!(result.step_count, 0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,7 +41,18 @@ fn action_result_layout_is_guarded_for_c_consumers() {
|
|||
unsafe { common::ad_action_result_size() },
|
||||
agent_desktop_ffi::types::action_result::AD_ACTION_RESULT_SIZE
|
||||
);
|
||||
assert_eq!(size_of::<AdActionStep>(), 16);
|
||||
assert_eq!(
|
||||
agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE,
|
||||
16
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe { common::ad_action_step_size() },
|
||||
agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE
|
||||
);
|
||||
assert_eq!(
|
||||
size_of::<AdActionStep>(),
|
||||
agent_desktop_ffi::types::action_step::AD_ACTION_STEP_SIZE
|
||||
);
|
||||
assert_eq!(align_of::<AdActionStep>(), align_of::<usize>());
|
||||
assert_eq!(size_of::<AdActionResult>(), 40);
|
||||
assert_eq!(align_of::<AdActionResult>(), align_of::<usize>());
|
||||
|
|
|
|||
|
|
@ -62,11 +62,13 @@ int main(void) {
|
|||
(void)AD_RESULT_OK;
|
||||
_Static_assert(AD_ACTION_SIZE == sizeof(AdAction), "AdAction size macro drifted");
|
||||
_Static_assert(AD_DRAG_PARAMS_SIZE == sizeof(AdDragParams), "AdDragParams size macro drifted");
|
||||
_Static_assert(AD_ACTION_STEP_SIZE == sizeof(AdActionStep), "AdActionStep size macro drifted");
|
||||
_Static_assert(AD_ACTION_RESULT_SIZE == sizeof(AdActionResult), "AdActionResult size macro drifted");
|
||||
_Static_assert(offsetof(AdActionResult, steps) == 24, "AdActionResult.steps offset changed");
|
||||
_Static_assert(offsetof(AdActionResult, step_count) == 32, "AdActionResult.step_count offset changed");
|
||||
_Static_assert(offsetof(AdActionStep, outcome) == 8, "AdActionStep.outcome offset changed");
|
||||
_Static_assert(AD_ELEMENT_STATE_SIZE == sizeof(AdElementState), "AdElementState size macro drifted");
|
||||
(void)ad_action_step_size;
|
||||
(void)ad_ref_entry_size;
|
||||
(void)ad_last_error_details;
|
||||
_Static_assert(AD_REF_ENTRY_SIZE == sizeof(AdRefEntry), "AdRefEntry size macro drifted");
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ pub use std::os::raw::c_char;
|
|||
unsafe extern "C" {
|
||||
pub fn ad_ref_entry_size() -> usize;
|
||||
pub fn ad_action_size() -> usize;
|
||||
pub fn ad_action_step_size() -> usize;
|
||||
pub fn ad_action_result_size() -> usize;
|
||||
pub fn ad_element_state_size() -> usize;
|
||||
|
||||
|
|
@ -65,6 +66,7 @@ unsafe extern "C" {
|
|||
policy: i32,
|
||||
out: *mut AdActionResult,
|
||||
) -> AdResult;
|
||||
pub fn ad_free_action_result(result: *mut AdActionResult);
|
||||
|
||||
pub fn ad_find(
|
||||
adapter: *const AdAdapter,
|
||||
|
|
|
|||
|
|
@ -123,17 +123,17 @@ mod imp {
|
|||
}
|
||||
|
||||
fn element_text_contains(el: &AXElement, needle: &str, depth: usize) -> bool {
|
||||
if depth > 8 {
|
||||
return false;
|
||||
}
|
||||
find_descendant_value(el, depth, &|candidate| {
|
||||
element_own_text_contains(candidate, needle).then_some(())
|
||||
})
|
||||
.is_some()
|
||||
}
|
||||
|
||||
fn element_own_text_contains(el: &AXElement, needle: &str) -> bool {
|
||||
["AXTitle", "AXDescription", "AXValue", "AXHelp"]
|
||||
.into_iter()
|
||||
.filter_map(|attr| crate::tree::copy_string_attr(el, attr))
|
||||
.any(|value| value.contains(needle))
|
||||
|| crate::tree::copy_ax_array(el, "AXChildren")
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.any(|child| element_text_contains(child, needle, depth + 1))
|
||||
}
|
||||
|
||||
fn select_containing_item(el: &AXElement) -> Result<bool, AdapterError> {
|
||||
|
|
@ -154,9 +154,10 @@ mod imp {
|
|||
}
|
||||
|
||||
fn selected_items_menu_button(root: &AXElement) -> Option<AXElement> {
|
||||
find_descendant(root, 0, &|el| {
|
||||
crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXMenuButton")
|
||||
&& is_selected_items_control(el)
|
||||
find_descendant_value(root, 0, &|el| {
|
||||
(crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXMenuButton")
|
||||
&& is_selected_items_control(el))
|
||||
.then(|| el.clone())
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -181,19 +182,19 @@ mod imp {
|
|||
value.contains("selected item")
|
||||
}
|
||||
|
||||
fn find_descendant(
|
||||
fn find_descendant_value<T>(
|
||||
el: &AXElement,
|
||||
depth: usize,
|
||||
predicate: &impl Fn(&AXElement) -> bool,
|
||||
) -> Option<AXElement> {
|
||||
mapper: &impl Fn(&AXElement) -> Option<T>,
|
||||
) -> Option<T> {
|
||||
if depth > 8 {
|
||||
return None;
|
||||
}
|
||||
if predicate(el) {
|
||||
return Some(el.clone());
|
||||
if let Some(value) = mapper(el) {
|
||||
return Some(value);
|
||||
}
|
||||
for child in crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default() {
|
||||
if let Some(found) = find_descendant(&child, depth + 1, predicate) {
|
||||
if let Some(found) = find_descendant_value(&child, depth + 1, mapper) {
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
|
|
@ -204,25 +205,26 @@ mod imp {
|
|||
#[cfg(not(target_os = "macos"))]
|
||||
mod imp {
|
||||
use crate::tree::AXElement;
|
||||
use agent_desktop_core::error::AdapterError;
|
||||
|
||||
pub fn show_menu(_el: &AXElement) -> bool {
|
||||
false
|
||||
pub(crate) fn show_menu(_el: &AXElement) -> Result<bool, AdapterError> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn show_menu_on_ancestors(_el: &AXElement) -> bool {
|
||||
false
|
||||
pub(crate) fn show_menu_on_ancestors(_el: &AXElement) -> Result<bool, AdapterError> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn show_menu_on_children(_el: &AXElement) -> bool {
|
||||
false
|
||||
pub(crate) fn show_menu_on_children(_el: &AXElement) -> Result<bool, AdapterError> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn select_then_show_menu(_el: &AXElement) -> bool {
|
||||
false
|
||||
pub(crate) fn select_then_show_menu(_el: &AXElement) -> Result<bool, AdapterError> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn select_then_selected_items_menu(_el: &AXElement) -> bool {
|
||||
false
|
||||
pub(crate) fn select_then_selected_items_menu(_el: &AXElement) -> Result<bool, AdapterError> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -238,10 +238,6 @@ mod imp {
|
|||
)
|
||||
.with_suggestion("Use the top-level command (e.g. 'hover', 'drag', 'key-down') instead of targeting an element."));
|
||||
}
|
||||
|
||||
_ => {
|
||||
return Err(AdapterError::not_supported(label));
|
||||
}
|
||||
}
|
||||
|
||||
let mut result = ActionResult::new(label).with_steps(steps);
|
||||
|
|
|
|||
|
|
@ -14,7 +14,18 @@ pub(crate) fn read_post_state(
|
|||
| Action::Clear
|
||||
| Action::Expand
|
||||
| Action::Collapse => 0,
|
||||
_ => return None,
|
||||
Action::DoubleClick
|
||||
| Action::RightClick
|
||||
| Action::TripleClick
|
||||
| Action::SetFocus
|
||||
| Action::Select(_)
|
||||
| Action::Scroll(_, _)
|
||||
| Action::ScrollTo
|
||||
| Action::PressKey(_)
|
||||
| Action::KeyDown(_)
|
||||
| Action::KeyUp(_)
|
||||
| Action::Hover
|
||||
| Action::Drag(_) => return None,
|
||||
};
|
||||
if delay_ms > 0 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
use agent_desktop_core::error::AdapterError;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const KILL_CONFIRM_FLOOR: Duration = Duration::from_millis(500);
|
||||
|
||||
pub(crate) fn terminate_app(id: &str, pids: &[i32], timeout: Duration) -> Result<(), AdapterError> {
|
||||
let start = Instant::now();
|
||||
let mut failures = signal_failures(pids, Signal::Term);
|
||||
|
|
@ -10,8 +12,7 @@ pub(crate) fn terminate_app(id: &str, pids: &[i32], timeout: Duration) -> Result
|
|||
}
|
||||
|
||||
failures.extend(signal_failures(&remaining, Signal::Kill));
|
||||
let still_running =
|
||||
remaining_pids_after_wait(&remaining, timeout.saturating_sub(start.elapsed()));
|
||||
let still_running = remaining_pids_after_wait(&remaining, kill_confirm_budget(timeout, start));
|
||||
if still_running.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -27,6 +28,15 @@ pub(crate) fn terminate_app(id: &str, pids: &[i32], timeout: Duration) -> Result
|
|||
Err(err)
|
||||
}
|
||||
|
||||
fn kill_confirm_budget(timeout: Duration, start: Instant) -> Duration {
|
||||
if timeout.is_zero() {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
timeout
|
||||
.saturating_sub(start.elapsed())
|
||||
.max(KILL_CONFIRM_FLOOR)
|
||||
}
|
||||
|
||||
fn signal_failures(pids: &[i32], signal: Signal) -> Vec<String> {
|
||||
collect_signal_failures(pids, signal, signal_result)
|
||||
}
|
||||
|
|
@ -216,4 +226,22 @@ mod tests {
|
|||
assert!(first.has_exited());
|
||||
assert!(second.has_exited());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kill_confirm_budget_keeps_a_small_floor_after_term_timeout() {
|
||||
let start = Instant::now() - Duration::from_secs(1);
|
||||
|
||||
assert_eq!(
|
||||
kill_confirm_budget(Duration::from_millis(10), start),
|
||||
KILL_CONFIRM_FLOOR
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kill_confirm_budget_preserves_explicit_zero_timeout() {
|
||||
assert_eq!(
|
||||
kill_confirm_budget(Duration::ZERO, Instant::now()),
|
||||
Duration::ZERO
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ pub trait PlatformAdapter: Send + Sync {
|
|||
|
||||
### Key Supporting Types
|
||||
|
||||
- `Action` — `#[non_exhaustive]` enum. Current variants: Click, DoubleClick, TripleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Check, Uncheck, Scroll(Direction, Amount), ScrollTo, PressKey(KeyCombo), KeyDown(KeyCombo), KeyUp(KeyCombo), TypeText(String), Clear, Hover, Drag(DragParams)
|
||||
- `Action` — closed core enum whose platform dispatch arms must stay exhaustive. Current variants: Click, DoubleClick, TripleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Check, Uncheck, Scroll(Direction, Amount), ScrollTo, PressKey(KeyCombo), KeyDown(KeyCombo), KeyUp(KeyCombo), TypeText(String), Clear, Hover, Drag(DragParams)
|
||||
- `ActionRequest` — `{ action, policy }`; default policy forbids focus stealing and cursor movement
|
||||
- `PermissionReport` — `{ accessibility, screen_recording, automation }`, each `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`
|
||||
- `MouseEvent`, `DragParams`, `KeyCombo` — dedicated types (not unified under an `InputEvent` enum)
|
||||
|
|
@ -244,7 +244,7 @@ crates/macos/src/
|
|||
│ ├── extras.rs # select_value helpers
|
||||
│ ├── post_state.rs # Post-action state reads
|
||||
│ ├── scroll.rs # scroll semantics and explicit physical policy paths
|
||||
│ └── type_text.rs # headless AX text insertion and physical typing
|
||||
│ └── type_text.rs # focus-fallback text insertion and physical typing
|
||||
├── input/
|
||||
│ ├── mod.rs # re-exports
|
||||
│ ├── keyboard.rs # CGEventCreateKeyboardEvent, key synthesis, text typing
|
||||
|
|
@ -281,7 +281,7 @@ crates/macos/src/
|
|||
- Ref actions take `ActionRequest`, not bare `Action`
|
||||
- Default policy forbids focus stealing and cursor movement
|
||||
- Click/right-click/scroll chains run semantic AX steps first and return structured errors instead of silently using physical/headed paths
|
||||
- Type in headless mode mutates settable AX text values; physical keyboard typing is policy-gated
|
||||
- Type uses the focus-fallback policy floor; SetValue/Clear are the pure headless AX value-mutation paths
|
||||
- SetValue/Clear: `AXUIElementSetAttributeValue(kAXValueAttribute, value)`
|
||||
- SetFocus/Press/Hover/Drag/Mouse: explicit focus/cursor/physical commands
|
||||
- Keyboard/Mouse: `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` via CoreGraphics
|
||||
|
|
@ -648,7 +648,7 @@ These items are tracked under P2-O16 below: registry migration via `build.rs` fi
|
|||
|
||||
Phase 2 brings agent-desktop to Windows. It is also the phase that closes the cross-platform feature-parity gaps surfaced after the v0.1.13 FFI ship — shipping Windows meaningfully requires new core abstractions (stable identifiers, event subscriptions, text-range primitives, shell surfaces, and Windows-specific tray/taskbar affordances) that Windows UIA exposes natively and the macOS adapter currently does not surface. Every new trait method added here is implemented on both platforms in the same PR pair when there is a real cross-platform analogue. True Windows shell concepts return `PLATFORM_NOT_SUPPORTED` on other adapters through the same core command path, never through side-channel code. Linux (Phase 3) mirrors the portable parts against AT-SPI2.
|
||||
|
||||
Core engine, CLI parser, JSON contract invariants, and command-registration pattern are preserved. What Phase 2 legitimately changes: `AccessibilityNode` field set, `Action` enum variants, `ErrorCode` variants, `PlatformAdapter` trait size. Every change is additive (`#[non_exhaustive]` already guards the enums) and every macOS backfill lands atomically with the Windows implementation so the two platforms never drift.
|
||||
Core engine, CLI parser, JSON contract invariants, and command-registration pattern are preserved. What Phase 2 legitimately changes: `AccessibilityNode` field set, `Action` enum variants, `ErrorCode` variants, `PlatformAdapter` trait size. Every new `Action` variant must update core actionability, capability maps, platform dispatch, CLI/FFI conversion, and contract tests in the same change; exhaustive compiler checks are the guard against adapter drift. Every macOS backfill lands atomically with the Windows implementation so the two platforms never drift.
|
||||
|
||||
Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, every new command added in Phase 2 (`watch`, `text select-range`, `text get-selection`, `text insert-at-caret`, etc.) lives in **exactly one file** under `crates/core/src/commands/` and is wired through the shared typed command path. If Phase 2 adds codegen, it uses deterministic `build.rs` filesystem enumeration, not linker registries. The per-platform work is the three `PlatformAdapter` method implementations (one each in `crates/macos/`, `crates/windows/`, `crates/linux/`) — nothing repeats across transports.
|
||||
|
||||
|
|
@ -673,7 +673,7 @@ Cross-platform core extensions (new, landed alongside Windows):
|
|||
| ID | Objective | Metric |
|
||||
|----|-----------|--------|
|
||||
| P2-O8 | `AccessibilityNode` stable-selector fields | Nodes may carry `identifier`, `subrole`, `role_description`, `placeholder`, `dom_id`, `dom_classes` (all `Option<String>` / `Vec<String>` with `skip_serializing_if`). Populated where the platform/app exposes stable selectors: Windows UIA `AutomationId` / `LocalizedControlType` / `HelpText`; macOS `kAXIdentifierAttribute` / `kAXSubroleAttribute` / `kAXRoleDescriptionAttribute` / `kAXPlaceholderValueAttribute` / `kAXDOMIdentifierAttribute` / `kAXDOMClassListAttribute`. Resolver prefers stable selectors when present and falls back to the existing fingerprint; tests require known controls with explicit IDs to preserve them across re-drills, not every real-app node |
|
||||
| P2-O9 | `Action` enum expansion for 2026 agent workloads | New variants: `LongPress { duration_ms }`, `ForceClick`, `ShowMenu`, `DeliverFiles(Vec<PathBuf>)` (renamed from `FileDrop` — the original name implied `NSDraggingSession` which is not headless-compatible on macOS; see Phase 2 plan §Headless-First Invariant and Unit 12), `WindowRaise`, `Cancel`, `SelectRange { start, len }`, `InsertAtCaret(String)`. `watch_element` is an adapter method, **not** an `Action` variant (plan §KD8 + origin brainstorm §D8). Each has a macOS AX API mapping (all AX calls on main thread per plan §KD9), a Windows UIA pattern mapping, and a new CLI subcommand. The `#[non_exhaustive]` attribute keeps this SemVer-safe; C-ABI consumers use `default:` / wildcard fallthrough plus the exported `AD_RESULT_UNKNOWN = -99` sentinel (plan §KD17) |
|
||||
| P2-O9 | `Action` enum expansion for 2026 agent workloads | New variants: `LongPress { duration_ms }`, `ForceClick`, `ShowMenu`, `DeliverFiles(Vec<PathBuf>)` (renamed from `FileDrop` — the original name implied `NSDraggingSession` which is not headless-compatible on macOS; see Phase 2 plan §Headless-First Invariant and Unit 12), `WindowRaise`, `Cancel`, `SelectRange { start, len }`, `InsertAtCaret(String)`. `watch_element` is an adapter method, **not** an `Action` variant (plan §KD8 + origin brainstorm §D8). Each has a macOS AX API mapping (all AX calls on main thread per plan §KD9), a Windows UIA pattern mapping, a new CLI subcommand, FFI conversion coverage where applicable, and exhaustive platform-dispatch tests in the same change. |
|
||||
| P2-O10 | `ErrorCode` expansion | Add `PermissionRevoked` (distinct from `PermDenied` — TCC yanked mid-session), `ResourceExhausted` (refmap >1 MB, tree node-count cap), `AxMessagingTimeout` (AX-specific timeout separate from orchestration `Timeout`), `AutomationPermissionDenied` (macOS `osascript` grant). Tri-state permission probe at startup distinguishes "never granted" from "revoked" |
|
||||
| P2-O11 | Event-subscription primitive (push, not poll) | New trait method `watch_element(handle, events: &[EventKind], timeout_ms: u64) -> Result<Vec<ElementEvent>>`. macOS: `AXObserverCreate` + `AXObserverAddNotification` + `CFRunLoopSource` (no more polling in `system/wait.rs`). Windows: `IUIAutomation.AddAutomationEventHandler` + `AddFocusChangedEventHandler` + `AddPropertyChangedEventHandler`. New `wait --event value-changed --ref @e5 --timeout 3000` CLI flag. Linux mirrors in Phase 3 via AT-SPI2 D-Bus signals |
|
||||
| P2-O12 | Text range primitives | Read caret, read selection, select a range by offsets, read text at range, insert at caret. macOS: `kAXSelectedTextRangeAttribute` (settable), `AXStringForRangeParameterizedAttribute`, `AXBoundsForRangeParameterizedAttribute`, `AXRangeForLineParameterizedAttribute`, `AXValueCreate(kAXValueCFRangeType, …)`. Windows: `TextPattern.GetSelection`, `TextPattern.DocumentRange`, `TextRange.Select`, `TextRange.Move`, `TextRange.GetText`, `TextRange.GetBoundingRectangles`. Commands: `text get-selection`, `text select-range <ref> <start> <len>`, `text insert-at-caret <ref> <string>`, `text at-offset <ref> <start> <len>` |
|
||||
|
|
|
|||
|
|
@ -63,12 +63,12 @@ Four reference topics, loaded as needed:
|
|||
balances the internal `CFRetain`; on Windows/Linux the call is a no-op
|
||||
but safe to issue.
|
||||
|
||||
- **Action policy.** `ad_execute_action` uses the headless policy by
|
||||
default, matching CLI ref commands: no focus stealing and no cursor
|
||||
movement. Use `ad_execute_action_with_policy(...,
|
||||
AD_POLICY_KIND_FOCUS_FALLBACK, ...)` only when focus-changing behavior is
|
||||
intended, and `AD_POLICY_KIND_HEADED` only for explicit headed input
|
||||
semantics (cursor movement and focus stealing).
|
||||
- **Action policy.** `ad_execute_action` uses headless policy by default.
|
||||
`ad_execute_ref_action_with_policy` should also use headless for semantic
|
||||
ref actions that must fail closed, except `AD_ACTION_KIND_TYPE_TEXT`: use
|
||||
`AD_POLICY_KIND_FOCUS_FALLBACK` for CLI `type` parity because typing needs
|
||||
focus but not cursor movement. Use `AD_POLICY_KIND_HEADED` only for explicit
|
||||
headed input semantics.
|
||||
|
||||
- **Ref-action preflight.** `ad_execute_ref_action_with_policy` resolves the
|
||||
ref strictly and runs the live actionability preflight (visible, stable,
|
||||
|
|
@ -94,9 +94,11 @@ Four reference topics, loaded as needed:
|
|||
actionability preflight inside `ad_execute_ref_action_with_policy` is the
|
||||
equivalent per-call readiness check.
|
||||
|
||||
- **Text input privacy.** On macOS, the focus-fallback or headed policy can use
|
||||
the clipboard briefly for non-ASCII text insertion. Keep the default headless
|
||||
policy or set values directly for sensitive text when the target supports it.
|
||||
- **Text input privacy.** On macOS, focus-fallback or headed text insertion may
|
||||
briefly use the clipboard for non-ASCII text. For sensitive text, prefer
|
||||
`AD_ACTION_KIND_SET_VALUE` with `AD_POLICY_KIND_HEADLESS` when the target
|
||||
supports settable values. Do not use headless `AD_ACTION_KIND_TYPE_TEXT` as
|
||||
CLI-parity ref input; actionability rejects it before dispatch.
|
||||
|
||||
- **Enum discriminants.** Every `#[repr(i32)]` enum field is validated
|
||||
at the C boundary — invalid discriminants return
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ free function. Always call it; the allocator the FFI uses is Rust's
|
|||
| `ad_resolve_element(adapter, entry, &handle)` | `ad_free_handle(adapter, &handle)` — `*mut AdNativeHandle`; the call zeroes `handle.ptr` on success so a follow-up call is a no-op |
|
||||
| `ad_find(adapter, win, query, &handle)` | same as `ad_resolve_element` |
|
||||
| `ad_execute_action(adapter, handle, action, &out)` | `ad_free_action_result(&out)` |
|
||||
| `ad_execute_action_with_policy(adapter, handle, action, policy, &out)` | `ad_free_action_result(&out)` |
|
||||
| `ad_execute_ref_action_with_policy(adapter, entry, action, policy, &out)` | `ad_free_action_result(&out)` |
|
||||
| `ad_notification_action(adapter, idx, expected_app, expected_title, name, &out)` | `ad_free_action_result(&out)` — pass the `app_name`/`title` you observed in `ad_list_notifications` (either may be null) so NC reorder between list and press is caught with `ERR_NOTIFICATION_NOT_FOUND` instead of pressing a different notification |
|
||||
| `ad_screenshot(adapter, target, &buf)` | `ad_image_buffer_free(buf)` (buf is opaque; read via `ad_image_buffer_{data,size,width,height,format}`) |
|
||||
| `ad_get_clipboard(adapter, &text)` | `ad_free_string(text)` |
|
||||
|
|
@ -46,6 +48,10 @@ free function. Always call it; the allocator the FFI uses is Rust's
|
|||
`AdNotificationInfo.body`, etc.) are freed by the struct's owning
|
||||
free function (list_free / release_fields) — do not
|
||||
`ad_free_string()` them individually.
|
||||
- `AdActionResult` owns `action`, `ref_id`, `post_state`, `post_state.states`,
|
||||
`steps`, and each `steps[i].label` / `steps[i].outcome`; free all of them
|
||||
only through `ad_free_action_result(&out)`. Treat the returned counts as
|
||||
read-only metadata; release the unmodified result struct.
|
||||
- Ownership does **not** transfer back to Rust after you free. Keep a
|
||||
local `NULL` to prevent accidental reuse.
|
||||
|
||||
|
|
|
|||
|
|
@ -94,9 +94,9 @@ Use **progressive skeleton traversal** as the default approach. It reduces token
|
|||
- **Scoped invalidation:** re-drilling `--root @e3` only replaces refs from @e3's previous drill — refs from other regions and the skeleton itself are preserved
|
||||
- **Strict resolution:** stale refs return `STALE_REF`; duplicate plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily.
|
||||
- **Actionability:** ref actions check live visibility, stability, enabled state, supported action, policy, and editability before dispatch.
|
||||
- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed with `POLICY_DENIED` when only a physical gesture would work. Pass the global `--headed` flag to permit cursor movement and focus stealing so the physical click/double-click/scroll/keypress fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw cursor commands (`hover`, `drag`, `mouse-*`) are physical and require `--headed`; keyboard commands (`press`, `key-down`, `key-up`) are explicit low-level input.
|
||||
- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed when only a physical gesture would work. `type` uses a focus-fallback base policy because typing needs focus but never moves the cursor. Pass the global `--headed` flag to permit cursor movement and focus stealing so physical fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw cursor commands (`hover`, `drag`, `mouse-*`) are physical and require `--headed`; keyboard commands (`press`, `key-down`, `key-up`) are explicit low-level input.
|
||||
- **Sessions:** use `--session <id>` for concurrent or multi-agent runs that share a latest snapshot pointer; batch entries may override with `"session": "id"`.
|
||||
- **Trace:** use `--trace <path>` for JSONL diagnostics outside stdout; `--trace-strict` fails on trace setup and pre-action writes. Post-action success traces are best-effort because the desktop mutation already happened. Trace fields whose keys contain `text`, `value`, `expected`, `name`, `description`, `message`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, or `placeholder` are redacted to `{ "redacted": true, "chars_bucket": "..." }` at every nesting depth (substring match, so composite keys like `source_window_title` redact too) — do not expect raw values in trace files. Top-level `--trace` is inherited by every `batch` entry, including entries with a `session` override.
|
||||
- **Trace:** use `--trace <path>` for JSONL diagnostics outside stdout; `--trace-strict` fails on trace setup and pre-action writes. Post-action success traces are best-effort because the desktop mutation already happened. Trace fields with sensitive key tokens such as `text`, `value`, `expected`, `name`, `username`, `description`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, or `placeholder` are redacted to `{ "redacted": true }` at every nesting depth. Token matching handles snake_case, kebab-case, and camelCase keys (for example, `source_window_title`, `api_token`, and `typedText` redact, while `filename` stays readable). Top-level `--trace` is inherited by every `batch` entry, including entries with a `session` override.
|
||||
|
||||
## JSON Output Contract
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and
|
|||
|
||||
Ref-based actions run in two modes, Playwright-style:
|
||||
|
||||
- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it **fails closed** with `POLICY_DENIED` rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.)
|
||||
- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it fails closed rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.)
|
||||
- **`--headed`.** A global flag (`agent-desktop --headed click @e5`) that upgrades every ref action to permit focus stealing **and** cursor movement, unlocking the physical click/double-click/scroll/keypress fallbacks in the action chain. The AX path is still tried first, so `--headed` never regresses elements that work headlessly — it only adds fallbacks for elements that need a real gesture (e.g. a gesture-only button with no `AXOpen`).
|
||||
|
||||
Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`, `key-up`) are physical by nature. Cursor-moving commands (`hover`, `drag`, `mouse-*`) require `--headed`; keyboard commands are explicit low-level input.
|
||||
|
|
@ -19,7 +19,8 @@ The command surface is platform-agnostic: every ref action builds an `Action` an
|
|||
|
||||
| Command | Headless path (macOS) | Notes |
|
||||
|---------|---------------|-------|
|
||||
| `click`, `set-value`, `type`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions; the default and most reliable surface |
|
||||
| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions; the default and most reliable surface |
|
||||
| `type` | focus fallback | CLI `type` may focus the target field but never moves the cursor; use `set-value` for pure headless value mutation when supported |
|
||||
| `double-click` | partial | `AXOpen` works headless on items that advertise it (Finder/list/outline rows, table cells). Falls back to `--headed` only for gesture-only targets with no `AXOpen`. |
|
||||
| `triple-click` | no | macOS exposes no triple-click action; it is purely 3 physical clicks → `--headed` only |
|
||||
| `hover` | no | hovering *is* moving the cursor over an element; no AX equivalent |
|
||||
|
|
@ -68,9 +69,9 @@ Performs a semantic right-click/context-menu action and includes `menu` plus `me
|
|||
agent-desktop type @e2 "hello@example.com"
|
||||
agent-desktop type @e2 "multi line\ntext"
|
||||
```
|
||||
In headless mode (default), `type` inserts text by mutating the element's AX value and may focus the target field (typing requires focus) but never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs).
|
||||
`type` uses the focus-fallback policy floor: it may focus the target field because typing requires focus, but it never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs).
|
||||
|
||||
Under `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer the default value path or `set-value` when the target supports it.
|
||||
Under focus-fallback or `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer `set-value` when the target supports it.
|
||||
|
||||
### set-value
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -80,10 +80,10 @@ The default activation-chain deadline is 10 seconds. Set `AGENT_DESKTOP_CHAIN_TI
|
|||
Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. macOS actions split semantic AX steps from explicit physical/headed paths:
|
||||
|
||||
- `click`, `right-click`, `scroll`, `set-value`, `clear`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, and `scroll-to` try AX-first semantics and fail clearly when the headless path is unavailable.
|
||||
- `type` mutates a settable AX text value in headless mode. It does not call `ensure_app_focused` or use the pasteboard in the default CLI path.
|
||||
- `type` uses focus fallback in the CLI/ref-action path. It may focus the target field, never moves the cursor, and can use the pasteboard for non-ASCII insertion. Use `set-value` for pure headless value mutation when supported.
|
||||
- `focus`, `press`, `hover`, `drag`, and `mouse-*` are explicit physical/focus/cursor commands.
|
||||
- FFI callers can opt into focus-only or physical policy explicitly; CLI ref commands do not do that implicitly.
|
||||
- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Keep the default headless path or use `set-value` for sensitive text when possible.
|
||||
- FFI ref-action callers should use focus fallback for `type` to match CLI behavior; direct-handle `ad_execute_action` is lower-level and defaults to headless.
|
||||
- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Use `set-value` for sensitive text when possible.
|
||||
- If a command would need a forbidden physical path, it returns a structured error with a recovery hint.
|
||||
|
||||
### Surfaces
|
||||
|
|
|
|||
Loading…
Reference in a new issue