fix: harden desktop action reliability

This commit is contained in:
Lahfir 2026-06-17 00:03:23 -07:00
parent 8b8073ee3a
commit 298ea1036f
48 changed files with 1173 additions and 512 deletions

View file

@ -32,7 +32,25 @@ pub fn check_live(
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
let mut observed = entry.clone();
let live = adapter.get_live_element(handle)?;
let live = match adapter.get_live_element(handle) {
Ok(live) => live,
Err(err)
if matches!(
err.code,
ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported
) =>
{
return check_with_stability(entry.bounds_hash, &observed, request);
}
Err(err) => return Err(err),
};
if live_element_is_stale(&live) {
return Err(AdapterError::new(
ErrorCode::StaleRef,
"Resolved element no longer exposes live accessibility state",
)
.with_suggestion("Run 'snapshot' again and retry with the refreshed ref"));
}
if let Some(state) = live.state {
observed.role = state.role;
observed.states = state.states;
@ -47,6 +65,15 @@ pub fn check_live(
check_with_stability(entry.bounds_hash, &observed, request)
}
fn live_element_is_stale(live: &crate::adapter::LiveElement) -> bool {
let role_unknown = live
.state
.as_ref()
.is_none_or(|state| state.role == "unknown");
let actions_empty = live.available_actions.as_ref().is_none_or(Vec::is_empty);
role_unknown && live.bounds.is_none() && actions_empty
}
fn check_with_stability(
expected_bounds_hash: Option<u64>,
entry: &RefEntry,
@ -120,7 +147,10 @@ fn action_supported_check(entry: &RefEntry, request: &ActionRequest) -> Actionab
if request.action.requires_cursor_policy() {
return pass("supported_action");
}
if supported_by_available_actions(&request.action, &entry.available_actions) {
if capability::contains_any(
&entry.available_actions,
capability::for_action(&request.action),
) {
return pass("supported_action");
}
if may_use_fallback(&request.action, request) {
@ -156,11 +186,7 @@ fn editable_check(entry: &RefEntry, action: &Action) -> ActionabilityCheck {
if entry.role == "textfield" || entry.role == "combobox" {
return pass("editable");
}
if entry
.available_actions
.iter()
.any(|action| action == capability::SET_VALUE)
{
if capability::contains(&entry.available_actions, capability::SET_VALUE) {
return pass("editable");
}
fail("editable", format!("role {} is not editable", entry.role))
@ -179,12 +205,6 @@ fn failure_reasons(report: &ActionabilityReport) -> String {
.join(", ")
}
fn supported_by_available_actions(action: &Action, available_actions: &[String]) -> bool {
capability::for_action(action)
.iter()
.any(|expected| available_actions.iter().any(|action| action == expected))
}
fn may_use_fallback(action: &Action, request: &ActionRequest) -> bool {
action.may_use_focus_fallback() && request.policy.allow_focus_steal
}
@ -216,3 +236,7 @@ fn unknown(name: &'static str, reason: impl Into<String>) -> ActionabilityCheck
#[cfg(test)]
#[path = "../actionability_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "../actionability_live_tests.rs"]
mod live_tests;

View file

@ -0,0 +1,305 @@
use super::*;
use crate::{
action::Action,
action_request::ActionRequest,
adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface},
capability,
element_state::ElementState,
node::Rect,
refs::RefEntry,
};
struct LiveAdapter {
state: Option<ElementState>,
bounds: Option<Rect>,
actions: Option<Vec<String>>,
}
impl PlatformAdapter for LiveAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Ok(self.state.clone())
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
Ok(self.bounds)
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
) -> Result<Option<Vec<String>>, AdapterError> {
Ok(self.actions.clone())
}
}
struct CombinedLiveAdapter;
impl PlatformAdapter for CombinedLiveAdapter {
fn get_live_element(&self, _handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
bounds: Some(Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}),
available_actions: Some(vec![capability::CLICK.into()]),
})
}
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
) -> Result<Option<Vec<String>>, AdapterError> {
panic!("check_live should use get_live_element")
}
}
struct LiveReadErrorAdapter;
impl PlatformAdapter for LiveReadErrorAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Err(AdapterError::permission_denied())
}
}
struct UnsupportedLiveAdapter;
impl PlatformAdapter for UnsupportedLiveAdapter {}
struct DeadLiveElementAdapter;
impl PlatformAdapter for DeadLiveElementAdapter {
fn get_live_element(&self, _handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
state: Some(ElementState {
role: "unknown".into(),
states: vec![],
value: None,
}),
bounds: None,
available_actions: Some(vec![]),
})
}
}
fn entry() -> RefEntry {
let bounds = Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
};
RefEntry {
pid: 1,
role: "button".into(),
name: Some("OK".into()),
value: None,
description: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
source_surface: SnapshotSurface::Window,
root_ref: None,
path_is_absolute: true,
path: smallvec::SmallVec::new(),
}
}
#[test]
fn live_actionability_overrides_stale_snapshot_state() {
let mut stale = entry();
stale.states.push("disabled".into());
let adapter = LiveAdapter {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
bounds: stale.bounds,
actions: Some(vec![capability::CLICK.into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_uses_combined_live_element_read() {
let mut stale = entry();
stale.states.push("disabled".into());
stale.bounds = Some(Rect {
x: 1.0,
y: 1.0,
width: 0.0,
height: 20.0,
});
stale.available_actions = vec![];
let report = check_live(
&stale,
&NativeHandle::null(),
&CombinedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_uses_actions_gained_after_snapshot() {
let mut stale = entry();
stale.available_actions = vec![];
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec![capability::CLICK.into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_fails_when_action_disappears_after_snapshot() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec![capability::SET_VALUE.into()]),
};
let err = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert!(err.message.contains("supported_action"));
}
#[test]
fn live_actionability_allows_identity_resolved_bounds_change() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: Some(Rect {
x: 100.0,
y: 100.0,
width: 20.0,
height: 20.0,
}),
actions: Some(vec![capability::CLICK.into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
let stable = report
.checks
.iter()
.find(|check| check.name == "stable")
.unwrap();
assert_eq!(stable.status, ActionabilityStatus::Unknown);
}
#[test]
fn empty_live_actions_do_not_erase_snapshot_capabilities() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec![]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn unsupported_live_reads_fall_back_to_snapshot_entry() {
let report = check_live(
&entry(),
&NativeHandle::null(),
&UnsupportedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn empty_live_element_fails_as_stale_before_dispatch() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&DeadLiveElementAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::StaleRef);
assert!(err.message.contains("no longer exposes live"));
}
#[test]
fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&LiveReadErrorAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::PermDenied);
}

View file

@ -2,83 +2,12 @@ use super::*;
use crate::{
action::{Action, Direction},
action_request::ActionRequest,
adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface},
element_state::ElementState,
adapter::SnapshotSurface,
capability,
node::Rect,
refs::RefEntry,
};
struct LiveAdapter {
state: Option<ElementState>,
bounds: Option<Rect>,
actions: Option<Vec<String>>,
}
impl PlatformAdapter for LiveAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Ok(self.state.clone())
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
Ok(self.bounds)
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
) -> Result<Option<Vec<String>>, AdapterError> {
Ok(self.actions.clone())
}
}
struct CombinedLiveAdapter;
impl PlatformAdapter for CombinedLiveAdapter {
fn get_live_element(&self, _handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
bounds: Some(Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}),
available_actions: Some(vec!["Click".into()]),
})
}
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
) -> Result<Option<Vec<String>>, AdapterError> {
panic!("check_live should use get_live_element")
}
}
struct LiveReadErrorAdapter;
impl PlatformAdapter for LiveReadErrorAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Err(AdapterError::permission_denied())
}
}
struct UnsupportedLiveAdapter;
impl PlatformAdapter for UnsupportedLiveAdapter {}
fn entry() -> RefEntry {
let bounds = Rect {
x: 1.0,
@ -95,7 +24,7 @@ fn entry() -> RefEntry {
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -164,7 +93,7 @@ fn cursor_movement_requires_physical_policy() {
fn headless_type_text_fails_policy_before_dispatch() {
let mut target = entry();
target.role = "textfield".into();
target.available_actions = vec!["SetValue".into()];
target.available_actions = vec![capability::SET_VALUE.into()];
let err = check(
&target,
@ -194,11 +123,11 @@ fn command_aliases_match_platform_capabilities() {
let mut editable = entry();
editable.role = "textfield".into();
editable.available_actions = vec!["SetValue".into()];
editable.available_actions = vec![capability::SET_VALUE.into()];
assert!(check(&editable, &ActionRequest::headless(Action::Clear)).is_ok());
let mut scrollable = entry();
scrollable.available_actions = vec!["Scroll".into()];
scrollable.available_actions = vec![capability::SCROLL.into()];
assert!(
check(
&scrollable,
@ -208,7 +137,7 @@ fn command_aliases_match_platform_capabilities() {
);
assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_err());
scrollable.available_actions = vec!["ScrollTo".into()];
scrollable.available_actions = vec![capability::SCROLL_TO.into()];
assert!(
check(
&scrollable,
@ -218,170 +147,3 @@ fn command_aliases_match_platform_capabilities() {
);
assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_ok());
}
#[test]
fn live_actionability_overrides_stale_snapshot_state() {
let mut stale = entry();
stale.states.push("disabled".into());
let adapter = LiveAdapter {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
bounds: stale.bounds,
actions: Some(vec!["Click".into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_uses_combined_live_element_read() {
let mut stale = entry();
stale.states.push("disabled".into());
stale.bounds = Some(Rect {
x: 1.0,
y: 1.0,
width: 0.0,
height: 20.0,
});
stale.available_actions = vec![];
let report = check_live(
&stale,
&NativeHandle::null(),
&CombinedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_uses_actions_gained_after_snapshot() {
let mut stale = entry();
stale.available_actions = vec![];
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec!["Click".into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_actionability_fails_when_action_disappears_after_snapshot() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec!["SetValue".into()]),
};
let err = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert!(err.message.contains("supported_action"));
}
#[test]
fn live_actionability_allows_identity_resolved_bounds_change() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: Some(Rect {
x: 100.0,
y: 100.0,
width: 20.0,
height: 20.0,
}),
actions: Some(vec!["Click".into()]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
let stable = report
.checks
.iter()
.find(|check| check.name == "stable")
.unwrap();
assert_eq!(stable.status, ActionabilityStatus::Unknown);
}
#[test]
fn empty_live_actions_do_not_erase_snapshot_capabilities() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec![]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn unsupported_live_reads_fall_back_to_snapshot_entry() {
let report = check_live(
&entry(),
&NativeHandle::null(),
&UnsupportedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&LiveReadErrorAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::PermDenied);
}

View file

@ -284,11 +284,15 @@ pub trait PlatformAdapter: Send + Sync {
}
fn get_live_element(&self, handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
let live = LiveElement {
state: optional_live_read(self.get_live_state(handle))?,
bounds: optional_live_read(self.get_element_bounds(handle))?,
available_actions: optional_live_read(self.get_live_actions(handle))?,
})
};
if live.state.is_none() && live.bounds.is_none() && live.available_actions.is_none() {
return Err(AdapterError::not_supported("get_live_element"));
}
Ok(live)
}
fn press_key_for_app(

View file

@ -2,7 +2,8 @@ use crate::{
action::DragParams,
adapter::PlatformAdapter,
commands::point_resolve::{
PointResolveArgs, focus_for_physical_input, resolve_point_from_ref_or_xy_with_context,
PointResolveArgs, focus_for_physical_input, require_cursor_policy,
resolve_point_from_ref_or_xy_with_context,
},
context::CommandContext,
error::AppError,
@ -24,6 +25,7 @@ pub fn execute(
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "drag")?;
let from = resolve_point_from_ref_or_xy_with_context(
PointResolveArgs {
ref_id: args.from_ref.as_deref(),

View file

@ -2,6 +2,7 @@ use super::*;
use crate::{
action::DragParams,
adapter::{NativeHandle, PlatformAdapter},
capability,
error::AdapterError,
node::Rect,
refs::{RefEntry, RefMap},
@ -65,7 +66,12 @@ fn xy_args(drop_delay_ms: Option<u64>) -> DragArgs {
fn drop_delay_is_threaded_into_drag_params_and_response() {
let adapter = DragCaptureAdapter::new();
let value = execute(xy_args(Some(750)), &adapter, &CommandContext::default()).unwrap();
let value = execute(
xy_args(Some(750)),
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap();
assert_eq!(value["dragged"], true);
assert_eq!(value["drop_delay_ms"], 750);
@ -79,7 +85,12 @@ fn drop_delay_is_threaded_into_drag_params_and_response() {
fn drop_delay_omitted_uses_adapter_default_and_no_response_field() {
let adapter = DragCaptureAdapter::new();
let value = execute(xy_args(None), &adapter, &CommandContext::default()).unwrap();
let value = execute(
xy_args(None),
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap();
assert!(value.get("drop_delay_ms").is_none());
let captured = adapter.captured.lock().unwrap().clone().unwrap();
@ -96,7 +107,7 @@ fn ref_entry(pid: i32) -> RefEntry {
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -145,27 +156,28 @@ fn drag_resolves_ref_bounds_to_center_point() {
duration_ms: None,
drop_delay_ms: Some(300),
};
execute(args, &adapter, &CommandContext::default()).unwrap();
execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap();
let captured = adapter.captured.lock().unwrap().clone().unwrap();
assert_eq!((captured.from.x, captured.from.y), (30.0, 50.0));
}
#[test]
fn headless_ref_drag_never_steals_focus() {
fn headless_ref_drag_is_policy_denied_before_cursor_move() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = DragCaptureAdapter::new();
let value = execute(
let err = execute(
cross_app_args(snapshot_id),
&adapter,
&CommandContext::default(),
)
.unwrap();
.unwrap_err();
assert_eq!(err.code(), "POLICY_DENIED");
assert!(adapter.focused_pids.lock().unwrap().is_empty());
assert!(value.get("focused").is_none());
assert!(adapter.captured.lock().unwrap().is_none());
}
#[test]

View file

@ -60,11 +60,11 @@ pub(crate) fn resolve_ref_with_context<'a>(
}
};
tracing::debug!(
"resolve: {} -> pid={} role={} name={:?}",
"resolve: {} -> pid={} role={} name_chars={:?}",
ref_id,
entry.pid,
entry.role,
entry.name.as_deref().unwrap_or("(none)")
entry.name.as_deref().map(|name| name.chars().count())
);
context.trace_lazy("ref.resolve.entry", || {
json!({

View file

@ -2,6 +2,7 @@ use super::test_support::entry;
use super::*;
use crate::action::Action;
use crate::adapter::NativeHandle;
use crate::capability;
use crate::error::{AdapterError, ErrorCode};
use crate::node::AppInfo;
use crate::refs::RefMap;
@ -171,7 +172,15 @@ impl PlatformAdapter for CountingPipelineAdapter {
_handle: &NativeHandle,
) -> Result<crate::adapter::LiveElement, AdapterError> {
self.live_reads.fetch_add(1, Ordering::SeqCst);
Ok(crate::adapter::LiveElement::default())
Ok(crate::adapter::LiveElement {
state: Some(crate::element_state::ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
bounds: None,
available_actions: Some(vec![capability::CLICK.into()]),
})
}
fn execute_action(

View file

@ -2,7 +2,7 @@ use crate::{
action::{MouseButton, MouseEvent, MouseEventKind},
adapter::PlatformAdapter,
commands::point_resolve::{
PointResolveArgs, ResolvedPoint, focus_for_physical_input,
PointResolveArgs, focus_for_physical_input, require_cursor_policy,
resolve_point_from_ref_or_xy_with_context,
},
context::CommandContext,
@ -22,7 +22,17 @@ pub fn execute(
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let resolved = resolve_hover_point(&args, adapter, context)?;
require_cursor_policy(context, "hover")?;
let resolved = resolve_point_from_ref_or_xy_with_context(
PointResolveArgs {
ref_id: args.ref_id.as_deref(),
xy: args.xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide a ref (@e1) or --xy x,y",
},
adapter,
context,
)?;
let focused = focus_for_physical_input(resolved.pid, adapter, context)?;
adapter.mouse_event(MouseEvent {
kind: MouseEventKind::Move,
@ -39,23 +49,6 @@ pub fn execute(
Ok(response)
}
fn resolve_hover_point(
args: &HoverArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<ResolvedPoint, AppError> {
resolve_point_from_ref_or_xy_with_context(
PointResolveArgs {
ref_id: args.ref_id.as_deref(),
xy: args.xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide a ref (@e1) or --xy x,y",
},
adapter,
context,
)
}
#[cfg(test)]
#[path = "hover_tests.rs"]
mod tests;

View file

@ -1,6 +1,7 @@
use super::*;
use crate::{
adapter::NativeHandle,
capability,
error::AdapterError,
node::Rect,
refs::{RefEntry, RefMap},
@ -60,7 +61,7 @@ fn ref_snapshot(pid: i32) -> String {
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -82,16 +83,16 @@ fn ref_args(snapshot_id: String) -> HoverArgs {
}
#[test]
fn headless_ref_hover_never_steals_focus() {
fn headless_ref_hover_is_policy_denied_before_cursor_move() {
let _guard = HomeGuard::new();
let snapshot_id = ref_snapshot(42);
let adapter = HoverCaptureAdapter::new();
let value = execute(ref_args(snapshot_id), &adapter, &CommandContext::default()).unwrap();
let err = execute(ref_args(snapshot_id), &adapter, &CommandContext::default()).unwrap_err();
assert_eq!(value["hovered"], true);
assert_eq!(err.code(), "POLICY_DENIED");
assert!(adapter.focused_pids.lock().unwrap().is_empty());
assert!(value.get("focused").is_none());
assert!(adapter.moved_to.lock().unwrap().is_none());
}
#[test]

View file

@ -1,6 +1,8 @@
use crate::{
action::{MouseButton, MouseEvent, MouseEventKind, Point},
adapter::PlatformAdapter,
commands::point_resolve::require_cursor_policy,
context::CommandContext,
error::AppError,
};
use serde_json::{Value, json};
@ -12,7 +14,12 @@ pub struct MouseClickArgs {
pub count: u32,
}
pub fn execute(args: MouseClickArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
pub fn execute(
args: MouseClickArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "mouse-click")?;
adapter.mouse_event(MouseEvent {
kind: MouseEventKind::Click { count: args.count },
point: Point {

View file

@ -1,6 +1,8 @@
use crate::{
action::{MouseButton, MouseEvent, MouseEventKind, Point},
adapter::PlatformAdapter,
commands::point_resolve::require_cursor_policy,
context::CommandContext,
error::AppError,
};
use serde_json::{Value, json};
@ -11,7 +13,12 @@ pub struct MouseDownArgs {
pub button: MouseButton,
}
pub fn execute(args: MouseDownArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
pub fn execute(
args: MouseDownArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "mouse-down")?;
adapter.mouse_event(MouseEvent {
kind: MouseEventKind::Down,
point: Point {

View file

@ -1,6 +1,8 @@
use crate::{
action::{MouseButton, MouseEvent, MouseEventKind, Point},
adapter::PlatformAdapter,
commands::point_resolve::require_cursor_policy,
context::CommandContext,
error::AppError,
};
use serde_json::{Value, json};
@ -10,7 +12,12 @@ pub struct MouseMoveArgs {
pub y: f64,
}
pub fn execute(args: MouseMoveArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
pub fn execute(
args: MouseMoveArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "mouse-move")?;
adapter.mouse_event(MouseEvent {
kind: MouseEventKind::Move,
point: Point {

View file

@ -1,6 +1,8 @@
use crate::{
action::{MouseButton, MouseEvent, MouseEventKind, Point},
adapter::PlatformAdapter,
commands::point_resolve::require_cursor_policy,
context::CommandContext,
error::AppError,
};
use serde_json::{Value, json};
@ -11,7 +13,12 @@ pub struct MouseUpArgs {
pub button: MouseButton,
}
pub fn execute(args: MouseUpArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
pub fn execute(
args: MouseUpArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "mouse-up")?;
adapter.mouse_event(MouseEvent {
kind: MouseEventKind::Up,
point: Point {

View file

@ -1,6 +1,9 @@
use crate::{
action::Point, adapter::PlatformAdapter, commands::helpers::resolve_ref_with_context,
context::CommandContext, error::AppError,
action::Point,
adapter::PlatformAdapter,
commands::helpers::resolve_ref_with_context,
context::CommandContext,
error::{AdapterError, AppError},
};
use serde_json::json;
@ -16,6 +19,21 @@ pub(crate) struct ResolvedPoint {
pub pid: Option<i32>,
}
pub(crate) fn require_cursor_policy(
context: &CommandContext,
command: &str,
) -> Result<(), AppError> {
let policy = context.physical_input_policy();
if policy.allow_cursor_move {
return Ok(());
}
Err(AdapterError::policy_denied_for_policy(
format!("{command} moves the cursor and is disabled in headless mode"),
policy,
)
.into())
}
pub(crate) fn resolve_point_from_ref_or_xy_with_context(
args: PointResolveArgs<'_>,
adapter: &dyn PlatformAdapter,
@ -67,3 +85,20 @@ pub(crate) fn focus_for_physical_input(
context.trace_lazy("input.focus_app", || json!({ "pid": pid, "ok": focused }))?;
Ok(focused)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn physical_input_requires_headed_context() {
let err = require_cursor_policy(&CommandContext::default(), "mouse-move").unwrap_err();
assert_eq!(err.code(), "POLICY_DENIED");
}
#[test]
fn headed_context_allows_physical_input() {
require_cursor_policy(&CommandContext::default().with_headed(true), "mouse-move").unwrap();
}
}

View file

@ -8,7 +8,7 @@ use crate::{
},
context::CommandContext,
error::{AppError, ErrorCode},
notification::NotificationFilter,
notification::{NotificationFilter, NotificationInfo},
refs_store::RefStore,
search_text, snapshot,
};
@ -234,17 +234,17 @@ fn wait_for_notification(
};
let start = Instant::now();
let timeout = Duration::from_millis(timeout_ms);
let mut baseline_indices: Option<std::collections::HashSet<usize>> = None;
let mut baseline: Option<std::collections::HashMap<NotificationFingerprint, usize>> = None;
let mut last_error = None;
loop {
match adapter.list_notifications(&filter) {
Ok(current) => match &baseline_indices {
Ok(current) => match &baseline {
None => {
baseline_indices = Some(current.iter().map(|n| n.index).collect());
baseline = Some(notification_counts(&current));
}
Some(baseline) => {
if let Some(notif) = current.iter().find(|n| !baseline.contains(&n.index)) {
if let Some(notif) = first_new_notification(&current, baseline) {
let elapsed = start.elapsed().as_millis();
return Ok(json!({
"condition": "notification",
@ -272,6 +272,53 @@ fn wait_for_notification(
}
}
#[derive(Clone, Eq, Hash, PartialEq)]
struct NotificationFingerprint {
app_name: String,
title: String,
body: Option<String>,
actions: Vec<String>,
}
impl From<&NotificationInfo> for NotificationFingerprint {
fn from(info: &NotificationInfo) -> Self {
Self {
app_name: info.app_name.clone(),
title: info.title.clone(),
body: info.body.clone(),
actions: info.actions.clone(),
}
}
}
fn notification_counts(
notifications: &[NotificationInfo],
) -> std::collections::HashMap<NotificationFingerprint, usize> {
let mut counts = std::collections::HashMap::new();
for notification in notifications {
*counts
.entry(NotificationFingerprint::from(notification))
.or_insert(0) += 1;
}
counts
}
fn first_new_notification<'a>(
current: &'a [NotificationInfo],
baseline: &std::collections::HashMap<NotificationFingerprint, usize>,
) -> Option<&'a NotificationInfo> {
let mut seen = std::collections::HashMap::new();
for notification in current {
let fingerprint = NotificationFingerprint::from(notification);
let current_count = seen.entry(fingerprint.clone()).or_insert(0);
*current_count += 1;
if *current_count > baseline.get(&fingerprint).copied().unwrap_or(0) {
return Some(notification);
}
}
None
}
#[cfg(test)]
#[path = "wait_test_support.rs"]
mod test_support;

View file

@ -89,14 +89,14 @@ pub(crate) fn wait_for_element(
});
if fixed_refmap.is_none() {
if let Some(cache) = latest_cache.as_mut() {
cache.refresh_if_due();
cache.refresh_if_due()?;
}
}
}
Err(err) => return Err(AppError::Adapter(err)),
}
} else if let Some(cache) = latest_cache.as_mut() {
cache.refresh_if_due();
cache.refresh_if_due()?;
}
let remaining = timeout.saturating_sub(start.elapsed());

View file

@ -28,24 +28,26 @@ impl<'a> LatestRefCache<'a> {
self.refmap.get(ref_id).cloned()
}
pub(crate) fn refresh_if_due(&mut self) {
pub(crate) fn refresh_if_due(&mut self) -> Result<(), AppError> {
if self.last_refresh.elapsed() < Duration::from_millis(500) {
return;
return Ok(());
}
self.last_refresh = Instant::now();
if let Some(snapshot_id) = self.store.latest_snapshot_id() {
if self.snapshot_id.as_deref() == Some(snapshot_id.as_str()) {
return;
return Ok(());
}
match self.store.load_snapshot(&snapshot_id) {
Ok(refmap) => {
self.snapshot_id = Some(snapshot_id);
self.refmap = refmap;
Ok(())
}
Err(err) => {
tracing::warn!(
"latest snapshot {snapshot_id} unreadable during wait refresh: {err}"
);
Err(err)
}
}
} else {
@ -53,9 +55,11 @@ impl<'a> LatestRefCache<'a> {
Ok(refmap) => {
self.refmap = refmap;
self.snapshot_id = self.store.latest_snapshot_id();
Ok(())
}
Err(err) => {
tracing::warn!("latest refmap unreadable during wait refresh: {err}");
Err(err)
}
}
}

View file

@ -1,6 +1,7 @@
use super::*;
use crate::{
adapter::SnapshotSurface,
capability,
refs::{RefEntry, RefMap},
refs_test_support::HomeGuard,
};
@ -16,7 +17,7 @@ fn save_ref(pid: i32, name: Option<&str>) -> String {
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -41,7 +42,7 @@ fn latest_ref_cache_picks_up_newer_snapshot_after_refresh() {
assert_ne!(first_id, second_id);
cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2);
cache.refresh_if_due();
cache.refresh_if_due().unwrap();
assert_eq!(cache.snapshot_id.as_deref(), Some(second_id.as_str()));
assert!(cache.entry("@e1").is_some());
@ -60,8 +61,32 @@ fn latest_ref_cache_debounces_consecutive_refreshes() {
let debounced_refresh = std::time::Instant::now();
cache.last_refresh = debounced_refresh;
cache.refresh_if_due();
cache.refresh_if_due().unwrap();
assert_eq!(cache.snapshot_id, pinned_snapshot_id);
assert_eq!(cache.last_refresh, debounced_refresh);
}
#[test]
fn latest_ref_cache_fails_closed_when_new_latest_snapshot_disappears() {
let _guard = HomeGuard::new();
let first_id = save_ref(1, Some("First"));
let store = RefStore::new().unwrap();
let mut cache = LatestRefCache::new(&store).unwrap();
assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str()));
let second_id = save_ref(2, Some("Second"));
let home = crate::refs::home_dir().unwrap();
let snapshot_dir = home
.join(".agent-desktop")
.join("snapshots")
.join(&second_id);
std::fs::remove_dir_all(snapshot_dir).unwrap();
cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2);
let err = cache.refresh_if_due().unwrap_err();
assert_eq!(err.code(), "SNAPSHOT_NOT_FOUND");
assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str()));
}

View file

@ -1,6 +1,7 @@
use super::*;
use crate::{
adapter::{NativeHandle, PlatformAdapter},
capability,
commands::wait_predicate,
error::{AdapterError, ErrorCode},
refs::{RefEntry, RefMap},
@ -87,7 +88,7 @@ fn snapshot_with_one_ref() -> String {
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -154,7 +155,7 @@ fn element_wait_passes_remaining_budget_to_resolver() {
"@e1".into(),
Some(snapshot_id),
wait_predicate::ElementPredicate::Exists,
75,
500,
&adapter,
&crate::context::CommandContext::default(),
)
@ -163,7 +164,7 @@ fn element_wait_passes_remaining_budget_to_resolver() {
assert_eq!(value["observed"]["exists"], true);
let captured = adapter.captured_ms.lock().unwrap();
assert_eq!(captured.len(), 1);
assert!(captured[0] <= 75);
assert!(captured[0] <= 500);
}
#[test]

View file

@ -140,6 +140,24 @@ fn notification_wait_retries_transient_baseline_errors() {
assert_eq!(value["notification"]["title"], "fresh");
}
#[test]
fn notification_wait_fingerprint_ignores_reindexed_existing_notification() {
let baseline = notification_counts(&[notification(0, "old")]);
let current = vec![notification(4, "old")];
assert!(first_new_notification(&current, &baseline).is_none());
}
#[test]
fn notification_wait_fingerprint_detects_duplicate_new_notification() {
let baseline = notification_counts(&[notification(0, "same")]);
let current = vec![notification(4, "same"), notification(5, "same")];
let found = first_new_notification(&current, &baseline).unwrap();
assert_eq!(found.index, 5);
}
#[test]
fn notification_wait_times_out_with_last_error_after_transient_failures() {
let adapter = FlakyNotificationAdapter::with_responses(vec![]);

View file

@ -49,9 +49,9 @@ impl CommandContext {
}
}
/// Policy for the raw physical-input commands (hover, drag, mouse-*).
/// They have no semantic action chain, so the base is fully headless:
/// a ref-addressed gesture may not steal focus unless `--headed` opts in.
/// Policy for raw physical-input commands (hover, drag, mouse-*). They
/// have no semantic action chain, so headless denies both focus stealing
/// and cursor movement unless `--headed` opts in.
pub fn physical_input_policy(&self) -> InteractionPolicy {
self.policy_with_base(InteractionPolicy::headless())
}
@ -232,7 +232,7 @@ mod tests {
"value": "hidden",
"name": "private label",
"description": "private desc",
"message": "private error",
"message": "diagnostic error",
"post_state": { "value": "deep secret" },
"target_label": "button secret",
"nested": { "expected": "token" },
@ -251,7 +251,7 @@ mod tests {
assert_eq!(event["value"]["redacted"], true);
assert_eq!(event["name"]["redacted"], true);
assert_eq!(event["description"]["redacted"], true);
assert_eq!(event["message"]["redacted"], true);
assert_eq!(event["message"], "diagnostic error");
assert_eq!(event["post_state"]["value"]["redacted"], true);
assert_eq!(event["target_label"]["redacted"], true);
assert_eq!(event["nested"]["expected"]["redacted"], true);
@ -263,7 +263,6 @@ mod tests {
assert!(!body.contains("hidden"));
assert!(!body.contains("private label"));
assert!(!body.contains("private desc"));
assert!(!body.contains("private error"));
assert!(!body.contains("token"));
assert!(!body.contains("private window title"));
assert!(!body.contains("internal.example"));

View file

@ -2,6 +2,8 @@ use serde::Serialize;
use serde_json::Value;
use thiserror::Error;
use crate::interaction_policy::InteractionPolicy;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum ErrorCode {
@ -158,6 +160,21 @@ impl AdapterError {
"Use an explicit mouse/focus command if physical interaction is intended",
)
}
pub fn policy_denied_for_policy(message: impl Into<String>, policy: InteractionPolicy) -> Self {
Self::new(ErrorCode::PolicyDenied, message)
.with_suggestion(policy_denied_suggestion(policy))
}
}
fn policy_denied_suggestion(policy: InteractionPolicy) -> &'static str {
if policy.allow_focus_steal && !policy.allow_cursor_move {
"Retry with --headed to permit cursor movement, or use an explicit mouse command if physical input is intended"
} else if !policy.allow_focus_steal && !policy.allow_cursor_move {
"Headless mode allows only accessibility-backed actions; refresh the snapshot or target an element with the needed semantic action"
} else {
"Use an explicit mouse/focus command if physical interaction is intended"
}
}
#[derive(Debug, Error)]
@ -250,4 +267,15 @@ mod tests {
assert_eq!(err.code.as_str(), "AMBIGUOUS_TARGET");
assert!(err.suggestion.is_some());
}
#[test]
fn policy_denied_suggestion_is_mode_aware() {
let headless =
AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::headless());
assert!(!headless.suggestion.unwrap().contains("--headed"));
let focus_fallback =
AdapterError::policy_denied_for_policy("blocked", InteractionPolicy::focus_fallback());
assert!(focus_fallback.suggestion.unwrap().contains("--headed"));
}
}

View file

@ -6,6 +6,7 @@ use crate::{
context::CommandContext,
error::{AdapterError, AppError},
refs::RefEntry,
resolved_element::ResolvedElement,
};
use serde_json::json;
@ -73,17 +74,18 @@ pub fn execute_entry(
request: ActionRequest,
) -> Result<ActionResult, AdapterError> {
let handle = adapter.resolve_element_strict(entry)?;
let handle = ResolvedElement::new(adapter, handle);
let context = CommandContext::default();
let result = execute_resolved(
ResolvedRefAction {
adapter,
entry,
handle: &handle,
ref_id: "",
context: &CommandContext::default(),
handle: handle.handle(),
ref_id: "<ffi>",
context: &context,
},
request,
);
let _ = adapter.release_handle(&handle);
result.map_err(into_adapter_error)
}

View file

@ -3,7 +3,9 @@ use crate::{
action::Action,
action_result::ActionResult,
adapter::{NativeHandle, SnapshotSurface},
capability,
};
use std::sync::atomic::{AtomicU32, Ordering};
struct ReleaseFailingAdapter;
@ -25,6 +27,29 @@ impl PlatformAdapter for ReleaseFailingAdapter {
}
}
struct ErrorReleasingAdapter {
releases: AtomicU32,
}
impl PlatformAdapter for ErrorReleasingAdapter {
fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> {
Ok(NativeHandle::null())
}
fn execute_action(
&self,
_handle: &NativeHandle,
_request: ActionRequest,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::internal("dispatch failed"))
}
fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> {
self.releases.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
fn entry() -> RefEntry {
RefEntry {
pid: 1,
@ -35,7 +60,7 @@ fn entry() -> RefEntry {
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -57,3 +82,16 @@ fn successful_action_survives_release_failure() {
assert_eq!(result.action, "click");
}
#[test]
fn failed_action_still_releases_resolved_handle() {
let adapter = ErrorReleasingAdapter {
releases: AtomicU32::new(0),
};
let err =
execute_entry(&adapter, &entry(), ActionRequest::headless(Action::Click)).unwrap_err();
assert_eq!(err.code, crate::error::ErrorCode::Internal);
assert_eq!(adapter.releases.load(Ordering::SeqCst), 1);
}

View file

@ -48,6 +48,33 @@ fn snapshot_roundtrip_updates_latest_pointer() {
assert_eq!(store.load(None).unwrap().len(), 1);
}
#[test]
fn concurrent_writers_preserve_all_snapshots() {
let _guard = HomeGuard::new();
let store = RefStore::new().unwrap();
let mut handles = Vec::new();
for i in 0..8 {
let store = store.clone();
handles.push(std::thread::spawn(move || {
store
.save_new_snapshot(&map_with(&format!("Snapshot {i}")))
.unwrap()
}));
}
let ids = handles
.into_iter()
.map(|handle| handle.join().unwrap())
.collect::<Vec<_>>();
for id in &ids {
assert_eq!(store.load_snapshot(id).unwrap().len(), 1);
}
let latest = store.latest_snapshot_id().unwrap();
assert!(ids.iter().any(|id| id == &latest));
}
#[test]
fn sessions_are_isolated_from_default_store() {
let _guard = HomeGuard::new();

View file

@ -3,6 +3,8 @@ use serde_json::{Map, Value, json};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
const MAX_TRACE_FILE_BYTES: u64 = 64 * 1024 * 1024;
#[derive(Debug, Clone, Default)]
pub struct TraceConfig {
path: Option<PathBuf>,
@ -79,6 +81,7 @@ fn open_trace_file(path: &Path) -> Result<std::fs::File, AppError> {
}
fn write_event(file: &mut std::fs::File, event: &str, fields: Value) -> Result<(), AppError> {
reject_oversized_trace(file)?;
let mut body = Map::new();
body.insert("event".to_string(), json!(event));
body.insert(
@ -100,6 +103,17 @@ fn write_event(file: &mut std::fs::File, event: &str, fields: Value) -> Result<(
file.write_all(b"\n").map_err(AppError::from)
}
fn reject_oversized_trace(file: &std::fs::File) -> Result<(), AppError> {
let len = file.metadata()?.len();
if len < MAX_TRACE_FILE_BYTES {
return Ok(());
}
Err(AppError::invalid_input_with_suggestion(
"Trace file reached the maximum supported size",
"Start a new --trace file or rotate the existing trace before retrying.",
))
}
#[cfg(unix)]
fn reject_loose_trace_permissions(file: &std::fs::File) -> Result<(), AppError> {
use std::os::unix::fs::PermissionsExt;
@ -145,7 +159,6 @@ fn is_sensitive_trace_key(key: &str) -> bool {
"expected",
"name",
"description",
"message",
"label",
"query",
"secret",
@ -208,4 +221,69 @@ mod tests {
let _ = std::fs::remove_file(&link);
let _ = std::fs::remove_file(&target);
}
#[test]
fn trace_redacts_sensitive_fields_but_preserves_messages() {
let value = sanitize_trace_value(json!({
"text": "secret",
"message": "Target is not actionable: supported_action failed",
"details": { "name": "Private Button" },
"title": "Window"
}));
assert_eq!(value["text"]["redacted"], true);
assert_eq!(value["details"]["name"]["redacted"], true);
assert_eq!(value["title"]["redacted"], true);
assert_eq!(
value["message"],
"Target is not actionable: supported_action failed"
);
}
#[test]
fn trace_redaction_covers_nested_shapes_and_substring_keys() {
let value = sanitize_trace_value(json!({
"action": {
"typed_text": ["secret", "another"],
"api_token": {"kind": "bearer"},
"password": null,
"counter": 3
}
}));
assert_eq!(value["action"]["typed_text"]["redacted"], true);
assert_eq!(value["action"]["typed_text"]["items"], 2);
assert_eq!(value["action"]["api_token"]["redacted"], true);
assert_eq!(value["action"]["api_token"]["keys"], 1);
assert!(value["action"]["password"].is_null());
assert_eq!(value["action"]["counter"], 3);
assert_eq!(char_count_bucket(0), "0");
assert_eq!(char_count_bucket(8), "1-8");
assert_eq!(char_count_bucket(65), "33-128");
}
#[test]
fn trace_write_rejects_files_at_size_cap() {
let path = std::env::temp_dir().join(format!(
"agent-desktop-trace-cap-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let file = std::fs::File::create(&path).unwrap();
file.set_len(MAX_TRACE_FILE_BYTES).unwrap();
drop(file);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
}
let mut file = open_trace_file(&path).unwrap();
let err = write_event(&mut file, "event", json!({})).unwrap_err();
assert_eq!(err.code(), "INVALID_ARGS");
let _ = std::fs::remove_file(path);
}
}

View file

@ -65,8 +65,9 @@ mod imp {
);
}
if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) {
return Err(AdapterError::policy_denied(
return Err(AdapterError::policy_denied_for_policy(
"Physical click fallback is disabled by the current interaction policy",
policy,
));
}
let label = step_label(step);
@ -248,22 +249,25 @@ mod imp {
target: &str,
deadline: Option<Instant>,
) -> Result<bool, AdapterError> {
let target: f64 = match target.parse() {
Ok(t) => t,
Err(_) => return Ok(false),
const MAX_INCREMENT_STEPS: usize = 1024;
let target = match finite_target(target) {
Some(target) => target,
None => return Ok(false),
};
let read = || crate::tree::copy_value_typed(el).and_then(|v| v.parse::<f64>().ok());
let mut current = match read() {
Some(c) => c,
None => return Ok(false),
};
if !ax_helpers::has_ax_action(el, "AXIncrement")
&& !ax_helpers::has_ax_action(el, "AXDecrement")
let actions = ax_helpers::list_ax_actions(el);
if !actions.iter().any(|action| action == "AXIncrement")
&& !actions.iter().any(|action| action == "AXDecrement")
{
return Ok(false);
}
let start = current;
for _ in 0..1024 {
for _ in 0..MAX_INCREMENT_STEPS {
if (current - target).abs() < 0.5 {
return Ok(true);
}
@ -285,7 +289,19 @@ mod imp {
_ => break,
}
}
Ok((current - target).abs() < 0.5)
if (current - target).abs() < 0.5 {
return Ok(true);
}
if (current - start).abs() >= f64::EPSILON {
return Err(chain_verify::increment_step_limit_error(
start, current, target,
));
}
Ok(false)
}
fn finite_target(target: &str) -> Option<f64> {
target.parse::<f64>().ok().filter(|value| value.is_finite())
}
fn set_bool_verified(el: &AXElement, attr: &str, value: bool) -> Result<bool, AdapterError> {
@ -296,6 +312,20 @@ mod imp {
crate::tree::copy_bool_attr(el, attr),
))
}
#[cfg(test)]
mod tests {
use super::finite_target;
#[test]
fn finite_target_rejects_non_finite_numbers() {
assert_eq!(finite_target("42.5"), Some(42.5));
assert_eq!(finite_target("NaN"), None);
assert_eq!(finite_target("inf"), None);
assert_eq!(finite_target("-inf"), None);
assert_eq!(finite_target("not-a-number"), None);
}
}
}
#[cfg(not(target_os = "macos"))]

View file

@ -186,7 +186,10 @@ mod imp {
.any(|value| {
let matches = selected_items_text(&value);
if matches {
tracing::debug!("selected-items menu: matched control text {value:?}");
tracing::debug!(
text_chars = value.chars().count(),
"selected-items menu: matched control text"
);
}
matches
})

View file

@ -110,7 +110,11 @@ mod imp {
std::thread::sleep(std::time::Duration::from_millis(150));
let final_val = crate::tree::copy_string_attr(&owner, "AXValue");
if final_val.as_deref() != Some(label.as_str()) {
tracing::debug!("value_relay: reverted to {final_val:?}, expected {label:?}");
tracing::debug!(
observed_chars = final_val.as_deref().map(|value| value.chars().count()),
expected_chars = label.chars().count(),
"value_relay: value reverted"
);
}
Ok(final_val.as_deref() == Some(label.as_str()))
}

View file

@ -19,6 +19,20 @@ pub(crate) fn increment_deadline_error(start: f64, current: f64, target: f64) ->
}))
}
pub(crate) fn increment_step_limit_error(start: f64, current: f64, target: f64) -> AdapterError {
AdapterError::timeout("Chain step limit was reached while stepping the value toward the target")
.with_suggestion(
"Re-read the element value before retrying; the control may expose a step size too small for the requested target.",
)
.with_details(serde_json::json!({
"kind": "chain_step_limit",
"value_before": start,
"value_at_limit": current,
"target": target,
"mutated": (current - start).abs() >= f64::EPSILON,
}))
}
/// Error for the chain deadline truncating a disclosure settle wait. The
/// triggering action may still land after the truncated wait, so the
/// outcome is unknown — TIMEOUT with the observed state, mirroring
@ -72,7 +86,10 @@ fn numbers_match(expected: &str, observed: Option<&str>) -> bool {
#[cfg(test)]
mod tests {
use super::{bool_write_had_effect, dynamic_write_had_effect, increment_deadline_error};
use super::{
bool_write_had_effect, dynamic_write_had_effect, increment_deadline_error,
increment_step_limit_error,
};
#[test]
fn increment_deadline_error_is_timeout_and_reports_partial_mutation() {
@ -96,6 +113,17 @@ mod tests {
assert_eq!(details["kind"], "chain_deadline");
}
#[test]
fn increment_step_limit_error_reports_partial_mutation() {
let err = increment_step_limit_error(0.0, 1024.0, 5000.0);
assert_eq!(err.code, agent_desktop_core::error::ErrorCode::Timeout);
let details = err.details.unwrap();
assert_eq!(details["kind"], "chain_step_limit");
assert_eq!(details["value_at_limit"], 1024.0);
assert_eq!(details["mutated"], true);
}
#[test]
fn disclosure_deadline_error_is_timeout_with_observed_state() {
let err = super::disclosure_deadline_error(true, Some(false));

View file

@ -24,8 +24,9 @@ mod imp {
policy: InteractionPolicy,
) -> Result<(), AdapterError> {
if !policy.allow_cursor_move || !policy.allow_focus_steal {
return Err(AdapterError::policy_denied(
return Err(AdapterError::policy_denied_for_policy(
"Physical click fallback is disabled by the current interaction policy",
policy,
));
}
if let Some(pid) = crate::system::app_ops::pid_from_element(el) {

View file

@ -128,8 +128,9 @@ pub(crate) fn ax_scroll(
}
if policy.allow_focus_steal && !policy.allow_cursor_move {
return Err(AdapterError::policy_denied(
return Err(AdapterError::policy_denied_for_policy(
"Cursor-moving scroll fallback is disabled by the current interaction policy",
policy,
));
}

View file

@ -49,8 +49,8 @@ impl Drop for ClipboardRestore {
fn type_via_clipboard_paste(el: &AXElement, text: &str) -> Result<(), AdapterError> {
let before = readable_value(el);
let previous = crate::input::clipboard::get().unwrap_or_default();
crate::input::clipboard::set(text)?;
let _restore = ClipboardRestore { previous };
crate::input::clipboard::set(text)?;
std::thread::sleep(std::time::Duration::from_millis(50));
let combo = agent_desktop_core::action::KeyCombo {
key: "v".into(),

View file

@ -38,18 +38,10 @@ mod imp {
tracing::debug!("clipboard: get");
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"stringForType:".as_ptr());
let send: unsafe extern "C" fn(Id, Sel, Id) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let ns_string = send(pb, sel, NSPasteboardTypeString);
if ns_string.is_null() {
let Some(result) = read_string(pb) else {
tracing::debug!("clipboard: get -> empty");
return Ok(String::new());
}
let cf_str = core_foundation::string::CFString::wrap_under_get_rule(
ns_string as core_foundation_sys::string::CFStringRef,
);
let result = cf_str.to_string();
};
tracing::debug!("clipboard: get -> {} chars", result.len());
Ok(result)
}
@ -59,18 +51,10 @@ mod imp {
tracing::debug!("clipboard: set {} chars", text.len());
unsafe {
let pb = pasteboard()?;
let clear_sel = sel_registerName(c"clearContents".as_ptr());
let send_void: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send_void(pb, clear_sel);
let cf_text = core_foundation::string::CFString::new(text);
let ns_text = cf_text.as_concrete_TypeRef() as Id;
let set_sel = sel_registerName(c"setString:forType:".as_ptr());
let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool =
std::mem::transmute(objc_msgSend as *const c_void);
let ok = send_two(pb, set_sel, ns_text, NSPasteboardTypeString);
if !ok {
let previous = read_string(pb);
clear_pasteboard(pb);
if !write_string(pb, text) {
restore_after_failed_set(pb, previous.as_deref());
return Err(AdapterError::internal(
"NSPasteboard setString:forType: failed",
));
@ -83,13 +67,57 @@ mod imp {
tracing::debug!("clipboard: clear");
unsafe {
let pb = pasteboard()?;
let sel = sel_registerName(c"clearContents".as_ptr());
let send: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send(pb, sel);
clear_pasteboard(pb);
Ok(())
}
}
unsafe fn read_string(pb: Id) -> Option<String> {
unsafe {
let sel = sel_registerName(c"stringForType:".as_ptr());
let send: unsafe extern "C" fn(Id, Sel, Id) -> Id =
std::mem::transmute(objc_msgSend as *const c_void);
let ns_string = send(pb, sel, NSPasteboardTypeString);
if ns_string.is_null() {
return None;
}
let cf_str = core_foundation::string::CFString::wrap_under_get_rule(
ns_string as core_foundation_sys::string::CFStringRef,
);
Some(cf_str.to_string())
}
}
unsafe fn clear_pasteboard(pb: Id) {
unsafe {
let clear_sel = sel_registerName(c"clearContents".as_ptr());
let send_void: unsafe extern "C" fn(Id, Sel) =
std::mem::transmute(objc_msgSend as *const c_void);
send_void(pb, clear_sel);
}
}
unsafe fn write_string(pb: Id, text: &str) -> bool {
unsafe {
let cf_text = core_foundation::string::CFString::new(text);
let ns_text = cf_text.as_concrete_TypeRef() as Id;
let set_sel = sel_registerName(c"setString:forType:".as_ptr());
let send_two: unsafe extern "C" fn(Id, Sel, Id, Id) -> bool =
std::mem::transmute(objc_msgSend as *const c_void);
send_two(pb, set_sel, ns_text, NSPasteboardTypeString)
}
}
unsafe fn restore_after_failed_set(pb: Id, previous: Option<&str>) {
unsafe {
match previous {
Some(previous) => {
let _ = write_string(pb, previous);
}
None => clear_pasteboard(pb),
}
}
}
}
#[cfg(not(target_os = "macos"))]

View file

@ -62,9 +62,16 @@ mod imp {
let duration_ms = params.duration_ms.unwrap_or(300);
let steps = (duration_ms / DWELL_TICK_MS).max(4) as usize;
let step_delay = Duration::from_millis(duration_ms / steps as u64);
let source = event_source()?;
post_event(CGEventType::LeftMouseDown, from, CGMouseButton::Left)?;
post_event_with_source(
&source,
CGEventType::LeftMouseDown,
from,
CGMouseButton::Left,
)?;
let mut release = MouseUpGuard {
source: &source,
origin: from,
armed: true,
};
@ -74,7 +81,8 @@ mod imp {
let t = i as f64 / steps as f64;
let x = params.from.x + (params.to.x - params.from.x) * t;
let y = params.from.y + (params.to.y - params.from.y) * t;
post_event(
post_event_with_source(
&source,
CGEventType::LeftMouseDragged,
CGPoint::new(x, y),
CGMouseButton::Left,
@ -83,6 +91,7 @@ mod imp {
}
dwell_over_destination(
&source,
to,
params.drop_delay_ms.unwrap_or(DEFAULT_DROP_DELAY_MS),
DWELL_TICK_MS,
@ -103,28 +112,40 @@ mod imp {
/// and the cursor wherever the last successful event put it), and a
/// drop target under the origin still sees a self-drop, which most
/// targets treat as a no-op.
struct MouseUpGuard {
struct MouseUpGuard<'a> {
source: &'a CGEventSource,
origin: CGPoint,
armed: bool,
}
impl MouseUpGuard {
impl MouseUpGuard<'_> {
fn release_at(&mut self, point: CGPoint) -> Result<(), AdapterError> {
post_event(CGEventType::LeftMouseUp, point, CGMouseButton::Left)?;
post_event_with_source(
self.source,
CGEventType::LeftMouseUp,
point,
CGMouseButton::Left,
)?;
self.armed = false;
Ok(())
}
}
impl Drop for MouseUpGuard {
impl Drop for MouseUpGuard<'_> {
fn drop(&mut self) {
if self.armed {
let _ = post_event(
let _ = post_event_with_source(
self.source,
CGEventType::LeftMouseDragged,
self.origin,
CGMouseButton::Left,
);
let _ = post_event(CGEventType::LeftMouseUp, self.origin, CGMouseButton::Left);
let _ = post_event_with_source(
self.source,
CGEventType::LeftMouseUp,
self.origin,
CGMouseButton::Left,
);
}
}
}
@ -135,6 +156,7 @@ mod imp {
/// drop rather than a bare drag — macOS targets can drop the highlight if
/// no movement arrives. A zero delay still posts one settling event.
fn dwell_over_destination(
source: &CGEventSource,
to: CGPoint,
drop_delay_ms: u64,
tick_ms: u64,
@ -144,7 +166,12 @@ mod imp {
let ticks = drop_delay_ms.div_ceil(tick_ms).max(1);
for _ in 0..ticks {
post_event(CGEventType::LeftMouseDragged, to, CGMouseButton::Left)?;
post_event_with_source(
source,
CGEventType::LeftMouseDragged,
to,
CGMouseButton::Left,
)?;
sleep(Duration::from_millis(tick_ms));
}
Ok(())
@ -192,12 +219,25 @@ mod imp {
point: CGPoint,
button: CGMouseButton,
) -> Result<CGEvent, AdapterError> {
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|()| AdapterError::internal("Failed to create CGEventSource"))?;
CGEvent::new_mouse_event(source, event_type, point, button)
let source = event_source()?;
create_event_with_source(&source, event_type, point, button)
}
fn create_event_with_source(
source: &CGEventSource,
event_type: CGEventType,
point: CGPoint,
button: CGMouseButton,
) -> Result<CGEvent, AdapterError> {
CGEvent::new_mouse_event(source.clone(), event_type, point, button)
.map_err(|()| AdapterError::internal("CGEvent::new_mouse_event failed"))
}
fn event_source() -> Result<CGEventSource, AdapterError> {
CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.map_err(|()| AdapterError::internal("Failed to create CGEventSource"))
}
fn post_event(
event_type: CGEventType,
point: CGPoint,
@ -208,6 +248,17 @@ mod imp {
Ok(())
}
fn post_event_with_source(
source: &CGEventSource,
event_type: CGEventType,
point: CGPoint,
button: CGMouseButton,
) -> Result<(), AdapterError> {
let ev = create_event_with_source(source, event_type, point, button)?;
ev.post(CGEventTapLocation::HID);
Ok(())
}
fn to_cg_button(button: &MouseButton) -> CGMouseButton {
match button {
MouseButton::Left => CGMouseButton::Left,

View file

@ -1,4 +1,9 @@
use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo};
use agent_desktop_core::{
adapter::WindowFilter,
error::{AdapterError, ErrorCode},
node::WindowInfo,
};
use std::time::Duration;
#[cfg(target_os = "macos")]
pub fn pid_from_element(el: &crate::tree::AXElement) -> Option<i32> {
@ -72,29 +77,7 @@ pub fn focus_window_impl(win: &WindowInfo) -> Result<(), AdapterError> {
win.app,
win.title
);
use accessibility_sys::{AXUIElementCreateApplication, AXUIElementSetAttributeValue};
use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString};
let app_el = crate::tree::AXElement(unsafe { AXUIElementCreateApplication(win.pid) });
if app_el.0.is_null() {
return Err(AdapterError::internal("Failed to create AX app element"));
}
let frontmost_attr = CFString::new("AXFrontmost");
let err = unsafe {
AXUIElementSetAttributeValue(
app_el.0,
frontmost_attr.as_concrete_TypeRef(),
CFBoolean::true_value().as_CFTypeRef(),
)
};
if err != accessibility_sys::kAXErrorSuccess {
return Err(AdapterError::internal(format!(
"Failed to set AXFrontmost (err={err})"
)));
}
wait_until_frontmost(&app_el);
ensure_app_focused(win.pid)?;
let main_win = crate::tree::window_element_for(win.pid, &win.title);
crate::system::window_ops::raise_window(&main_win);
Ok(())
@ -225,7 +208,15 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> {
if force {
let mut command = Command::new("/usr/bin/pkill");
command.arg("-x").arg(id);
crate::system::process::run_with_timeout(&mut command, "pkill", QUIT_TIMEOUT)?;
let output = crate::system::process::run_with_timeout(&mut command, "pkill", QUIT_TIMEOUT)?;
if !output.status.success() {
return Err(AdapterError::new(
ErrorCode::AppNotFound,
format!("App '{id}' was not running or could not be matched for force close"),
)
.with_suggestion("Use 'list-apps' to verify the running app name before retrying."));
}
wait_until_app_exits(id, QUIT_TIMEOUT)?;
} else {
let pid = crate::system::key_dispatch::find_pid_by_name(id)?;
let app_ax = crate::tree::element_for_pid(pid);
@ -255,6 +246,25 @@ end tell"#
Ok(())
}
#[cfg(target_os = "macos")]
fn wait_until_app_exits(id: &str, timeout: Duration) -> Result<(), AdapterError> {
use std::time::Instant;
let start = Instant::now();
while start.elapsed() <= timeout {
if crate::system::app_list::pid_for_app_name(id).is_none() {
return Ok(());
}
std::thread::sleep(Duration::from_millis(50));
}
Err(
AdapterError::timeout(format!("App '{id}' did not terminate after force close"))
.with_suggestion(
"Retry after checking for save dialogs or helper processes with 'list-apps'.",
),
)
}
#[cfg(target_os = "macos")]
fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool {
use accessibility_sys::{AXUIElementPerformAction, kAXErrorSuccess};

View file

@ -43,7 +43,8 @@ pub(super) fn candidate_roots(
prepare_for_read(&root, deadline)?;
let mut roots = Vec::new();
let mut dedupe = ElementDedupe;
if let Some(window) = exact_source_window_root(entry, deadline)? {
let windows = copy_ax_array(&root, "AXWindows").unwrap_or_default();
if let Some(window) = exact_source_window_from_windows(&windows, entry, deadline) {
dedupe.push(&mut roots, window);
}
prepare_for_read(&root, deadline)?;
@ -54,8 +55,7 @@ pub(super) fn candidate_roots(
if let Some(main) = copy_element_attr(&root, "AXMainWindow") {
dedupe.push(&mut roots, main);
}
prepare_for_read(&root, deadline)?;
for window in copy_ax_array(&root, "AXWindows").unwrap_or_default() {
for window in windows {
dedupe.push(&mut roots, window);
}
ensure_before_deadline(deadline)?;
@ -150,14 +150,19 @@ fn exact_source_window_root(
let Some(windows) = windows_for_pid(entry.pid, deadline)? else {
return Ok(None);
};
if let Some(window) = window_by_number(&windows, source_window_number(entry), deadline) {
return Ok(Some(window));
Ok(exact_source_window_from_windows(&windows, entry, deadline))
}
#[cfg(target_os = "macos")]
fn exact_source_window_from_windows(
windows: &[AXElement],
entry: &RefEntry,
deadline: Instant,
) -> Option<AXElement> {
if let Some(window) = window_by_number(windows, source_window_number(entry), deadline) {
return Some(window);
}
Ok(window_by_title(
&windows,
entry.source_window_title.as_deref(),
deadline,
))
window_by_title(windows, entry.source_window_title.as_deref(), deadline)
}
#[cfg(target_os = "macos")]

View file

@ -77,7 +77,7 @@ Ref actions use `ActionRequest { action, policy }`. The default `InteractionPoli
- Semantic AX steps run first.
- Physical fallbacks are explicit and policy-gated.
- Non-mouse ref commands must not silently focus apps or move the cursor.
- Raw cursor commands (`hover`, `drag`, `mouse-*`) require `--headed`; other commands must not silently focus apps or move the cursor.
- Expected OS denials return specific error codes such as `PERM_DENIED`, `SNAPSHOT_NOT_FOUND`, or `POLICY_DENIED`, not generic `INTERNAL`.
Windows and Linux should implement the same signatures rather than copying macOS-specific fallback decisions.
@ -437,7 +437,7 @@ The `ErrorCode` enum in `crates/core/src/error.rs` exposes these machine-readabl
| `INVALID_ARGS` | Input | Bad CLI argument or unknown ref format | Fix the argument per CLI help |
| `NOTIFICATION_NOT_FOUND` | Notification | Notification ID not found / NC reordered | Run 'list-notifications' to see current notifications |
| `SNAPSHOT_NOT_FOUND` | Ref | Requested snapshot ID is missing | Run 'snapshot' again and use the returned snapshot_id |
| `POLICY_DENIED` | Action policy | Physical fallback blocked by headless policy | Use an explicit focus/cursor/mouse command if physical interaction is intended |
| `POLICY_DENIED` | Action policy | Physical input blocked by headless policy | Retry with `--headed` for explicit cursor movement, or use a semantic AX action when available |
| `INTERNAL` | Internal | Unexpected error or caught panic | Re-run with verbose logging |
Exit codes: `0` success, `1` structured error (JSON on stdout), `2` argument/parse error.
@ -621,7 +621,7 @@ These items are tracked under P2-O16 below: registry migration via `build.rs` fi
### Core invariants (research-driven — from Phase 2 plan §Headless-First Invariant)
1. **Headless-first inside the active desktop session.** Every command — existing and Phase 2 — must run without an agent-desktop GUI, foreground activation, focus steal, or physical cursor movement (except for explicit mouse commands). Windows, macOS, and Linux still require the target app to exist in the current user's interactive desktop/display session for accessibility and capture APIs. Session 0, Server Core, secure desktops, locked desktops, and other-user sessions return `PLATFORM_NOT_SUPPORTED`, `PERM_DENIED`, or `WINDOW_NOT_FOUND` with `platform_detail`, not silent best effort. The invariant is enforced by integration tests: target window is NOT focused at test entry; `list-windows --focused-only` returns the same window before/after; cursor position unchanged for non-mouse commands.
1. **Headless-first inside the active desktop session.** Every command — existing and Phase 2 — must run without an agent-desktop GUI, foreground activation, focus steal, or physical cursor movement unless `--headed` explicitly opts into cursor input. Windows, macOS, and Linux still require the target app to exist in the current user's interactive desktop/display session for accessibility and capture APIs. Session 0, Server Core, secure desktops, locked desktops, and other-user sessions return `PLATFORM_NOT_SUPPORTED`, `PERM_DENIED`, or `WINDOW_NOT_FOUND` with `platform_detail`, not silent best effort. The invariant is enforced by integration tests: target window is NOT focused at test entry; `list-windows --focused-only` returns the same window before/after; cursor position unchanged for headless commands.
2. **Skeleton traversal is platform-agnostic.** The novel progressive skeleton pattern (depth-3 clamp + `children_count` annotation + drill-down via `--root @ref` + scoped invalidation via `RefMap::remove_by_root_ref`) lives entirely in `crates/core/src/snapshot_ref.rs`. Windows adapter contributes ~50 LOC glue: `ControlViewWalker` (NOT `RawViewWalker` or `ContentViewWalker`) + `FindAll(TreeScope_Children, TrueCondition)` for `children_count` + fresh `UICacheRequest` per drill-down.
3. **Asymmetric event threading.** `watch_element` uses main-thread `AXObserver` on macOS (research-confirmed: Apple DTS says all AX is main-thread-only; AXSwift / Hammerspoon / Phoenix all do this); worker-thread MTA `IUIAutomation` event handler on Windows (Microsoft 2025 threading doc: UIA supports cross-thread event delivery).
4. **No `inventory` / `linkme` command registry.** Research confirmed neither survives link-GC reliably across ld64, ld-prime, GNU ld, lld, MSVC for cdylib consumers. Phase 2 uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe, zero linker magic. The repository's "one command per file" rule becomes the codegen contract.

View file

@ -44,7 +44,7 @@ The answer is **per-gesture and per-platform**, because a gesture is headless-ca
2. **A new platform that exposes a headless path lights it up automatically — adapter-only change.** If a future Windows (UIA) or Linux (AT-SPI) adapter has a headless action for `double-click`/`triple-click`, it maps the `Action` there and the command succeeds headlessly on that platform with **zero change to the command or core**. The `InteractionPolicy` flows through the request; each adapter honors it per its own capabilities. The agent just sees success (or `POLICY_DENIED` → retry `--headed`) — it never needs to know the platform.
3. **`hover`/`drag`/`mouse-*` are modeled as raw cursor gestures, not semantic `Action`s** (they call `adapter.mouse_event`/`adapter.drag` with coordinates). They stay physical on every platform by design, because hovering/dragging *are* cursor operations universally. A semantic drag (AX reorder) would be a *new* `Action`, not a change to `drag`. When a gesture is ref-addressed, the target app's frontmost state is ensured first **only under `--headed`** (`focus_for_physical_input`, gated on `InteractionPolicy::allow_focus_steal`; the response reports `"focused": true` when confirmed — already-frontmost apps skip the raise). Headless gestures never change the frontmost app, and `--xy` input never focuses — the caller owns the target there. The chain's physical click fallback goes one step further than the gesture focus path: it also raises the target element's **own window** (AXRaise, AXMain fallback) before posting events, because CGEvents land on the topmost window at the click point and app-frontmost alone is not enough when the element lives in a background window of that app.
3. **`hover`/`drag`/`mouse-*` are modeled as raw cursor gestures, not semantic `Action`s** (they call `adapter.mouse_event`/`adapter.drag` with coordinates). They stay physical on every platform by design, because hovering/dragging *are* cursor operations universally. A semantic drag (AX reorder) would be a *new* `Action`, not a change to `drag`. Headless gestures return `POLICY_DENIED` before resolving refs or moving the cursor. Under `--headed`, ref-addressed gestures first ensure the target app is frontmost (`focus_for_physical_input`, gated on `InteractionPolicy::allow_focus_steal`; the response reports `"focused": true` when confirmed — already-frontmost apps skip the raise). `--xy` input never focuses — the caller owns the target there. The chain's physical click fallback goes one step further than the gesture focus path: it also raises the target element's **own window** (AXRaise, AXMain fallback) before posting events, because CGEvents land on the topmost window at the click point and app-frontmost alone is not enough when the element lives in a background window of that app.
4. **`POLICY_DENIED` on a headless gesture is correct, not a bug** — it is the fail-closed signal that the headless AX path is unavailable and the caller must opt into `--headed`. Never widen the default policy to make it disappear.

View file

@ -30,12 +30,12 @@ KEYBOARD
key-up <combo> Release a held key or modifier
MOUSE
hover <ref|--xy> Move cursor to element or coordinates
drag Drag from one element/point to another
mouse-move --xy x,y Move cursor to absolute coordinates
mouse-click --xy x,y Click at coordinates (--button, --count)
mouse-down --xy x,y Press mouse button at coordinates
mouse-up --xy x,y Release mouse button at coordinates
hover <ref|--xy> Move cursor to element or coordinates (requires --headed)
drag Drag from one element/point to another (requires --headed)
mouse-move --xy x,y Move cursor to absolute coordinates (requires --headed)
mouse-click --xy x,y Click at coordinates (--button, --count; requires --headed)
mouse-down --xy x,y Press mouse button at coordinates (requires --headed)
mouse-up --xy x,y Release mouse button at coordinates (requires --headed)
APP & WINDOW
launch <app> Launch app and wait until window is visible
@ -111,11 +111,11 @@ EXAMPLES
agent-desktop check @e3 --snapshot <snapshot_id>
agent-desktop type @e2 --snapshot <snapshot_id> "hello@example.com"
agent-desktop press cmd+z
agent-desktop drag --from @e1 --to @e5
agent-desktop hover @e5
agent-desktop --headed drag --from @e1 --to @e5
agent-desktop --headed hover @e5
agent-desktop minimize --app TextEdit
agent-desktop resize-window --app TextEdit --width 800 --height 600
agent-desktop mouse-click --xy 500,300
agent-desktop --headed mouse-click --xy 500,300
agent-desktop wait --text "Loading complete" --app Safari --timeout 5000
agent-desktop wait --element @e5 --predicate actionable --timeout 5000
agent-desktop --session run-a --trace /tmp/ad.jsonl click @e5

View file

@ -63,7 +63,7 @@ pub(crate) struct Cli {
#[arg(
long,
global = true,
help = "Run ref actions headed: permit cursor movement and focus stealing so physical click/scroll/keypress fallbacks can complete. Default is headless (AX-only, no cursor)."
help = "Permit cursor movement and focus stealing for physical input commands and fallbacks. Default is headless (AX-only, no cursor)."
)]
pub headed: bool,
@ -123,17 +123,17 @@ pub(crate) enum Commands {
KeyDown(KeyComboArgs),
#[command(about = "Release a held key or modifier")]
KeyUp(KeyComboArgs),
#[command(about = "Move cursor to element center or coordinates")]
#[command(about = "Move cursor to element center or coordinates (requires --headed)")]
Hover(HoverArgs),
#[command(about = "Drag from one element or point to another")]
#[command(about = "Drag from one element or point to another (requires --headed)")]
Drag(DragCliArgs),
#[command(about = "Move cursor to absolute screen coordinates")]
#[command(about = "Move cursor to absolute screen coordinates (requires --headed)")]
MouseMove(MouseMoveArgs),
#[command(about = "Click at absolute screen coordinates")]
#[command(about = "Click at absolute screen coordinates (requires --headed)")]
MouseClick(MouseClickArgs),
#[command(about = "Press mouse button at coordinates (without releasing)")]
#[command(about = "Press mouse button at coordinates (requires --headed)")]
MouseDown(MousePointArgs),
#[command(about = "Release mouse button at coordinates")]
#[command(about = "Release mouse button at coordinates (requires --headed)")]
MouseUp(MousePointArgs),
#[command(about = "Launch application and wait until its window is visible")]
Launch(LaunchArgs),

View file

@ -114,7 +114,10 @@ pub(crate) struct KeyComboArgs {
#[derive(Parser, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct HoverArgs {
#[arg(value_name = "REF", help = "Element ref to hover over")]
#[arg(
value_name = "REF",
help = "Element ref to hover over; requires --headed"
)]
pub ref_id: Option<String>,
#[arg(
long,
@ -122,7 +125,7 @@ pub(crate) struct HoverArgs {
help = "Snapshot ID returned by snapshot; omit to use active session latest"
)]
pub snapshot: Option<String>,
#[arg(long, help = "Absolute coordinates as x,y")]
#[arg(long, help = "Absolute coordinates as x,y; requires --headed")]
pub xy: Option<String>,
#[arg(long, help = "Hold hover position for N milliseconds")]
pub duration: Option<u64>,
@ -131,13 +134,21 @@ pub(crate) struct HoverArgs {
#[derive(Parser, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct DragCliArgs {
#[arg(long, help = "Source element ref")]
#[arg(long, help = "Source element ref; requires --headed")]
pub from: Option<String>,
#[arg(long, name = "from-xy", help = "Source coordinates as x,y")]
#[arg(
long,
name = "from-xy",
help = "Source coordinates as x,y; requires --headed"
)]
pub from_xy: Option<String>,
#[arg(long, help = "Destination element ref")]
#[arg(long, help = "Destination element ref; requires --headed")]
pub to: Option<String>,
#[arg(long, name = "to-xy", help = "Destination coordinates as x,y")]
#[arg(
long,
name = "to-xy",
help = "Destination coordinates as x,y; requires --headed"
)]
pub to_xy: Option<String>,
#[arg(
long,
@ -158,14 +169,14 @@ pub(crate) struct DragCliArgs {
#[derive(Parser, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct MouseMoveArgs {
#[arg(long, help = "Absolute coordinates as x,y")]
#[arg(long, help = "Absolute coordinates as x,y; requires --headed")]
pub xy: String,
}
#[derive(Parser, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct MouseClickArgs {
#[arg(long, help = "Absolute coordinates as x,y")]
#[arg(long, help = "Absolute coordinates as x,y; requires --headed")]
pub xy: String,
#[arg(
long,
@ -182,7 +193,7 @@ pub(crate) struct MouseClickArgs {
#[derive(Parser, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct MousePointArgs {
#[arg(long, help = "Absolute coordinates as x,y")]
#[arg(long, help = "Absolute coordinates as x,y; requires --headed")]
pub xy: String,
#[arg(
long,

View file

@ -193,7 +193,7 @@ pub(crate) fn dispatch(
Commands::MouseMove(a) => {
let (x, y) = parse_xy(&a.xy)?;
mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter)
mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter, context)
}
Commands::MouseClick(a) => {
@ -206,6 +206,7 @@ pub(crate) fn dispatch(
count: a.count,
},
adapter,
context,
)
}
@ -218,6 +219,7 @@ pub(crate) fn dispatch(
button: parse_mouse_button(&a.button)?,
},
adapter,
context,
)
}
@ -230,6 +232,7 @@ pub(crate) fn dispatch(
button: parse_mouse_button(&a.button)?,
},
adapter,
context,
)
}

View file

@ -2,6 +2,7 @@ use agent_desktop_core::{
action_request::ActionRequest,
action_result::ActionResult,
adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface},
capability,
element_state::ElementState,
error::{AdapterError, ErrorCode},
node::Rect,
@ -16,6 +17,7 @@ mod ref_action_contract;
struct ContractAdapter {
resolve: ResolveMode,
live_bounds: Option<Rect>,
live_value: Option<String>,
dispatches: AtomicU32,
}
@ -44,13 +46,25 @@ impl PlatformAdapter for ContractAdapter {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
value: self.live_value.clone(),
}),
bounds: self.live_bounds,
available_actions: Some(vec!["Click".into()]),
available_actions: Some(vec![capability::CLICK.into()]),
})
}
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Ok(Some(ElementState {
role: "button".into(),
states: vec![],
value: self.live_value.clone(),
}))
}
fn get_live_value(&self, _handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
Ok(self.live_value.clone())
}
fn execute_action(
&self,
_handle: &NativeHandle,
@ -66,10 +80,16 @@ impl ContractAdapter {
Self {
resolve,
live_bounds,
live_value: None,
dispatches: AtomicU32::new(0),
}
}
fn with_live_value(mut self, value: &str) -> Self {
self.live_value = Some(value.into());
self
}
fn resolve(&self) -> Result<NativeHandle, AdapterError> {
match self.resolve {
ResolveMode::Ok => Ok(NativeHandle::null()),
@ -89,7 +109,7 @@ fn entry(bounds: Rect) -> RefEntry {
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),
available_actions: vec!["Click".into()],
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
@ -181,3 +201,42 @@ fn adapter_contract_wait_element_uses_session_snapshot() {
assert_eq!(result["ref"], "@e1");
assert_eq!(result["predicate"], "exists");
}
#[test]
fn adapter_contract_wait_predicates_cover_live_state_paths() {
let bounds = Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
};
let context =
agent_desktop_core::context::CommandContext::new(Some("shared-agent".into()), None, false)
.unwrap();
let enabled = ref_action_contract::run_wait_element_command_with_predicate(
&ContractAdapter::new(ResolveMode::Ok, Some(bounds)),
entry(bounds),
&context,
ref_action_contract::WaitPredicate::new("enabled"),
)
.unwrap();
let actionable = ref_action_contract::run_wait_element_command_with_predicate(
&ContractAdapter::new(ResolveMode::Ok, Some(bounds)),
entry(bounds),
&context,
ref_action_contract::WaitPredicate::new("actionable").with_action("click"),
)
.unwrap();
let value = ref_action_contract::run_wait_element_command_with_predicate(
&ContractAdapter::new(ResolveMode::Ok, Some(bounds)).with_live_value("ready"),
entry(bounds),
&context,
ref_action_contract::WaitPredicate::new("value").with_value("ready"),
)
.unwrap();
assert_eq!(enabled["observed"]["enabled"], true);
assert_eq!(actionable["observed"]["actionable"], true);
assert_eq!(value["observed"]["matched"], true);
}

View file

@ -31,7 +31,16 @@ pub fn run_wait_element_command(
entry: RefEntry,
context: &CommandContext,
) -> Result<serde_json::Value, agent_desktop_core::AppError> {
with_saved_entry(entry, context, |snapshot_id| {
run_wait_element_command_with_predicate(adapter, entry, context, WaitPredicate::new("exists"))
}
pub fn run_wait_element_command_with_predicate(
adapter: &dyn PlatformAdapter,
entry: RefEntry,
context: &CommandContext,
predicate: WaitPredicate<'_>,
) -> Result<serde_json::Value, agent_desktop_core::AppError> {
with_saved_entry(entry, context, |_| {
wait::execute(
wait::WaitArgs {
mode: wait::WaitModeArgs {
@ -44,10 +53,10 @@ pub fn run_wait_element_command(
notification: false,
},
predicate: wait::WaitPredicateArgs {
snapshot_id: Some(snapshot_id),
predicate: Some("exists".into()),
value: None,
action: None,
snapshot_id: None,
predicate: Some(predicate.name.into()),
value: predicate.value.map(String::from),
action: predicate.action.map(String::from),
count: None,
},
timeout_ms: 100,
@ -59,6 +68,32 @@ pub fn run_wait_element_command(
})
}
pub struct WaitPredicate<'a> {
name: &'a str,
value: Option<&'a str>,
action: Option<&'a str>,
}
impl<'a> WaitPredicate<'a> {
pub fn new(name: &'a str) -> Self {
Self {
name,
value: None,
action: None,
}
}
pub fn with_value(mut self, value: &'a str) -> Self {
self.value = Some(value);
self
}
pub fn with_action(mut self, action: &'a str) -> Self {
self.action = Some(action);
self
}
}
fn with_saved_entry<T>(
entry: RefEntry,
context: &CommandContext,

View file

@ -1,16 +1,4 @@
#!/usr/bin/env bash
# End-to-end tests: drive the real agent-desktop binary against the fixture
# app and verify real OS outcomes by INDEPENDENT OBSERVATION — never the
# command's own ok:true. EVERY check prints the values it observed (before/
# after, got/expected, the actual JSON fields) inline, so any failure is
# debuggable from the output alone and a command that returns ok:true without
# an effect is caught.
#
# The fixture (tests/fixture-app) is a fixed, diverse slice of real macOS UI.
# It is never tuned to make a command pass; a failure here is a finding about
# the CLI or this harness.
#
# Usage: tests/e2e/run.sh (needs: cargo build --release + AX permission)
set -uo pipefail
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
@ -18,24 +6,16 @@ bin="$repo/target/release/agent-desktop"
fixture_app="$repo/tests/fixture-app/build/AgentDeskFixture.app"
app="AgentDeskFixture"
pass=0; fail=0; declare -a failures
pass=0; skip_count=0; fail=0; declare -a failures
note() { printf '\n\033[1;34m== %s ==\033[0m\n' "$1"; }
okmsg() { printf ' \033[0;32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); }
badmsg(){ printf ' \033[0;31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); failures+=("$1"); }
skip() { printf ' \033[0;33mNOTE\033[0m %s\n' "$1"; }
# assert <label> <pass?0|1> <detail-with-observed-values>
skip() { printf ' \033[0;33mNOTE\033[0m %s\n' "$1"; skip_count=$((skip_count+1)); }
assert() { if [ "$2" = "1" ]; then okmsg "$1 [$3]"; else badmsg "$1 [$3]"; fi; }
# Interaction mode is Playwright-style: headless is the default (AX-only, no
# cursor), --headed opts in to cursor/physical fallbacks. Observations
# (resolve/read_value/snapshot) are ALWAYS headless; only the action under test
# carries $MODE_FLAG, set per-suite so every interaction is proven in BOTH modes.
MODE_FLAG=""
act() { "$bin" $MODE_FLAG "$@"; }
# timed <label> <cmd...> : run the command, append its wall-clock ms to
# $PERF_FILE, and echo stdout. Timing brackets only the subprocess (perf_counter
# inside python), so the python interpreter's own startup is excluded.
PERF_FILE="$(mktemp -t agentdesk-perf.XXXXXX)"
timed() {
python3 - "$PERF_FILE" "$@" <<'PY'
@ -60,8 +40,6 @@ resolve() { "$bin" find --app "$app" --role "$1" --name "$2" --first 2>/dev/n
read_value() { "$bin" find --app "$app" --role statictext --name "$1" --first 2>/dev/null | field "['data']['match']['value']"; }
running() { "$bin" list-apps 2>/dev/null | python3 -c "import json,sys;print(any(a['name']=='$app' for a in json.load(sys.stdin)['data']['apps']))" 2>/dev/null; }
# verify <label> <status-name> <expected> <subcmd...> : observe before, run the
# subcommand in the CURRENT $MODE_FLAG (no leading $bin), observe after.
verify() {
local label="$1" status="$2" expected="$3"; shift 3
local before after out cmd_ok err
@ -74,10 +52,6 @@ verify() {
"before='$before' after='$after' expected='$expected' cmd_ok=$cmd_ok${err:+ err=$err}"
}
# interaction_suite <headless|headed> : drive every ref-action command in the
# given mode with mode-specific target values, so a regression in EITHER mode is
# caught by an independent before/after observation. Headed must not regress the
# AX path (it is tried first); it only adds cursor/physical fallbacks.
interaction_suite() {
local MODE="$1" MODE_FLAG="" sel sld stp dir
if [ "$MODE" = headed ]; then MODE_FLAG="--headed"; sel=Gamma; sld=60; stp=6; dir=up
@ -121,7 +95,6 @@ interaction_suite() {
cleanup() { "$bin" close-app "$app" --force >/dev/null 2>&1 || true; rm -f "$PERF_FILE"; }
trap cleanup EXIT
# --- Setup -----------------------------------------------------------------
note "Setup"
[ -x "$bin" ] || { echo "release binary missing; run 'cargo build --release'"; exit 2; }
ax_state="$("$bin" permissions 2>/dev/null | field "['data']['accessibility']['state']")"
@ -130,9 +103,6 @@ if [ "$ax_state" != "granted" ]; then
echo "Grant Accessibility trust to this terminal/runner in System Settings > Privacy & Security, then retry." >&2
exit 2
fi
# Rebuild when the bundle is missing OR any fixture source is newer than it, so
# the suite never silently exercises a stale fixture. Keep swiftc stderr
# visible: a failed build must abort loudly, not cascade into launch noise.
fixture_stale() {
[ ! -d "$fixture_app" ] ||
[ -n "$(find "$repo/tests/fixture-app" -maxdepth 1 -name '*.swift' -newer "$fixture_app" 2>/dev/null)" ]
@ -154,7 +124,6 @@ for _ in $(seq 1 20); do
done
assert "fixture launched and tree exposed" "$ready" "primary-button resolvable after $tries focus attempts"
# --- Observation -----------------------------------------------------------
note "snapshot role diversity (observed roles vs expected)"
snap="$("$bin" snapshot --app "$app" --max-depth 30 2>/dev/null)"
refc="$(echo "$snap" | field "['data']['ref_count']")"
@ -172,11 +141,6 @@ assert "textarea alias -> textfield" "$([ -n "$ta" ] && echo 1 || echo 0)" "alia
hint="$("$bin" find --app "$app" --role navbar 2>/dev/null | field "['data']['roles_present']")"
assert "absent role returns roles_present hint" "$([ -n "$hint" ] && echo 1 || echo 0)" "roles_present=${hint:0:60}..."
# --- Interaction in BOTH modes (the headed/headless contract) ---------------
# Every ref-action command is driven twice — default headless, then --headed —
# and verified by independent before/after observation each time. This is the
# most important guarantee: actions must work in both modes, and --headed must
# not regress the AX path.
interaction_suite headless
interaction_suite headed
@ -229,12 +193,6 @@ else
assert "--headed hover triggers onHover" 0 "hover-target bounds not found in snapshot (no xy to hover)"
fi
# --- Strict resolution -----------------------------------------------------
# Two buttons share role+name "twin-control" and each records a distinct effect
# (twin-a / twin-b). `--first` addresses the first in document order, which the
# resolver must map to twin-a by path. Honest contract: it either fires the
# ADDRESSED twin or fails closed with AMBIGUOUS_TARGET — but must NEVER silently
# fire the other twin or no-op while claiming success.
note "strict resolution acts on the addressed twin, never the other"
"$bin" focus-window --app "$app" >/dev/null 2>&1
tw="$(resolve button twin-control)"
@ -269,7 +227,6 @@ case "$code" in
*) assert "removed failed closed" 0 "ref='$stale' code='$code' ok=$ok (acted on removed element!)" ;;
esac
# --- Reliability core (observed values inline) -----------------------------
note "wait predicates"
"$bin" click "$(resolve button enable-later)" >/dev/null 2>&1
we="$("$bin" wait --element "$(resolve button delayed-button)" --predicate enabled --timeout 5000 2>/dev/null)"
@ -308,9 +265,6 @@ assert "skeleton + drill-down" "$([ -n "$anchor" ] && [ -n "$drilled" ] && [ "$d
"skeleton_refs=$sk_refs anchor='$anchor' drilled_refs='$drilled'"
note "session isolation + session-independent explicit snapshot"
# Take run-a's snapshot once and pull a ref FROM THAT SAME tree, so the ref is
# guaranteed to live in snapshot sa's refmap. (Resolving a ref from a separate
# find snapshot against sa is racy on any UI that re-renders between calls.)
sa_out="$("$bin" --session run-a snapshot --app "$app" 2>/dev/null)"
sa="$(echo "$sa_out" | field "['data']['snapshot_id']")"
sb="$("$bin" --session run-b snapshot --app "$app" 2>/dev/null | field "['data']['snapshot_id']")"
@ -337,7 +291,6 @@ assert "trace recorded resolver events" "$has_events" "bytes=$bytes resolver_eve
assert "typed secret NOT in trace" "$([ "$leaked" = "0" ] && echo 1 || echo 0)" "secret_present=$([ "$leaked" = 1 ] && echo YES || echo no) redaction_markers=$([ "$redacted" = 1 ] && echo yes || echo no)"
rm -f "$trf"
# --- Surfaces / drag / expand (before/after) -------------------------------
note "surface: open sheet, list-surfaces sees it, act inside"
sheet_b="$(read_value sheet-status)"
"$bin" click "$(resolve button open-sheet)" >/dev/null 2>&1; sleep 0.6
@ -381,13 +334,11 @@ def f(n):
b=f(d['data']['tree'])
print(f\"{b['x']+20},{b['y']+b['height']/2} {b['x']+b['width']-20},{b['y']+b['height']/2}\" if b else '')" 2>/dev/null)"
from_xy="$(echo "$cxy" | awk '{print $1}')"; to_xy="$(echo "$cxy" | awk '{print $2}')"
"$bin" drag --from-xy "$from_xy" --to-xy "$to_xy" --duration 400 --drop-delay 300 >/dev/null 2>&1; sleep 0.4
"$bin" --headed drag --from-xy "$from_xy" --to-xy "$to_xy" --duration 400 --drop-delay 300 >/dev/null 2>&1; sleep 0.4
dr_a="$(read_value drag-canvas-status)"
assert "drag delivered a gesture" "$(echo "$dr_a" | grep -q '^dragged-' && echo 1 || echo 0)" "from='$from_xy' to='$to_xy' canvas before='$dr_b' after='$dr_a'"
note "expand a press-toggled disclosure (verified by disclosure value)"
# Force-collapse first so the expand below proves a real state flip — an
# already-expanded disclosure would make "expand succeeded" vacuous.
"$bin" collapse "$(resolve disclosure disclosure-section)" >/dev/null 2>&1; sleep 0.4
exp_b="$("$bin" get "$(resolve disclosure disclosure-section)" 2>/dev/null | field "['data']['value']")"
eout="$("$bin" expand "$(resolve disclosure disclosure-section)" 2>&1)"; sleep 0.4
@ -403,7 +354,6 @@ else
assert "expand flipped disclosure from collapsed to expanded" 0 "claimed success but value before='$exp_b' after='$exp_a' cmd_ok=$eok"
fi
# --- Performance (CLI wall-clock per operation) ----------------------------
note "performance (CLI execution time per operation, ms)"
"$bin" focus-window --app "$app" >/dev/null 2>&1
pbref="$(resolve button primary-button)"; tfref="$(resolve textfield text-input)"
@ -434,12 +384,11 @@ skip "cross-target native drag-and-drop (onDrop) needs the OS dragging-session/p
skip "SwiftUI Slider/Stepper/DisclosureGroup are not AX-actionable; native AppKit equivalents are (set-value/expand work on those)"
skip "SwiftUI CommandMenu (menu-bar) items accept AXPress (verified_press succeeds) but do not route to their action closure, so firing a custom top-menu item has no effect; native AppKit menu items DO fire via AX. Menu-bar enumeration (--surface menubar), menu opening, and .contextMenu item selection all work."
# --- Summary ---------------------------------------------------------------
note "Summary"
printf ' passed: %d failed: %d\n' "$pass" "$fail"
printf ' passed: %d skipped: %d failed: %d\n' "$pass" "$skip_count" "$fail"
if [ "$fail" -gt 0 ]; then
printf '\n failures (observed values inline above):\n'
for f in "${failures[@]}"; do printf ' - %s\n' "$f"; done
exit 1
fi
echo " all E2E scenarios passed"
echo " all asserted E2E scenarios passed"

View file

@ -51,6 +51,7 @@ struct ContentView: View {
@State private var showSheet = false
@State private var showPopover = false
@State private var sheetStatus = "idle"
@State private var sheetFieldValue = ""
var body: some View {
ScrollView([.vertical, .horizontal]) {
@ -332,7 +333,7 @@ struct ContentView: View {
private var sheetContent: some View {
VStack(spacing: 12) {
Text("Sheet Title").font(.headline).accessibilityLabel("sheet-title")
TextField("Sheet Field", text: .constant("")).accessibilityLabel("sheet-field").frame(width: 200)
TextField("Sheet Field", text: $sheetFieldValue).accessibilityLabel("sheet-field").frame(width: 200)
Button("Confirm Sheet") { sheetStatus = "confirmed"; showSheet = false }
.accessibilityLabel("confirm-sheet")
Button("Cancel Sheet") { sheetStatus = "cancelled"; showSheet = false }