From ec247a6f7dadda6083ab63e73451ea13e2e13876 Mon Sep 17 00:00:00 2001 From: Lahfir Date: Sun, 12 Jul 2026 18:47:01 -0700 Subject: [PATCH] fix: enforce explicit headed delivery Keep semantic ref actions strictly headless, make natural headed actions physical-first, and prevent notification observation from opening system UI without headed permission. Gate the native suite on background non-interference and assert delivered mechanisms. --- README.md | 19 +++--- crates/core/src/action.rs | 5 +- crates/core/src/action_tests.rs | 12 ++-- crates/core/src/actionability/evaluate.rs | 2 +- crates/core/src/actionability/requirements.rs | 8 ++- .../src/actionability/requirements_tests.rs | 22 ++++++- crates/core/src/adapter/system.rs | 10 ++-- crates/core/src/commands/execute_by_ref.rs | 10 +--- .../core/src/commands/execute_by_ref_tests.rs | 2 +- .../core/src/commands/list_notifications.rs | 9 ++- crates/core/src/commands/ref_policy_tests.rs | 15 ++--- crates/core/src/commands/wait.rs | 5 +- crates/core/src/commands/wait_predicate.rs | 4 +- .../core/src/commands/wait_predicate_tests.rs | 4 +- crates/core/src/commands/wait_tests.rs | 2 + crates/core/src/hit_test_tests.rs | 4 +- crates/core/src/interaction_policy.rs | 4 ++ crates/core/src/ref_action_tests.rs | 4 +- crates/ffi/include/agent_desktop.h | 15 +++-- crates/ffi/src/commands/execute_by_ref.rs | 15 +++-- crates/ffi/src/notifications/list.rs | 6 +- crates/macos/src/actions/chain.rs | 10 ---- crates/macos/src/actions/chain_defs.rs | 23 +++++--- crates/macos/src/actions/chain_step_exec.rs | 5 +- crates/macos/src/actions/chain_tests.rs | 19 +++++- crates/macos/src/actions/mod.rs | 1 - crates/macos/src/actions/mutation_delivery.rs | 32 ---------- crates/macos/src/actions/physical_keyboard.rs | 13 ----- crates/macos/src/actions/scroll.rs | 33 ++--------- crates/macos/src/actions/toggle_state.rs | 4 +- crates/macos/src/actions/type_text.rs | 21 +++---- crates/macos/src/adapter_system.rs | 3 +- crates/macos/src/input/keyboard.rs | 37 ------------ crates/macos/src/notifications/actions.rs | 6 +- crates/macos/src/notifications/list.rs | 7 ++- crates/macos/src/notifications/nc_session.rs | 58 +++++++++++++++---- docs/phases.md | 4 +- ...tion-policy-aligned-with-cli-2026-05-12.md | 4 +- ...-gesture-headless-capability-2026-06-10.md | 9 +-- ...ht-grade-desktop-reliability-2026-06-02.md | 7 ++- skills/agent-desktop/SKILL.md | 4 +- .../references/commands-interaction.md | 24 ++++---- skills/agent-desktop/references/macos.md | 9 ++- src/cli/help_after.txt | 2 +- src/cli/mod.rs | 4 +- src/cli/root.rs | 2 +- src/dispatch/mod.rs | 2 +- src/dispatch/notifications.rs | 2 + tests/e2e/json_tool.py | 15 +++++ tests/e2e/lib.sh | 11 ++-- tests/e2e/run.sh | 6 ++ tests/e2e/scenarios-interaction.sh | 42 ++++++++------ tests/e2e/scenarios-notifications.sh | 8 +-- tests/e2e/test_harness_contract.py | 7 +++ tests/e2e/test_json_tool.py | 19 +++++- 55 files changed, 325 insertions(+), 305 deletions(-) delete mode 100644 crates/macos/src/actions/mutation_delivery.rs diff --git a/README.md b/README.md index 0eecef4e..adfca85a 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,9 @@ agent-desktop list-surfaces --app Notes # list menus, sheets, popovers, ### Interaction ```bash -agent-desktop click @s8f3k2p9:e3 # semantic AX-first click -agent-desktop double-click @s8f3k2p9:e3 # AXOpen; physical double-click uses --headed mouse-click --count 2 +agent-desktop click @s8f3k2p9:e3 # strict headless AX click +agent-desktop --headed click @s8f3k2p9:e3 # physical click, focus/cursor allowed +agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click agent-desktop triple-click @s8f3k2p9:e3 # POLICY_DENIED if physical input is disabled agent-desktop right-click @s8f3k2p9:e3 # open context menu; inspect effect before retrying agent-desktop type @s8f3k2p9:e5 "hello world" # insert text into element @@ -204,11 +205,11 @@ agent-desktop check @s8f3k2p9:e12 # idempotent check agent-desktop uncheck @s8f3k2p9:e12 # idempotent uncheck agent-desktop expand @s8f3k2p9:e15 # expand disclosure/tree item agent-desktop collapse @s8f3k2p9:e15 # collapse disclosure/tree item -agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # scroll (AX-first) +agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # strict headless AX scroll agent-desktop scroll-to @s8f3k2p9:e20 # scroll element into view ``` -> **(macOS, Phase 1)** Pure cursor gestures have no accessibility equivalent, so `triple-click`, `hover`, and `drag` are always physical; `double-click` is headless via `AXOpen` and only needs `--headed` for gesture-only targets. Windows (UIA) and Linux (AT-SPI) adapters may expose different capabilities. See `skills/agent-desktop/references/commands-interaction.md`. +> **(macOS, Phase 1)** Default ref actions are strict headless semantic operations. `--headed` prefers physical delivery for natural input commands; double/triple-click, hover, and drag are physical-only. Semantic-only commands remain semantic. See `skills/agent-desktop/references/commands-interaction.md`. ### Keyboard @@ -253,9 +254,9 @@ agent-desktop restore --window-id w-4521 ### Notifications *(macOS only)* ```bash -agent-desktop list-notifications # list all notifications -agent-desktop list-notifications --app "Slack" # filter by app -agent-desktop list-notifications --text "deploy" --limit 5 # filter by text +agent-desktop --headed list-notifications # open Notification Center if needed, then list +agent-desktop --headed list-notifications --app "Slack" # filter by app +agent-desktop --headed list-notifications --text "deploy" --limit 5 # filter by text agent-desktop dismiss-notification 1 --expected-app "Slack" --expected-title "Deploy complete" agent-desktop dismiss-all-notifications # dismiss all agent-desktop dismiss-all-notifications --app "Slack" # dismiss all from app @@ -263,8 +264,8 @@ agent-desktop notification-action 1 "Reply" --expected-app "Slack" --expected-ti ``` Single-notification mutations require an app or title fingerprint from the -same listing. Semantic dismiss is headless; pass global `--headed` only when -Notification Center requires its hover-revealed close control. +same listing. Headless listing observes an already-open Notification Center; +pass `--headed` to allow opening it and restoring the prior frontmost app. ### Clipboard diff --git a/crates/core/src/action.rs b/crates/core/src/action.rs index 4829e9fb..9ab2d4ad 100644 --- a/crates/core/src/action.rs +++ b/crates/core/src/action.rs @@ -95,10 +95,9 @@ impl Action { matches!(self, Self::TypeText(_) | Self::PressKey(_)) } - /// Returns the least-permissive interaction policy that can execute this - /// action with the same fallback behavior as its command entrypoint. + /// Returns the command's minimum interaction policy. pub fn base_interaction_policy(&self) -> crate::interaction_policy::InteractionPolicy { - if self.may_use_focus_fallback() { + if matches!(self, Self::PressKey(_)) { crate::interaction_policy::InteractionPolicy::focus_fallback() } else { crate::interaction_policy::InteractionPolicy::headless() diff --git a/crates/core/src/action_tests.rs b/crates/core/src/action_tests.rs index c55e534f..81940606 100644 --- a/crates/core/src/action_tests.rs +++ b/crates/core/src/action_tests.rs @@ -71,21 +71,21 @@ fn pure_ax_actions_base_policy_is_headless() { } #[test] -fn press_key_and_type_text_base_policy_is_focus_fallback() { - let focus_fallback = InteractionPolicy::focus_fallback(); +fn type_text_is_headless_but_explicit_press_allows_focus() { + let headless = InteractionPolicy::headless(); assert_eq!( Action::PressKey(KeyCombo { key: "a".into(), modifiers: vec![Modifier::Meta], }) .base_interaction_policy(), - focus_fallback, - "PressKey must permit the focus fallback needed to target keystrokes" + InteractionPolicy::focus_fallback(), + "PressKey is an explicit physical-input command" ); assert_eq!( Action::TypeText("hello".into()).base_interaction_policy(), - focus_fallback, - "TypeText must permit the focus fallback needed by physical typing" + headless, + "TypeText must not gain implicit focus or keyboard permission" ); } diff --git a/crates/core/src/actionability/evaluate.rs b/crates/core/src/actionability/evaluate.rs index c72074af..a984f0a1 100644 --- a/crates/core/src/actionability/evaluate.rs +++ b/crates/core/src/actionability/evaluate.rs @@ -82,7 +82,7 @@ pub(super) fn check_with_stability( ) -> Result { let requirements = ActionabilityRequirements::for_action(&request.action); let pointer_delivery = - requirements.pointer_delivery(&request.action, &evidence.available_actions); + requirements.pointer_delivery(&request.action, &evidence.available_actions, request.policy); let mut checks = Vec::new(); let mut verified_point = None; if requirements.visible { diff --git a/crates/core/src/actionability/requirements.rs b/crates/core/src/actionability/requirements.rs index 5d8ae290..d6d1a698 100644 --- a/crates/core/src/actionability/requirements.rs +++ b/crates/core/src/actionability/requirements.rs @@ -44,11 +44,17 @@ impl ActionabilityRequirements { &self, action: &Action, available_actions: &[String], + policy: crate::InteractionPolicy, ) -> PointerDelivery { if !self.receives_events { return PointerDelivery::NotApplicable; } - if crate::capability::supports_direct_semantic_pointer_delivery(action, available_actions) { + if policy.is_headed() { + PointerDelivery::Physical + } else if crate::capability::supports_direct_semantic_pointer_delivery( + action, + available_actions, + ) { PointerDelivery::Semantic } else { PointerDelivery::Physical diff --git a/crates/core/src/actionability/requirements_tests.rs b/crates/core/src/actionability/requirements_tests.rs index aa7f6804..19f34723 100644 --- a/crates/core/src/actionability/requirements_tests.rs +++ b/crates/core/src/actionability/requirements_tests.rs @@ -31,15 +31,31 @@ fn pointer_delivery_is_selected_from_live_semantic_capability_evidence() { let requirements = ActionabilityRequirements::for_action(&Action::Click); assert_eq!( - requirements.pointer_delivery(&Action::Click, &[crate::capability::CLICK.into()]), + requirements.pointer_delivery( + &Action::Click, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headless(), + ), PointerDelivery::Semantic ); assert_eq!( - requirements.pointer_delivery(&Action::Click, &[]), + requirements.pointer_delivery(&Action::Click, &[], crate::InteractionPolicy::headless(),), PointerDelivery::Physical ); assert_eq!( - requirements.pointer_delivery(&Action::DoubleClick, &[crate::capability::CLICK.into()]), + requirements.pointer_delivery( + &Action::DoubleClick, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headless(), + ), + PointerDelivery::Physical + ); + assert_eq!( + requirements.pointer_delivery( + &Action::Click, + &[crate::capability::CLICK.into()], + crate::InteractionPolicy::headed(), + ), PointerDelivery::Physical ); diff --git a/crates/core/src/adapter/system.rs b/crates/core/src/adapter/system.rs index 41db581e..36feb2a4 100644 --- a/crates/core/src/adapter/system.rs +++ b/crates/core/src/adapter/system.rs @@ -1,9 +1,10 @@ use crate::{ AdapterError, AdapterSession, AppInfo, Deadline, DismissAllNotificationsRequest, - DismissNotificationRequest, ImageBuffer, InteractionLease, KeyCombo, NotificationActionRequest, - NotificationFilter, NotificationInfo, PermissionReport, PermissionState, ProcessIdentity, - SessionAffinity, SignalBaseline, SignalFilter, WindowInfo, WindowOp, - action_result::ActionResult, display_info::DisplayInfo, screenshot_target::ScreenshotTarget, + DismissNotificationRequest, ImageBuffer, InteractionLease, InteractionPolicy, KeyCombo, + NotificationActionRequest, NotificationFilter, NotificationInfo, PermissionReport, + PermissionState, ProcessIdentity, SessionAffinity, SignalBaseline, SignalFilter, WindowInfo, + WindowOp, action_result::ActionResult, display_info::DisplayInfo, + screenshot_target::ScreenshotTarget, }; pub trait SystemOps: Send + Sync { @@ -192,6 +193,7 @@ pub trait SystemOps: Send + Sync { fn list_notifications( &self, _filter: &NotificationFilter, + _policy: InteractionPolicy, _deadline: Deadline, ) -> Result, AdapterError> { Err(AdapterError::not_supported("list_notifications")) diff --git a/crates/core/src/commands/execute_by_ref.rs b/crates/core/src/commands/execute_by_ref.rs index f209955f..3893859e 100644 --- a/crates/core/src/commands/execute_by_ref.rs +++ b/crates/core/src/commands/execute_by_ref.rs @@ -31,13 +31,9 @@ pub struct ExecuteByRefArgs<'a> { /// explicit ID must match the qualified ref. /// /// The effective `InteractionPolicy` is the join of `caller_policy` and the -/// action's CLI base policy, ensuring the result is always at least as -/// permissive as what the CLI would use for the same action, while allowing -/// FFI callers to opt in to higher-permission policies such as `headed`. -/// -/// Note on PressKey: its base policy is `focus_fallback` (derived from -/// `Action::base_interaction_policy`, shared with `TypeText`) because a -/// ref-targeted key press may need the target focused for keystrokes to land. +/// action's base policy. Semantic actions, including `TypeText`, are strict +/// headless; explicit `PressKey` permits focus without cursor movement. FFI +/// callers may opt in to `focus_fallback` or `headed`. pub fn execute( args: ExecuteByRefArgs<'_>, adapter: &dyn PlatformAdapter, diff --git a/crates/core/src/commands/execute_by_ref_tests.rs b/crates/core/src/commands/execute_by_ref_tests.rs index 43262e28..a8e781af 100644 --- a/crates/core/src/commands/execute_by_ref_tests.rs +++ b/crates/core/src/commands/execute_by_ref_tests.rs @@ -221,7 +221,7 @@ fn execute_with_timeout_zero_normalizes_to_single_attempt() { } #[test] -fn caller_cannot_downgrade_press_key_below_focus_fallback() { +fn explicit_press_key_keeps_its_focus_fallback_policy() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_with_ref("textfield", &["PressKey"]); let adapter = PolicyCaptureAdapter::new(); diff --git a/crates/core/src/commands/list_notifications.rs b/crates/core/src/commands/list_notifications.rs index 0f0d8482..c5b1df8c 100644 --- a/crates/core/src/commands/list_notifications.rs +++ b/crates/core/src/commands/list_notifications.rs @@ -1,4 +1,4 @@ -use crate::{AppError, NotificationFilter, adapter::PlatformAdapter}; +use crate::{AppError, NotificationFilter, adapter::PlatformAdapter, context::CommandContext}; use serde_json::{Value, json}; pub struct ListNotificationsArgs { @@ -10,13 +10,18 @@ pub struct ListNotificationsArgs { pub fn execute( args: ListNotificationsArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { let filter = NotificationFilter { app: args.app, text: args.text, limit: args.limit, }; - let notifications = adapter.list_notifications(&filter, crate::Deadline::standard()?)?; + let notifications = adapter.list_notifications( + &filter, + context.physical_input_policy(), + crate::Deadline::standard()?, + )?; Ok(json!({ "count": notifications.len(), "notifications": notifications, diff --git a/crates/core/src/commands/ref_policy_tests.rs b/crates/core/src/commands/ref_policy_tests.rs index b5742207..98e503df 100644 --- a/crates/core/src/commands/ref_policy_tests.rs +++ b/crates/core/src/commands/ref_policy_tests.rs @@ -236,7 +236,7 @@ fn default_ref_commands_use_least_permissive_supported_policy() { ) .unwrap(); let type_request = adapter.requests.lock().unwrap()[before_type].clone(); - assert_eq!(type_request.policy, InteractionPolicy::focus_fallback()); + assert_eq!(type_request.policy, InteractionPolicy::headless()); scroll::execute( scroll::ScrollArgs { ref_id: "@e1".into(), @@ -251,9 +251,6 @@ fn default_ref_commands_use_least_permissive_supported_policy() { .unwrap(); for request in adapter.requests.lock().unwrap().iter() { - if matches!(request.action, Action::TypeText(_)) { - continue; - } assert_headless(request); } } @@ -272,7 +269,7 @@ fn focus_command_is_explicit_headless_policy() { } #[test] -fn headed_context_preserves_physical_fallback_only_when_semantic_delivery_is_unavailable() { +fn headed_context_reaches_every_ref_action_without_policy_downgrade() { let _guard = HomeGuard::new(); let snapshot_id = snapshot_id(); let adapter = RecordingAdapter::new(); @@ -337,13 +334,9 @@ fn headed_context_preserves_physical_fallback_only_when_semantic_delivery_is_una .unwrap(); for request in adapter.requests.lock().unwrap().iter() { - let expected = if matches!(request.action, Action::Click | Action::RightClick) { - InteractionPolicy::headless() - } else { - InteractionPolicy::headed() - }; assert_eq!( - request.policy, expected, + request.policy, + InteractionPolicy::headed(), "unexpected policy for {:?}", request.action ); diff --git a/crates/core/src/commands/wait.rs b/crates/core/src/commands/wait.rs index 6dbc50f5..89a62e78 100644 --- a/crates/core/src/commands/wait.rs +++ b/crates/core/src/commands/wait.rs @@ -62,7 +62,7 @@ pub fn execute( } WaitMode::Menu { app, open } => wait_for_menu(app, open, timeout_ms, adapter), WaitMode::Notification { app, text } => { - wait_for_notification(app, text, timeout_ms, adapter) + wait_for_notification(app, text, timeout_ms, adapter, context) } WaitMode::Element { ref_id, @@ -266,6 +266,7 @@ fn wait_for_notification( text: Option, timeout_ms: u64, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { let filter = NotificationFilter { app: app.clone(), @@ -281,7 +282,7 @@ fn wait_for_notification( if deadline.is_expired() { return wait_timeout::notification(app.as_ref(), text.as_ref(), timeout_ms, last_error); } - match adapter.list_notifications(&filter, deadline) { + match adapter.list_notifications(&filter, context.physical_input_policy(), deadline) { Ok(current) => match &baseline { None => { baseline = Some(notification_counts(¤t)); diff --git a/crates/core/src/commands/wait_predicate.rs b/crates/core/src/commands/wait_predicate.rs index 2b5ae98c..b1900d4b 100644 --- a/crates/core/src/commands/wait_predicate.rs +++ b/crates/core/src/commands/wait_predicate.rs @@ -76,9 +76,7 @@ impl ElementPredicate { fn parse_actionability_action(action: Option<&str>) -> Result { match action.unwrap_or("click") { "click" => Ok(ActionRequest::headless(Action::Click)), - "type" => Ok(ActionRequest::focus_fallback(Action::TypeText( - String::new(), - ))), + "type" => Ok(ActionRequest::headless(Action::TypeText(String::new()))), "set-value" => Ok(ActionRequest::headless(Action::SetValue(String::new()))), "clear" => Ok(ActionRequest::headless(Action::Clear)), other => Err(AppError::invalid_input_with_suggestion( diff --git a/crates/core/src/commands/wait_predicate_tests.rs b/crates/core/src/commands/wait_predicate_tests.rs index 412a9878..4d58fedc 100644 --- a/crates/core/src/commands/wait_predicate_tests.rs +++ b/crates/core/src/commands/wait_predicate_tests.rs @@ -315,7 +315,7 @@ fn element_wait_actionable_type_fails_on_uneditable_role() { "@e1".into(), Some(snapshot_id), wait_predicate::ElementPredicate::Actionable( - crate::action_request::ActionRequest::focus_fallback(crate::action::Action::TypeText( + crate::action_request::ActionRequest::headless(crate::action::Action::TypeText( String::new(), )), ), @@ -380,7 +380,7 @@ fn actionable_parse_mirrors_each_real_command_policy() { ); assert_eq!( request_for(Some("type")).policy, - InteractionPolicy::focus_fallback() + InteractionPolicy::headless() ); assert_eq!( request_for(Some("set-value")).policy, diff --git a/crates/core/src/commands/wait_tests.rs b/crates/core/src/commands/wait_tests.rs index bdf92495..aa2f23aa 100644 --- a/crates/core/src/commands/wait_tests.rs +++ b/crates/core/src/commands/wait_tests.rs @@ -28,6 +28,7 @@ impl SystemOps for NotificationErrorAdapter { fn list_notifications( &self, _filter: &NotificationFilter, + _policy: crate::InteractionPolicy, _deadline: crate::Deadline, ) -> Result, AdapterError> { Err(AdapterError::new( @@ -61,6 +62,7 @@ impl SystemOps for FlakyNotificationAdapter { fn list_notifications( &self, _filter: &NotificationFilter, + _policy: crate::InteractionPolicy, _deadline: crate::Deadline, ) -> Result, AdapterError> { self.responses diff --git a/crates/core/src/hit_test_tests.rs b/crates/core/src/hit_test_tests.rs index 1a2372a2..0f70e16a 100644 --- a/crates/core/src/hit_test_tests.rs +++ b/crates/core/src/hit_test_tests.rs @@ -186,7 +186,7 @@ fn run_receives_events_error(outcome: Result) -> Ad } #[test] -fn semantic_click_skips_inconclusive_screen_hit_testing() { +fn headless_semantic_click_skips_inconclusive_screen_hit_testing() { let adapter = HitTestAdapter { outcome: Ok(HitTestResult::Unknown), actions: vec![capability::CLICK.into()], @@ -197,7 +197,7 @@ fn semantic_click_skips_inconclusive_screen_hit_testing() { &clickable_entry(), &NativeHandle::null(), &adapter, - &ActionRequest::headed(Action::Click), + &ActionRequest::headless(Action::Click), ) .expect("a direct semantic click must not depend on screen hit testing"); diff --git a/crates/core/src/interaction_policy.rs b/crates/core/src/interaction_policy.rs index 2732ea23..140ca571 100644 --- a/crates/core/src/interaction_policy.rs +++ b/crates/core/src/interaction_policy.rs @@ -28,6 +28,10 @@ impl InteractionPolicy { } } + pub fn is_headed(self) -> bool { + self.allow_focus_steal && self.allow_cursor_move + } + pub fn join(self, other: InteractionPolicy) -> InteractionPolicy { InteractionPolicy { allow_focus_steal: self.allow_focus_steal || other.allow_focus_steal, diff --git a/crates/core/src/ref_action_tests.rs b/crates/core/src/ref_action_tests.rs index a21ceb1e..d4546e48 100644 --- a/crates/core/src/ref_action_tests.rs +++ b/crates/core/src/ref_action_tests.rs @@ -232,14 +232,14 @@ fn successful_action_drops_resolved_payload() { } #[test] -fn semantic_preflight_revokes_unverified_physical_fallback() { +fn headed_preflight_preserves_requested_physical_delivery() { let adapter = SuccessfulAdapter::new(); execute_entry(&adapter, &entry(), ActionRequest::headed(Action::Click)).unwrap(); assert_eq!( adapter.dispatched_policies.lock().unwrap().as_slice(), - &[crate::InteractionPolicy::headless()] + &[crate::InteractionPolicy::headed()] ); } diff --git a/crates/ffi/include/agent_desktop.h b/crates/ffi/include/agent_desktop.h index eb04e46b..4cd522ae 100644 --- a/crates/ffi/include/agent_desktop.h +++ b/crates/ffi/include/agent_desktop.h @@ -1331,12 +1331,10 @@ void ad_app_list_free(struct AdAppList *list); * strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live * actionability preflight → dispatch → owned-handle drop. * - * Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type` - * command); `PressKey` shares that `focus_fallback` base (a ref-targeted key - * press may need the target focused); every other action defaults to - * `headless`. An explicit `policy` discriminant may *elevate* to headed but - * must not downgrade an action below its base. Base and elevation are computed - * by `agent_desktop_core::commands::execute_by_ref::execute` via + * Policy: semantic actions, including `TypeText`, default to strict + * `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy + * discriminant may elevate to focus fallback or headed. Base and elevation + * are computed by `agent_desktop_core::commands::execute_by_ref::execute` via * `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and * FFI share a single source of policy truth. * @@ -1349,8 +1347,9 @@ void ad_app_list_free(struct AdAppList *list); * * `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, * 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` - * accepts the action's own CLI base (so `TypeText` still uses - * `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks. + * accepts the action's base policy. `FocusFallback (1)` explicitly permits + * focus without cursor movement. `Headed (2)` opts in to physical cursor and + * keyboard delivery. * * Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before * the actionability preflight, matching the CLI default. Call diff --git a/crates/ffi/src/commands/execute_by_ref.rs b/crates/ffi/src/commands/execute_by_ref.rs index d5466b44..b56e8b8f 100644 --- a/crates/ffi/src/commands/execute_by_ref.rs +++ b/crates/ffi/src/commands/execute_by_ref.rs @@ -9,12 +9,10 @@ use std::ffi::c_char; /// strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live /// actionability preflight → dispatch → owned-handle drop. /// -/// Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type` -/// command); `PressKey` shares that `focus_fallback` base (a ref-targeted key -/// press may need the target focused); every other action defaults to -/// `headless`. An explicit `policy` discriminant may *elevate* to headed but -/// must not downgrade an action below its base. Base and elevation are computed -/// by `agent_desktop_core::commands::execute_by_ref::execute` via +/// Policy: semantic actions, including `TypeText`, default to strict +/// `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy +/// discriminant may elevate to focus fallback or headed. Base and elevation +/// are computed by `agent_desktop_core::commands::execute_by_ref::execute` via /// `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and /// FFI share a single source of policy truth. /// @@ -27,8 +25,9 @@ use std::ffi::c_char; /// /// `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback, /// 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)` -/// accepts the action's own CLI base (so `TypeText` still uses -/// `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks. +/// accepts the action's base policy. `FocusFallback (1)` explicitly permits +/// focus without cursor movement. `Headed (2)` opts in to physical cursor and +/// keyboard delivery. /// /// Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before /// the actionability preflight, matching the CLI default. Call diff --git a/crates/ffi/src/notifications/list.rs b/crates/ffi/src/notifications/list.rs index 06f7c48e..96b9bad3 100644 --- a/crates/ffi/src/notifications/list.rs +++ b/crates/ffi/src/notifications/list.rs @@ -35,7 +35,11 @@ pub unsafe extern "C" fn ad_list_notifications( }; let adapter = crate::adapter::acquire_adapter!(adapter); let deadline = crate::operation::operation_deadline!(); - match adapter.inner.list_notifications(&core_filter, deadline) { + match adapter.inner.list_notifications( + &core_filter, + agent_desktop_core::InteractionPolicy::headless(), + deadline, + ) { Ok(notifications) => { if let Err(error) = crate::resource::validate_list_len(notifications.len(), "Notification list") diff --git a/crates/macos/src/actions/chain.rs b/crates/macos/src/actions/chain.rs index b476af0c..9e9ddc91 100644 --- a/crates/macos/src/actions/chain.rs +++ b/crates/macos/src/actions/chain.rs @@ -32,12 +32,6 @@ mod imp { for (i, step) in def.steps.iter().enumerate() { ctx.ensure_budget()?; - if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) { - return Err(AdapterError::policy_denied_for_policy( - "Physical click fallback is disabled by the current interaction policy", - policy, - )); - } let label = step_label(step); let outcome = execute_step(el, step, ctx, policy)?; if record_step_outcome( @@ -109,10 +103,6 @@ mod imp { ChainStep::CGDisclosureClick { .. } => "CGDisclosureClick", } } - - fn physical_click_permitted(policy: InteractionPolicy) -> bool { - policy.allow_focus_steal && policy.allow_cursor_move - } } #[cfg(all(test, target_os = "macos"))] diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs index 1af6302c..58131151 100644 --- a/crates/macos/src/actions/chain_defs.rs +++ b/crates/macos/src/actions/chain_defs.rs @@ -11,8 +11,11 @@ mod imp { pub(crate) static CLICK_CHAIN: ChainDef = ChainDef { steps: &[ + ChainStep::CGClick { + button: MouseButton::Left, + count: 1, + }, ChainStep::Action("AXPress"), - ChainStep::CGDisclosureClick { expanded: true }, ], suggestion: "Target an element that advertises Click or use an explicit point click.", continue_after_unverified_delivery: false, @@ -20,6 +23,10 @@ mod imp { pub(crate) static RIGHT_CLICK_CHAIN: ChainDef = ChainDef { steps: &[ + ChainStep::CGClick { + button: MouseButton::Right, + count: 1, + }, ChainStep::CustomWithDeadline { label: "show_menu", func: chain_menu_steps::show_menu, @@ -40,10 +47,6 @@ mod imp { label: "ancestor_show_menu", func: chain_menu_steps::show_menu_on_ancestors, }, - ChainStep::CGClick { - button: MouseButton::Right, - count: 1, - }, ], suggestion: "Try 'mouse-click --button right --xy X,Y'.", continue_after_unverified_delivery: false, @@ -84,13 +87,19 @@ mod imp { pub(crate) static CLEAR_CHAIN: ChainDef = ChainDef { steps: &[ - ChainStep::SetDynamic { attr: "AXValue" }, ChainStep::FocusThenClearByKeyboard, + ChainStep::SetDynamic { attr: "AXValue" }, ], suggestion: "Target an editable control or allow the verified keyboard fallback.", continue_after_unverified_delivery: true, }; + pub(crate) static SEMANTIC_CLICK_CHAIN: ChainDef = ChainDef { + steps: &[ChainStep::Action("AXPress")], + suggestion: "Target an element that advertises Click.", + continue_after_unverified_delivery: false, + }; + pub(crate) static FOCUS_CHAIN: ChainDef = ChainDef { steps: &[ChainStep::SetBool { attr: "AXFocused", @@ -171,5 +180,5 @@ mod imp {} #[cfg(target_os = "macos")] pub(crate) use imp::{ CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, FOCUS_CHAIN, RIGHT_CLICK_CHAIN, - SCROLL_TO_CHAIN, SET_VALUE_CHAIN, double_click, triple_click, + SCROLL_TO_CHAIN, SEMANTIC_CLICK_CHAIN, SET_VALUE_CHAIN, double_click, triple_click, }; diff --git a/crates/macos/src/actions/chain_step_exec.rs b/crates/macos/src/actions/chain_step_exec.rs index 386cfa65..ccd1be8c 100644 --- a/crates/macos/src/actions/chain_step_exec.rs +++ b/crates/macos/src/actions/chain_step_exec.rs @@ -56,7 +56,7 @@ mod imp { }, ChainStep::FocusThenClearByKeyboard => { - if !policy.allow_focus_steal { + if !policy.is_headed() { return Ok(DeliveryOutcome::NotDelivered); } crate::actions::physical_keyboard::press_sequence( @@ -80,6 +80,9 @@ mod imp { ChainStep::CustomWithDeadline { label: _, func } => func(el, ctx.deadline), ChainStep::CGClick { button, count } => { + if !policy.is_headed() { + return Ok(DeliveryOutcome::NotDelivered); + } physical_click(el, button.clone(), *count, ctx, policy)?; Ok(DeliveryOutcome::DeliveredUnverified) } diff --git a/crates/macos/src/actions/chain_tests.rs b/crates/macos/src/actions/chain_tests.rs index 350836be..53733a97 100644 --- a/crates/macos/src/actions/chain_tests.rs +++ b/crates/macos/src/actions/chain_tests.rs @@ -4,7 +4,7 @@ use agent_desktop_core::MouseButton; use agent_desktop_core::step_mechanism::StepMechanism; #[test] -fn right_click_restores_semantic_menu_fallbacks_before_physical_input() { +fn right_click_prefers_physical_input_before_semantic_fallbacks() { let labels: Vec<&str> = crate::actions::chain_defs::RIGHT_CLICK_CHAIN .steps .iter() @@ -19,16 +19,31 @@ fn right_click_restores_semantic_menu_fallbacks_before_physical_input() { assert_eq!( labels, [ + "CGClick", "show_menu", "select_then_show_menu", "selected_items_menu", "child_show_menu", "ancestor_show_menu", - "CGClick", ] ); } +#[test] +fn click_and_clear_prefer_physical_delivery_when_policy_allows_it() { + assert!(matches!( + crate::actions::chain_defs::CLICK_CHAIN.steps.first(), + Some(ChainStep::CGClick { + button: MouseButton::Left, + count: 1 + }) + )); + assert!(matches!( + crate::actions::chain_defs::CLEAR_CHAIN.steps.first(), + Some(ChainStep::FocusThenClearByKeyboard) + )); +} + #[test] fn step_mechanism_tags_physical_for_cgclick_and_keyboard_clear() { assert_eq!( diff --git a/crates/macos/src/actions/mod.rs b/crates/macos/src/actions/mod.rs index 7e867ff7..5bce1ca5 100644 --- a/crates/macos/src/actions/mod.rs +++ b/crates/macos/src/actions/mod.rs @@ -14,7 +14,6 @@ pub(crate) mod chain_value_write; pub(crate) mod chain_verify; pub(crate) mod dispatch; pub(crate) mod extras; -mod mutation_delivery; mod physical_click; mod physical_keyboard; pub(crate) mod post_state; diff --git a/crates/macos/src/actions/mutation_delivery.rs b/crates/macos/src/actions/mutation_delivery.rs deleted file mode 100644 index 28ba00a7..00000000 --- a/crates/macos/src/actions/mutation_delivery.rs +++ /dev/null @@ -1,32 +0,0 @@ -use agent_desktop_core::{AdapterError, ErrorCode}; - -pub(crate) fn fallback_is_safe(error: &AdapterError) -> bool { - error.code == ErrorCode::ActionFailed - && error.disposition == agent_desktop_core::DeliverySemantics::not_delivered() -} - -#[cfg(test)] -mod tests { - use agent_desktop_core::{AdapterError, ErrorCode}; - - use super::fallback_is_safe; - - #[test] - fn rejects_uncertain_or_non_action_failures() { - let uncertain = AdapterError::new(ErrorCode::ActionFailed, "uncertain") - .with_disposition(agent_desktop_core::DeliverySemantics::uncertain()); - assert!(!fallback_is_safe(&uncertain)); - assert!(!fallback_is_safe(&AdapterError::permission_denied())); - assert!(!fallback_is_safe(&AdapterError::new( - ErrorCode::AppUnresponsive, - "unresponsive", - ))); - } - - #[test] - fn accepts_definite_non_delivery() { - let not_delivered = AdapterError::new(ErrorCode::ActionFailed, "not delivered") - .with_disposition(agent_desktop_core::DeliverySemantics::not_delivered()); - assert!(fallback_is_safe(¬_delivered)); - } -} diff --git a/crates/macos/src/actions/physical_keyboard.rs b/crates/macos/src/actions/physical_keyboard.rs index fb2d6319..4a326085 100644 --- a/crates/macos/src/actions/physical_keyboard.rs +++ b/crates/macos/src/actions/physical_keyboard.rs @@ -46,19 +46,6 @@ pub(crate) fn type_text( }) } -pub(crate) fn repeat_keycode( - element: &AXElement, - key_code: u16, - repeats: u32, - policy: InteractionPolicy, - deadline: Deadline, -) -> Result<(), AdapterError> { - let identity = prepare_target(element, policy, deadline)?; - let pid = identity.pid(); - verify_delivery_target(element, identity, deadline)?; - crate::input::keyboard::synthesize_keycode(key_code, repeats, Some(pid), deadline) -} - fn prepare_target( element: &AXElement, policy: InteractionPolicy, diff --git a/crates/macos/src/actions/scroll.rs b/crates/macos/src/actions/scroll.rs index 00b38ce8..2e34159e 100644 --- a/crates/macos/src/actions/scroll.rs +++ b/crates/macos/src/actions/scroll.rs @@ -16,6 +16,10 @@ pub(crate) fn ax_scroll( validate_amount(amount)?; let scroll_area = find_scroll_area(element, deadline)?; let target = scroll_area.as_ref().unwrap_or(element); + if policy.is_headed() { + physical_wheel(target, direction, amount, deadline)?; + return Ok((StepMechanism::PhysicalSynthetic, false)); + } accept_optional_visibility_result(try_action(element, "AXScrollToVisible", deadline))?; let (bar_attribute, increment_action) = scroll_bar_action(direction); @@ -33,26 +37,6 @@ pub(crate) fn ax_scroll( if perform_repeated_action(target, page_action(direction), amount, deadline)? { return Ok((StepMechanism::SemanticApi, false)); } - if policy.allow_focus_steal { - let keycode = direction_keycode(direction); - match crate::actions::physical_keyboard::repeat_keycode( - target, keycode, amount, policy, deadline, - ) { - Ok(()) => return Ok((StepMechanism::PhysicalSynthetic, false)), - Err(error) if crate::actions::mutation_delivery::fallback_is_safe(&error) => {} - Err(error) => return Err(error), - } - } - if policy.allow_focus_steal && policy.allow_cursor_move { - physical_wheel(target, direction, amount, deadline)?; - return Ok((StepMechanism::PhysicalSynthetic, false)); - } - if policy.allow_focus_steal { - return Err(AdapterError::policy_denied_for_policy( - "Cursor-moving scroll fallback is disabled by the current interaction policy", - policy, - )); - } Err(AdapterError::new( ErrorCode::ActionNotSupported, "No scroll mechanism found on element", @@ -107,15 +91,6 @@ fn page_action(direction: &Direction) -> &'static str { } } -fn direction_keycode(direction: &Direction) -> u16 { - match direction { - Direction::Down => 121, - Direction::Up => 116, - Direction::Right => 124, - Direction::Left => 123, - } -} - fn perform_repeated_action( element: &AXElement, action: &'static str, diff --git a/crates/macos/src/actions/toggle_state.rs b/crates/macos/src/actions/toggle_state.rs index 647f1c42..caa1b0cc 100644 --- a/crates/macos/src/actions/toggle_state.rs +++ b/crates/macos/src/actions/toggle_state.rs @@ -42,7 +42,7 @@ pub(crate) fn toggle( verified_point: None, deadline, }; - let mut steps = execute_chain(el, &chain_defs::CLICK_CHAIN, &ctx, policy)?; + let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?; let verified = if let Some(before) = before { wait_for_value_change(el, &before, deadline).map_err(after_delivery)?; true @@ -93,7 +93,7 @@ pub(crate) fn check_uncheck( verified_point: None, deadline, }; - let mut steps = execute_chain(el, &chain_defs::CLICK_CHAIN, &ctx, policy)?; + let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?; wait_for_checked_state(el, want_checked, deadline).map_err(after_delivery)?; mark_last_verified(&mut steps, true); Ok(steps) diff --git a/crates/macos/src/actions/type_text.rs b/crates/macos/src/actions/type_text.rs index 09aa084c..7205a15e 100644 --- a/crates/macos/src/actions/type_text.rs +++ b/crates/macos/src/actions/type_text.rs @@ -17,20 +17,15 @@ pub(crate) fn execute_type( "Type requires a text field, secure text field, or combo box", )); } - match insert_selected_text(element, text, deadline) { - Ok(()) => { - return Ok(ActionStep::succeeded("AXSelectedText") - .with_mechanism(StepMechanism::SemanticApi) - .with_verified(false)); - } - Err(error) - if policy.allow_focus_steal - && crate::actions::mutation_delivery::fallback_is_safe(&error) => {} - Err(error) => return Err(error), + if policy.is_headed() { + crate::actions::physical_keyboard::type_text(element, text, policy, deadline)?; + return Ok(ActionStep::succeeded("PIDTargetedUnicodeText") + .with_mechanism(StepMechanism::PhysicalSynthetic) + .with_verified(false)); } - crate::actions::physical_keyboard::type_text(element, text, policy, deadline)?; - Ok(ActionStep::succeeded("PIDTargetedUnicodeText") - .with_mechanism(StepMechanism::PhysicalSynthetic) + insert_selected_text(element, text, deadline)?; + Ok(ActionStep::succeeded("AXSelectedText") + .with_mechanism(StepMechanism::SemanticApi) .with_verified(false)) } diff --git a/crates/macos/src/adapter_system.rs b/crates/macos/src/adapter_system.rs index bbcdb031..f18ad6fc 100644 --- a/crates/macos/src/adapter_system.rs +++ b/crates/macos/src/adapter_system.rs @@ -190,9 +190,10 @@ impl SystemOps for MacOSAdapter { fn list_notifications( &self, filter: &NotificationFilter, + policy: agent_desktop_core::InteractionPolicy, deadline: Deadline, ) -> Result, AdapterError> { - crate::notifications::list::list_notifications(filter, deadline) + crate::notifications::list::list_notifications(filter, policy, deadline) } fn dismiss_notification( diff --git a/crates/macos/src/input/keyboard.rs b/crates/macos/src/input/keyboard.rs index 5bde34ea..5947d7ad 100644 --- a/crates/macos/src/input/keyboard.rs +++ b/crates/macos/src/input/keyboard.rs @@ -60,33 +60,6 @@ pub(crate) fn preflight_text(text: &str, deadline: Deadline) -> Result<(), Adapt crate::input::keyboard_event::preflight_text(text, deadline) } -#[cfg(target_os = "macos")] -pub(crate) fn synthesize_keycode( - key_code: u16, - repeats: u32, - target_pid: Option, - deadline: Deadline, -) -> Result<(), AdapterError> { - const MAX_REPEATS: u32 = 1_000; - - if repeats == 0 || repeats > MAX_REPEATS { - return Err(AdapterError::new( - ErrorCode::InvalidArgs, - "Key repeat count must be between 1 and 1000", - )); - } - for delivered in 0..repeats { - crate::input::keyboard_event::post_key( - key_code, - core_graphics::event::CGEventFlags::empty(), - target_pid, - deadline, - (delivered as usize, repeats as usize), - )?; - } - Ok(()) -} - #[cfg(not(target_os = "macos"))] pub(crate) fn synthesize_key( _combo: &KeyCombo, @@ -119,16 +92,6 @@ pub(crate) fn preflight_text(_text: &str, _deadline: Deadline) -> Result<(), Ada Err(AdapterError::not_supported("synthesize_text")) } -#[cfg(not(target_os = "macos"))] -pub(crate) fn synthesize_keycode( - _key_code: u16, - _repeats: u32, - _target_pid: Option, - _deadline: Deadline, -) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("synthesize_keycode")) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/macos/src/notifications/actions.rs b/crates/macos/src/notifications/actions.rs index bf703e95..302047b1 100644 --- a/crates/macos/src/notifications/actions.rs +++ b/crates/macos/src/notifications/actions.rs @@ -17,7 +17,7 @@ pub fn dismiss_notification( deadline: Deadline, ) -> Result { require_foreground_policy(policy)?; - let session = NcSession::open(deadline)?; + let session = NcSession::open(policy, deadline)?; let result = dismiss_impl(index, app_filter, identity, policy, session.pid(), deadline); close_session(session, result) } @@ -28,7 +28,7 @@ pub fn dismiss_all( deadline: Deadline, ) -> Result<(Vec, Vec), AdapterError> { require_foreground_policy(policy)?; - let session = NcSession::open(deadline)?; + let session = NcSession::open(policy, deadline)?; let result = dismiss_all_impl(app_filter, policy, session.pid(), deadline); close_session(session, result) } @@ -41,7 +41,7 @@ pub fn notification_action( deadline: Deadline, ) -> Result { require_foreground_policy(policy)?; - let session = NcSession::open(deadline)?; + let session = NcSession::open(policy, deadline)?; let result = action_impl(index, identity, action_name, session.pid(), deadline); close_session(session, result) } diff --git a/crates/macos/src/notifications/list.rs b/crates/macos/src/notifications/list.rs index abff46fe..c9a0bbd1 100644 --- a/crates/macos/src/notifications/list.rs +++ b/crates/macos/src/notifications/list.rs @@ -1,12 +1,15 @@ -use agent_desktop_core::{AdapterError, Deadline, NotificationFilter, NotificationInfo}; +use agent_desktop_core::{ + AdapterError, Deadline, InteractionPolicy, NotificationFilter, NotificationInfo, +}; use super::nc_session::{NcSession, close_session}; pub fn list_notifications( filter: &NotificationFilter, + policy: InteractionPolicy, deadline: Deadline, ) -> Result, AdapterError> { - let session = NcSession::open(deadline)?; + let session = NcSession::open(policy, deadline)?; let result = list_from_nc(filter, session.pid(), deadline); close_session(session, result) } diff --git a/crates/macos/src/notifications/nc_session.rs b/crates/macos/src/notifications/nc_session.rs index e0a47922..fcb27801 100644 --- a/crates/macos/src/notifications/nc_session.rs +++ b/crates/macos/src/notifications/nc_session.rs @@ -1,4 +1,4 @@ -use agent_desktop_core::{AdapterError, Deadline, KeyCombo}; +use agent_desktop_core::{AdapterError, Deadline, InteractionPolicy, KeyCombo}; pub(crate) fn close_session( session: NcSession, @@ -24,18 +24,30 @@ pub(crate) struct NcSession { } impl NcSession { - pub(crate) fn open(deadline: Deadline) -> Result { + pub(crate) fn open( + policy: InteractionPolicy, + deadline: Deadline, + ) -> Result { + if let Some(pid) = nc_pid(deadline)? + && is_nc_open(pid, deadline) + { + return Ok(Self { + pid, + was_already_open: true, + previous_app: None, + closed: false, + deadline, + }); + } + if !policy.is_headed() { + return Err(closed_center_policy_error(policy)); + } let previous_app = frontmost_app(deadline); - let (was_already_open, pid) = match nc_pid(deadline)? { - Some(pid) if is_nc_open(pid, deadline) => (true, pid), - _ => { - open_nc(deadline)?; - (false, wait_for_nc_ready(deadline)?) - } - }; + open_nc(deadline)?; + let pid = wait_for_nc_ready(deadline)?; Ok(Self { pid, - was_already_open, + was_already_open: false, previous_app, closed: false, deadline, @@ -60,6 +72,16 @@ impl NcSession { } } +fn closed_center_policy_error(policy: InteractionPolicy) -> AdapterError { + AdapterError::policy_denied_for_policy( + "Notification Center is closed and observation cannot open it in headless mode", + policy, + ) + .with_suggestion( + "Open Notification Center yourself or pass --headed to allow opening and restoring desktop focus.", + ) +} + impl Drop for NcSession { fn drop(&mut self) { if self.closed { @@ -221,7 +243,7 @@ fn close_nc(deadline: Deadline) -> Result<(), AdapterError> { #[cfg(all(test, target_os = "macos"))] mod tests { - use super::{applescript_string, nc_pid_from_output}; + use super::{applescript_string, closed_center_policy_error, nc_pid_from_output}; use agent_desktop_core::AdapterError; #[test] @@ -247,6 +269,20 @@ mod tests { assert_eq!(error.code, agent_desktop_core::ErrorCode::Timeout); } + + #[test] + fn closed_notification_center_is_policy_denied_headlessly() { + let error = closed_center_policy_error(agent_desktop_core::InteractionPolicy::headless()); + + assert_eq!(error.code, agent_desktop_core::ErrorCode::PolicyDenied); + assert!(error.message.contains("headless")); + assert!( + error + .suggestion + .as_deref() + .is_some_and(|value| value.contains("--headed")) + ); + } } #[cfg(not(target_os = "macos"))] diff --git a/docs/phases.md b/docs/phases.md index 68f555af..14c7c2a7 100644 --- a/docs/phases.md +++ b/docs/phases.md @@ -249,7 +249,7 @@ crates/macos/src/ │ ├── extras.rs # select_value helpers │ ├── post_state.rs # Post-action state reads │ ├── scroll.rs # scroll semantics and explicit physical policy paths -│ └── type_text.rs # focus-fallback text insertion and physical typing +│ └── type_text.rs # strict semantic text insertion and headed physical typing ├── input/ │ ├── mod.rs # re-exports │ ├── keyboard.rs # CGEventCreateKeyboardEvent, key synthesis, text typing @@ -285,7 +285,7 @@ crates/macos/src/ **Action execution:** - Ref actions take `ActionRequest`, not bare `Action` - Default policy forbids focus stealing and cursor movement -- Click/right-click/scroll chains run semantic AX steps first and return structured errors instead of silently using physical/headed paths +- Click/right-click/scroll use semantic AX delivery headlessly and prefer physical delivery only under explicit `--headed` - Type uses the focus-fallback policy floor; SetValue/Clear are the pure headless AX value-mutation paths - SetValue/Clear: `AXUIElementSetAttributeValue(kAXValueAttribute, value)` - SetFocus/Press/Hover/Drag/Mouse: explicit focus/cursor/physical commands diff --git a/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md index ce43f46e..b3af774f 100644 --- a/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md +++ b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md @@ -26,8 +26,8 @@ high-level ref path can reproduce CLI observation-to-action behavior. `ad_execute_by_ref` and its timeout variant use the core command path. They load the ref map, resolve strictly, apply actionability, and compute policy through `Action::base_interaction_policy` joined with the caller's explicit -policy. Headless callers therefore retain an action's required focus fallback; -headed is an opt-in elevation. +policy. Semantic actions, including `type`, stay strictly headless; explicit +`press` retains its focus fallback and headed is an opt-in elevation. `ad_execute_action` and struct-based direct action entrypoints are deliberately lower-level escape hatches. They operate on a caller-held native handle or diff --git a/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md b/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md index dd4db899..79ba3f03 100644 --- a/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md +++ b/docs/solutions/best-practices/macos-gesture-headless-capability-2026-06-10.md @@ -25,11 +25,12 @@ future adapters copy macOS assumptions. ## Guidance Core creates an `ActionRequest` with the action's least-permissive base policy. -Most actions start headless. `type` and ref-targeted `press` may use focus -fallback. Explicit headed mode can elevate that policy but may not weaken it. +Semantic actions, including `type`, start strictly headless. Explicit `press` +may use focus fallback. Headed mode can elevate policy but may not weaken it. -The adapter chooses the best legal implementation: semantic accessibility APIs -first, then a policy-gated physical fallback when appropriate. `hover` and +The adapter chooses the requested legal implementation: strict headless uses +semantic accessibility APIs, while headed natural-input commands prefer +physical delivery. `hover` and `drag` are physical pointer commands by definition and require a cursor-moving policy before resolving or moving the pointer. A semantic reorder capability, if a future platform offers one, is a distinct action contract rather than a diff --git a/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md b/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md index b799ee7e..f2acb305 100644 --- a/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md +++ b/docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md @@ -42,9 +42,10 @@ is `AMBIGUOUS_TARGET`. Mutable field values are not stable identity. ### 3. Separate actionability from dispatch Semantic ref actions use the shared auto-wait and live actionability checks. -The command owns the base interaction policy: most actions are headless; -typing and ref-targeted key presses may use focus fallback; headed mode can -only elevate policy. A failed preflight must say why and preserve retry safety. +The command owns the base interaction policy: semantic actions, including +typing, are strictly headless; explicit ref-targeted key presses may use focus +fallback, and headed mode can only elevate policy. A failed preflight must say +why and preserve retry safety. Pointer commands are a separate physical family. They resolve a live point, verify visibility, geometry stability, and hit-test receipt, then require an diff --git a/skills/agent-desktop/SKILL.md b/skills/agent-desktop/SKILL.md index 53164688..eb717144 100644 --- a/skills/agent-desktop/SKILL.md +++ b/skills/agent-desktop/SKILL.md @@ -94,7 +94,7 @@ Use **progressive skeleton traversal** as the default approach. It reduces token - **Scoped invalidation:** re-drilling a qualified root ref only replaces refs from that root's previous drill — refs from other regions and the skeleton itself are preserved - **Strict resolution:** stale refs return `STALE_REF`; duplicate plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily. - **Actionability:** every ref-addressed action checks its applicable live visibility, stability, enabled, editability, policy, supported-action, and hit-test requirements under one bounded budget before a single dispatch. Pointer actions focus before their final geometry read, re-resolve moving endpoints, and return `TIMEOUT` with `details.kind: "actionability_timeout"` instead of sending input after the deadline. -- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed when only a physical gesture would work. `type` uses a focus-fallback base policy because typing needs focus but never moves the cursor. Pass the global `--headed` flag to permit cursor movement and focus stealing so physical fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw cursor commands (`hover`, `drag`, `mouse-*`) are physical and require `--headed`; keyboard commands (`press`, `key-down`, `key-up`) are explicit low-level input. +- **Headless vs headed:** ref actions are strictly headless by default: semantic accessibility APIs only, with no focus stealing, cursor movement, or synthesized keyboard input. Pass global `--headed` to prefer physical delivery for natural input commands (`click`, `right-click`, `type`, `clear`, `expand`, `collapse`, and `scroll`); semantic-only commands such as `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` remain semantic. Raw input commands (`press`, `hover`, `drag`, `mouse-*`) are explicit physical input; cursor commands require `--headed`. - **Sessions and tracing:** run `session start` once per agent run to create a manifest with `trace: on` (default), then pass its returned ID with `--session` or `AGENT_DESKTOP_SESSION`. Use `session start --screenshots` when you need replay artifacts (`artifacts: full`): pre/post-action PNGs and refmap copies under the session trace directory (sensitive — treat exports like screenshots). Commands in that explicit scope record JSONL automatically to per-process segments under `~/.agent-desktop/sessions//trace/-.jsonl` — no `--trace` on every call. Read traces back with `trace show` (bounded JSON for agents) or `trace export` (single-file HTML for humans). A session owns both its trace and its latest-snapshot namespace. Snapshot lookup never searches another namespace. **`--session ` alone** (no manifest from `session start`) selects only the snapshot namespace — existing callers see no surprise trace files. **`--trace `** still overrides to one atomic file for CI or one-offs. Activation precedence is `--session` > `AGENT_DESKTOP_SESSION` > no session; `session start` does not activate later processes. Multi-agent shared sessions: each agent acts on qualified refs from its own snapshot — implicit latest is not a cross-agent guarantee. Run `status` to see `session_id` and `tracing`. Trace lines include `ts_ms`, monotonic per-process `seq`, and redacted sensitive fields (`text`, `value`, `expected`, `name`, `username`, `description`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, `placeholder` → `{ "redacted": true }`). `--trace-strict` fails on trace setup and pre-action writes; post-action success traces are best-effort. ## JSON Output Contract @@ -151,7 +151,7 @@ agent-desktop list-surfaces --app "App" # Available surfaces ### Interaction ``` agent-desktop click @e5 --snapshot # AX-first click, no cursor move by default -agent-desktop double-click @s8f3k2p9:e3 # AXOpen; physical double-click uses --headed mouse-click --count 2 +agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click agent-desktop triple-click @s8f3k2p9:e2 # Physical triple-click uses mouse-click --count 3 agent-desktop right-click @s8f3k2p9:e5 # Right-click; inspect the resulting menu/effect separately agent-desktop type @e2 --snapshot "hello" # Headless AX text insertion when supported diff --git a/skills/agent-desktop/references/commands-interaction.md b/skills/agent-desktop/references/commands-interaction.md index 63a57409..e4d657a3 100644 --- a/skills/agent-desktop/references/commands-interaction.md +++ b/skills/agent-desktop/references/commands-interaction.md @@ -6,8 +6,8 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and Ref-based actions run in two modes, Playwright-style: -- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it fails closed rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.) -- **`--headed`.** A global flag (`agent-desktop --headed click @s8f3k2p9:e5`) that upgrades every ref action to permit focus stealing **and** cursor movement, unlocking the physical click/double-click/scroll/keypress fallbacks in the action chain. The AX path is still tried first, so `--headed` never regresses elements that work headlessly — it only adds fallbacks for elements that need a real gesture (e.g. a gesture-only button with no `AXOpen`). +- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the semantic path cannot perform the action it fails closed. +- **`--headed`.** A global flag (`agent-desktop --headed click @s8f3k2p9:e5`) that permits focus/cursor side effects and prefers physical delivery for natural input commands: `click`, `right-click`, `double-click`, `triple-click`, `type`, `clear`, `expand`, `collapse`, and `scroll`. Semantic-only commands (`set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, `scroll-to`) stay semantic. Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`, `key-up`) are physical by nature. Cursor-moving commands (`hover`, `drag`, `mouse-*`) require `--headed`; keyboard commands are explicit low-level input. @@ -47,9 +47,9 @@ The command surface is platform-agnostic: every ref action builds an `Action` an | Command | Headless path (macOS) | Notes | |---------|---------------|-------| -| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions; the default and most reliable surface | -| `type` | focus fallback | CLI `type` may focus the target field but never moves the cursor; use `set-value` for pure headless value mutation when supported | -| `double-click` | partial | `AXOpen` works headless on items that advertise it (Finder/list/outline rows, table cells). Falls back to `--headed` only for gesture-only targets with no `AXOpen`. | +| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions in strict headless mode | +| `type` | yes | uses `AXSelectedText` headlessly; `--headed` synthesizes keyboard input | +| `double-click` | no | a real two-click gesture; requires `--headed` | | `triple-click` | no | macOS exposes no triple-click action; it is purely 3 physical clicks → `--headed` only | | `hover` | no | hovering *is* moving the cursor over an element; no AX equivalent | | `drag` / drop | no | dragging *is* a cursor press-move-release; no general AX drag. Native cross-app drop needs the OS dragging-session/pasteboard protocol that synthetic events cannot start (works for same-view source-tracked gestures and web/Electron mouse-DnD) | @@ -69,20 +69,20 @@ Every ref-resolving action accepts `--timeout-ms` (default `5000`), but it budge ## Click Actions -Click commands use semantic AX activation first. In the default headless mode, coordinate click fallback is blocked; pass `--headed` to allow the physical click fallback, or use `agent-desktop --headed mouse-click` for a raw coordinate click. +Click commands use semantic AX activation in strict headless mode. Pass `--headed` to prefer a physical click, or use `agent-desktop --headed mouse-click` for a raw coordinate click. ### click ```bash agent-desktop click @s8f3k2p9:e5 agent-desktop click @e5 --snapshot ``` -Primary activation. Tries verified AXPress > AXConfirm > AXOpen > AXPick > child activation > selection/value relays > custom actions > ancestor activation. Focus-stealing and coordinate fallback steps are not used by the default ref command path. +Primary activation. Headless uses `AXPress`; `--headed` performs a physical click first and reports `physical_synthetic` in `data.steps`. ### double-click ```bash agent-desktop double-click @s8f3k2p9:e3 ``` -Tries AXOpen (headless). When the element advertises no `AXOpen`, the headless command fails closed with `POLICY_DENIED`; pass `--headed` to perform a real double-click (`agent-desktop --headed double-click @s8f3k2p9:e3`), or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for a raw coordinate double-click. +Double-click is a physical gesture and fails closed in headless mode. Pass `--headed` to perform it, or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for raw coordinates. ### triple-click ```bash @@ -94,7 +94,7 @@ Triple-click requires cursor/focus side effects and is blocked in headless mode; ```bash agent-desktop right-click @s8f3k2p9:e5 ``` -Performs a semantic right-click/context-menu action. On macOS, `AXShowMenu` can return `APP_UNRESPONSIVE` with `delivery: delivery_uncertain` and `retry: unsafe` after opening a modal context menu; inspect the resulting menu or target effect before deciding what to do, and never retry that outcome blindly. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked in headless mode; pass `--headed` to allow them. +Headless uses semantic context-menu actions. `--headed` performs a physical right-click first. On macOS, a semantic `AXShowMenu` can return `APP_UNRESPONSIVE` with uncertain delivery after opening a modal menu; inspect the effect and never retry blindly. Use `select` for combo boxes and menu buttons. ## Text Input @@ -103,9 +103,7 @@ Performs a semantic right-click/context-menu action. On macOS, `AXShowMenu` can agent-desktop type @s8f3k2p9:e2 "hello@example.com" agent-desktop type @s8f3k2p9:e2 "multi line\ntext" ``` -`type` uses the focus-fallback policy floor: it may focus the target field because typing requires focus, but it never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs). - -Under focus-fallback or `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer `set-value` when the target supports it. +Headless `type` uses `AXSelectedText` without focusing the app or synthesizing keys. Pass `--headed` to focus the target and synthesize keyboard input. Use `set-value` when direct semantic value assignment is the intended interaction. ### set-value ```bash @@ -117,7 +115,7 @@ Sets the value directly via the AX value attribute. Faster than `type` but may n ```bash agent-desktop clear @s8f3k2p9:e2 ``` -Clears the element's value to an empty string. Equivalent to `set-value @s8f3k2p9:e2 ""`. +Headless clears through `AXValue`. With `--headed`, it performs focus + Select All + Delete first. ### focus ```bash diff --git a/skills/agent-desktop/references/macos.md b/skills/agent-desktop/references/macos.md index 4f4da39b..e3f3ef3b 100644 --- a/skills/agent-desktop/references/macos.md +++ b/skills/agent-desktop/references/macos.md @@ -79,11 +79,10 @@ The default activation-chain deadline is 10 seconds. Set `AGENT_DESKTOP_CHAIN_TI Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. macOS actions split semantic AX steps from explicit physical/headed paths: -- `click`, `right-click`, `scroll`, `set-value`, `clear`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, and `scroll-to` try AX-first semantics and fail clearly when the headless path is unavailable. -- `type` uses focus fallback in the CLI/ref-action path. It may focus the target field, never moves the cursor, and can use the pasteboard for non-ASCII insertion. Use `set-value` for pure headless value mutation when supported. -- `focus`, `press`, `hover`, `drag`, and `mouse-*` are explicit physical/focus/cursor commands. -- FFI ref-action callers should use focus fallback for `type` to match CLI behavior; direct-handle `ad_execute_action` is lower-level and defaults to headless. -- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Use `set-value` for sensitive text when possible. +- `click`, `right-click`, `type`, `clear`, `scroll`, `expand`, and `collapse` use semantic AX delivery headlessly and prefer physical input with `--headed`. +- `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` are semantic-only even under `--headed`. +- `press`, `hover`, `drag`, and `mouse-*` are explicit physical input; cursor-moving commands require `--headed`. +- FFI ref-action callers get the same strict headless `type` default; direct-handle `ad_execute_action` is lower-level and applies the supplied policy verbatim. - If a command would need a forbidden physical path, it returns a structured error with a recovery hint. ### Surfaces diff --git a/src/cli/help_after.txt b/src/cli/help_after.txt index fa331ed0..7d67891c 100644 --- a/src/cli/help_after.txt +++ b/src/cli/help_after.txt @@ -8,7 +8,7 @@ OBSERVATION INTERACTION click Click element (kAXPress) - double-click Open via AXOpen; physical double-click uses mouse-click + double-click Physical double-click; requires --headed triple-click Triple-click element; POLICY_DENIED if physical input is disabled right-click Right-click; includes menu when verified type Insert text; may use focus fallback only for explicit policy paths diff --git a/src/cli/mod.rs b/src/cli/mod.rs index b6d42c87..feb0a430 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -42,13 +42,13 @@ pub(crate) enum Commands { Is(IsArgs), #[command(about = "Click element via accessibility press action")] Click(RefArgs), - #[command(about = "Open element via AXOpen; physical double-click uses mouse-click")] + #[command(about = "Physically double-click element; requires --headed")] DoubleClick(RefArgs), #[command( about = "Triple-click element; returns POLICY_DENIED when physical input is disabled" )] TripleClick(RefArgs), - #[command(about = "Right-click and include menu/menu_probe details when available")] + #[command(about = "Open a context menu semantically, or physically with --headed")] RightClick(RefArgs), #[command(about = "Insert text into a text target")] Type(TypeArgs), diff --git a/src/cli/root.rs b/src/cli/root.rs index dfd501c7..f08b4a7b 100644 --- a/src/cli/root.rs +++ b/src/cli/root.rs @@ -45,7 +45,7 @@ pub(crate) struct Cli { #[arg( long, global = true, - help = "Permit cursor movement and focus stealing for physical input commands and fallbacks. Default is headless (AX-only, no cursor)." + help = "Prefer physical delivery for natural input commands and permit focus/cursor side effects. Default is strict headless semantic delivery." )] pub headed: bool, #[command(flatten)] diff --git a/src/dispatch/mod.rs b/src/dispatch/mod.rs index a097d9ae..2cb371b9 100644 --- a/src/dispatch/mod.rs +++ b/src/dispatch/mod.rs @@ -72,7 +72,7 @@ pub(crate) fn dispatch( Commands::Maximize(args) => app_window::maximize(args, adapter), Commands::Restore(args) => app_window::restore(args, adapter), Commands::ListSurfaces(args) => app_window::list_surfaces(args, adapter), - Commands::ListNotifications(args) => notifications::list(args, adapter), + Commands::ListNotifications(args) => notifications::list(args, adapter, context), Commands::DismissNotification(args) => notifications::dismiss(args, adapter, context), Commands::DismissAllNotifications(args) => { notifications::dismiss_all(args, adapter, context) diff --git a/src/dispatch/notifications.rs b/src/dispatch/notifications.rs index 3f67d279..f3a59c93 100644 --- a/src/dispatch/notifications.rs +++ b/src/dispatch/notifications.rs @@ -16,6 +16,7 @@ use crate::cli_args::notifications::{ pub(super) fn list( args: ListNotificationsCliArgs, adapter: &dyn PlatformAdapter, + context: &CommandContext, ) -> Result { list_notifications::execute( list_notifications::ListNotificationsArgs { @@ -24,6 +25,7 @@ pub(super) fn list( limit: args.limit, }, adapter, + context, ) } diff --git a/tests/e2e/json_tool.py b/tests/e2e/json_tool.py index df83632f..ebb06480 100755 --- a/tests/e2e/json_tool.py +++ b/tests/e2e/json_tool.py @@ -287,6 +287,19 @@ def command_match_value(): print("" if match["value"] is None else match["value"]) +def command_delivered_mechanism(): + data = read_json() + steps = data.get("data", {}).get("steps", []) + delivered = [ + step.get("mechanism") + for step in steps + if isinstance(step, dict) and step.get("outcome") == "succeeded" + ] + if data.get("ok") is not True or not delivered or not delivered[0]: + raise SystemExit(2) + print(delivered[0]) + + def command_tree(name, mode): data = read_json() node = find_node(data.get("data", {}).get("tree", {}), name) @@ -377,6 +390,8 @@ def main(): command_target() elif command == "match-value": command_match_value() + elif command == "delivered-mechanism": + command_delivered_mechanism() elif command == "tree": command_tree(sys.argv[2], sys.argv[3]) elif command == "duplicate-ids": diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index 38ccdbcc..bb85e57f 100644 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -202,17 +202,18 @@ wait_target() { } verify() { - local label="$1" status="$2" expected="$3" target="$4" command="$5" - shift 5 - local before after output command_ok error + local label="$1" status="$2" expected="$3" expected_mechanism="$4" target="$5" command="$6" + shift 6 + local before after output command_ok error mechanism require_value before "$status" output="$(act_target "$target" "$command" "$@" 2>&1)" sleep 0.35 require_value after "$status" command_ok="$(json_field "$output" ok)" error="$(json_field "$output" error.code)" - assert "$label" "$([ "$after" = "$expected" ] && [ "$command_ok" = "True" ] && echo 1 || echo 0)" \ - "before='$before' after='$after' expected='$expected' ok=$command_ok${error:+ error=$error}" + mechanism="$(printf '%s' "$output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)" + assert "$label" "$([ "$after" = "$expected" ] && [ "$command_ok" = "True" ] && [ "$mechanism" = "$expected_mechanism" ] && echo 1 || echo 0)" \ + "before='$before' after='$after' expected='$expected' ok=$command_ok mechanism=$mechanism${error:+ error=$error}" } run_timed() { diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 47965db6..ca8a6bc3 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -18,6 +18,12 @@ if [ ! -x "$release_bin" ]; then echo "release binary missing at $release_bin; run 'cargo build --release'" >&2 exit 2 fi +note "Strict headless non-interference gate" +AGENT_DESKTOP_E2E_RELEASE_BIN="$release_bin" bash "$here/safe-semantic.sh" +safe_semantic_status=$? +if [ "$safe_semantic_status" -ne 0 ]; then + exit "$safe_semantic_status" +fi if ! prepare_native_harness; then exit 2 fi diff --git a/tests/e2e/scenarios-interaction.sh b/tests/e2e/scenarios-interaction.sh index c3c53b5c..d6820440 100644 --- a/tests/e2e/scenarios-interaction.sh +++ b/tests/e2e/scenarios-interaction.sh @@ -1,5 +1,5 @@ interaction_suite() { - local mode="$1" selection slider stepper direction + local mode="$1" selection slider stepper direction natural_mechanism MODE_FLAG="" if [ "$mode" = "headed" ]; then MODE_FLAG="--headed" @@ -7,16 +7,18 @@ interaction_suite() { slider="60" stepper="6" direction="up" + natural_mechanism="physical_synthetic" + "$bin" focus-window --app "$app" >/dev/null 2>&1 else selection="Beta" slider="50" stepper="4" direction="down" + natural_mechanism="semantic_api" fi - "$bin" focus-window --app "$app" >/dev/null 2>&1 note "[$mode] exact-once click and text actions" - local primary text_field click_before click_after before_number after_number click_output click_ok + local primary text_field click_before click_after before_number after_number click_output click_ok click_mechanism require_target primary button primary-button require_value click_before click-status click_output="$(act_target "$primary" click 2>&1)" @@ -27,45 +29,47 @@ interaction_suite() { case "$before_number" in *[!0-9]*|'') before_number=0 ;; esac case "$after_number" in *[!0-9]*|'') after_number=0 ;; esac click_ok="$(json_field "$click_output" ok)" + click_mechanism="$(printf '%s' "$click_output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)" assert "[$mode] click dispatches exactly once" \ - "$([ "$click_ok" = "True" ] && [ "$after_number" -eq $((before_number + 1)) ] && echo 1 || echo 0)" \ - "before=$click_before after=$click_after ok=$click_ok" + "$([ "$click_ok" = "True" ] && [ "$after_number" -eq $((before_number + 1)) ] && [ "$click_mechanism" = "$natural_mechanism" ] && echo 1 || echo 0)" \ + "before=$click_before after=$click_after ok=$click_ok mechanism=$click_mechanism" require_target text_field textfield text-input act_target "$text_field" clear >/dev/null 2>&1 sleep 0.2 require_target text_field textfield text-input - verify "[$mode] type sets field" text-echo "typed-$mode" "$text_field" type "typed-$mode" + verify "[$mode] type sets field" text-echo "typed-$mode" "$natural_mechanism" "$text_field" type "typed-$mode" require_target text_field textfield text-input - verify "[$mode] set-value sets field" text-echo "set-$mode" "$text_field" set-value "set-$mode" + verify "[$mode] set-value sets field" text-echo "set-$mode" semantic_api "$text_field" set-value "set-$mode" require_target text_field textfield text-input - verify "[$mode] clear empties field" text-echo "" "$text_field" clear + verify "[$mode] clear empties field" text-echo "" "$natural_mechanism" "$text_field" clear note "[$mode] state and value controls" - local toggle picker native_slider native_stepper scroll_area scroll_before scroll_after + local toggle picker native_slider native_stepper scroll_area scroll_before scroll_after scroll_output scroll_mechanism require_target toggle checkbox toggle-box act_target "$toggle" uncheck >/dev/null 2>&1 sleep 0.2 require_target toggle checkbox toggle-box - verify "[$mode] check turns toggle on" toggle-status on "$toggle" check + verify "[$mode] check turns toggle on" toggle-status on semantic_api "$toggle" check require_target toggle checkbox toggle-box - verify "[$mode] uncheck turns toggle off" toggle-status off "$toggle" uncheck + verify "[$mode] uncheck turns toggle off" toggle-status off semantic_api "$toggle" uncheck require_target_by_id picker combobox option-picker - verify "[$mode] select combobox" picker-status "$selection" "$picker" select "$selection" + verify "[$mode] select combobox" picker-status "$selection" semantic_api "$picker" select "$selection" require_target native_slider slider value-slider - verify "[$mode] set slider value" slider-status "$slider" "$native_slider" set-value "$slider" + verify "[$mode] set slider value" slider-status "$slider" semantic_api "$native_slider" set-value "$slider" require_target native_stepper incrementor value-stepper - verify "[$mode] set stepper value" stepper-status "$stepper" "$native_stepper" set-value "$stepper" + verify "[$mode] set stepper value" stepper-status "$stepper" semantic_api "$native_stepper" set-value "$stepper" require_target scroll_area scrollarea scroll-area require_value scroll_before scroll-offset scroll_output="$(act_target "$scroll_area" scroll --direction "$direction" --amount 10 2>&1)" sleep 0.4 require_value scroll_after scroll-offset + scroll_mechanism="$(printf '%s' "$scroll_output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)" assert "[$mode] scroll moves content" \ - "$([ "$scroll_before" != "$scroll_after" ] && echo 1 || echo 0)" \ - "before=$scroll_before after=$scroll_after direction=$direction cmd_ok=$(json_field "$scroll_output" ok) cmd_err=$(json_field "$scroll_output" error.code) mechanism=$(json_field "$scroll_output" data.steps.0.mechanism)" + "$([ "$scroll_before" != "$scroll_after" ] && [ "$scroll_mechanism" = "$natural_mechanism" ] && echo 1 || echo 0)" \ + "before=$scroll_before after=$scroll_after direction=$direction cmd_ok=$(json_field "$scroll_output" ok) cmd_err=$(json_field "$scroll_output" error.code) mechanism=$scroll_mechanism" } interaction_suite headless @@ -74,11 +78,11 @@ MODE_FLAG="" note "Radio and tab selection" require_target radio_two radiobutton Two -verify "click radio option Two" radio-status Two "$radio_two" click +verify "click radio option Two" radio-status Two semantic_api "$radio_two" click require_target tab_two radiobutton "Tab Two" -verify "select Tab Two" tab-status 1 "$tab_two" click +verify "select Tab Two" tab-status 1 semantic_api "$tab_two" click require_target tab_one radiobutton "Tab One" -verify "select Tab One" tab-status 0 "$tab_one" click +verify "select Tab One" tab-status 0 semantic_api "$tab_one" click note "Headed gesture fallback" require_target double_target button double-target diff --git a/tests/e2e/scenarios-notifications.sh b/tests/e2e/scenarios-notifications.sh index dbf7ba9b..3ffa2345 100644 --- a/tests/e2e/scenarios-notifications.sh +++ b/tests/e2e/scenarios-notifications.sh @@ -27,7 +27,7 @@ if [ -n "$nc_posted" ]; then nc_list="" nc_found="" for _ in $(seq 1 20); do - nc_list="$("$bin" list-notifications --text "$nc_title_a" 2>/dev/null)" + nc_list="$("$bin" --headed list-notifications --text "$nc_title_a" 2>/dev/null)" if [ "$(json_field "$nc_list" data.count)" = "1" ]; then nc_found=1 break @@ -51,7 +51,7 @@ if [ -n "$nc_posted" ]; then nc_gone="" nc_after="" for _ in $(seq 1 12); do - nc_after="$("$bin" list-notifications --text "$nc_title_a" 2>/dev/null)" + nc_after="$("$bin" --headed list-notifications --text "$nc_title_a" 2>/dev/null)" if [ "$(json_field "$nc_after" data.count)" = "0" ]; then nc_gone=1 break @@ -65,7 +65,7 @@ if [ -n "$nc_posted" ]; then osascript -e "display notification \"$nc_body\" with title \"$nc_title_b\"" >/dev/null 2>&1 nc_list_b="" for _ in $(seq 1 10); do - nc_list_b="$("$bin" list-notifications --text "$nc_title_b" 2>/dev/null)" + nc_list_b="$("$bin" --headed list-notifications --text "$nc_title_b" 2>/dev/null)" if [ "$(json_field "$nc_list_b" data.count)" = "1" ]; then break fi @@ -77,7 +77,7 @@ if [ -n "$nc_posted" ]; then nc_cleared="" nc_left="" for _ in $(seq 1 12); do - nc_left="$("$bin" list-notifications --text "$nc_prefix" 2>/dev/null)" + nc_left="$("$bin" --headed list-notifications --text "$nc_prefix" 2>/dev/null)" if [ "$(json_field "$nc_left" data.count)" = "0" ]; then nc_cleared=1 break diff --git a/tests/e2e/test_harness_contract.py b/tests/e2e/test_harness_contract.py index 07ea9eae..c1d90084 100644 --- a/tests/e2e/test_harness_contract.py +++ b/tests/e2e/test_harness_contract.py @@ -63,6 +63,13 @@ class HarnessContractTests(unittest.TestCase): self.assertIn("require_value actionable_before click-status", source) self.assertIn("require_value actionable_after click-status", source) + def test_native_runner_gates_on_strict_headless_non_interference(self): + source = (E2E_ROOT / "run.sh").read_text() + + safe_gate = source.index('bash "$here/safe-semantic.sh"') + focused_fixture = source.index("prepare_native_harness") + self.assertLess(safe_gate, focused_fixture) + def test_fixture_status_oracles_use_stable_native_identifiers(self): source = (E2E_ROOT / "lib.sh").read_text() diff --git a/tests/e2e/test_json_tool.py b/tests/e2e/test_json_tool.py index 8124e208..86144e2a 100644 --- a/tests/e2e/test_json_tool.py +++ b/tests/e2e/test_json_tool.py @@ -5,10 +5,27 @@ import tempfile import unittest from unittest import mock -from json_tool import run_bounded +from json_tool import command_delivered_mechanism, run_bounded class RunBoundedTests(unittest.TestCase): + def test_delivered_mechanism_reports_the_first_successful_step(self): + payload = { + "ok": True, + "data": { + "steps": [ + {"outcome": "skipped", "mechanism": "physical_synthetic"}, + {"outcome": "succeeded", "mechanism": "semantic_api"}, + ] + }, + } + with mock.patch("json_tool.read_json", return_value=payload), mock.patch( + "builtins.print" + ) as output: + command_delivered_mechanism() + + output.assert_called_once_with("semantic_api") + def test_marked_child_receives_the_canonical_interaction_lease_fd(self): with tempfile.TemporaryFile() as lease: os.set_inheritable(lease.fileno(), True)