From 9ad0e889b212db53bf544524ef5fa0db079a0025 Mon Sep 17 00:00:00 2001 From: Lahfir Date: Wed, 10 Jun 2026 01:06:36 -0700 Subject: [PATCH] feat: add Playwright-style headed/headless interaction mode Ref actions now run in exactly two modes. Headless is the default: semantic accessibility operations only, no cursor movement, and a fail-closed POLICY_DENIED when only a physical gesture would work. The global --headed flag upgrades every ref action to permit focus stealing and cursor movement, so the chain's physical click/double-click/scroll/keypress fallbacks can complete. The AX path is always tried first, so --headed never regresses headless-capable elements; it only adds fallbacks for elements that need a real gesture. - CommandContext::request(action, base) builds the per-command request: each command declares its headless base (pure-AX headless; type uses focus_fallback because typing requires focus but never moves the cursor) and --headed upgrades any base to the headed policy. - The internal/FFI "physical" policy is renamed "headed" throughout, including the C ABI enum (AD_POLICY_KIND_HEADED keeps discriminant 2) and bindings. - Raw-input commands (press, hover, drag, mouse-*, key-down/up) are unchanged: always physical, mode-independent low-level escape hatch. - Unit tests assert every ref command is headless by default and headed under --headed; docs (CONCEPTS, CLAUDE, skills) describe the two-mode contract. --- CLAUDE.md | 5 +- CONCEPTS.md | 6 +- crates/core/src/action_request.rs | 4 +- crates/core/src/commands/check.rs | 4 +- crates/core/src/commands/clear.rs | 4 +- crates/core/src/commands/click.rs | 4 +- crates/core/src/commands/collapse.rs | 4 +- crates/core/src/commands/double_click.rs | 7 +- crates/core/src/commands/expand.rs | 4 +- crates/core/src/commands/focus.rs | 4 +- crates/core/src/commands/press.rs | 3 +- crates/core/src/commands/ref_policy_tests.rs | 71 +++++++++++++++++++ crates/core/src/commands/right_click.rs | 4 +- crates/core/src/commands/right_click_tests.rs | 1 + crates/core/src/commands/scroll.rs | 7 +- crates/core/src/commands/scroll_to.rs | 4 +- crates/core/src/commands/select.rs | 6 +- crates/core/src/commands/set_value.rs | 6 +- crates/core/src/commands/toggle.rs | 4 +- crates/core/src/commands/triple_click.rs | 4 +- crates/core/src/commands/type_text.rs | 9 ++- crates/core/src/commands/uncheck.rs | 4 +- crates/core/src/context.rs | 31 +++++++- crates/core/src/interaction_policy.rs | 2 +- crates/ffi/include/agent_desktop.h | 2 +- crates/ffi/src/actions/execute.rs | 6 +- crates/ffi/src/enum_validation.rs | 2 +- crates/ffi/src/types/policy_kind.rs | 4 +- crates/ffi/tests/c_abi_actions.rs | 2 +- skills/agent-desktop/SKILL.md | 1 + .../references/commands-interaction.md | 25 ++++--- src/cli/mod.rs | 7 ++ src/main.rs | 2 +- 33 files changed, 191 insertions(+), 62 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 51ce2aa..c864321 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,10 +15,13 @@ cargo clippy --all-targets -- -D warnings # Lint (must pass, zero warnings) cargo fmt --all -- --check # Format check cargo fmt --all # Auto-format cargo tree -p agent-desktop-core # Verify no platform crate leaks (CI enforces) +bash tests/e2e/run.sh # E2E: real binary vs fixture app, verify by observation (needs --release + AX permission) ``` Run the binary: `./target/release/agent-desktop snapshot --app Finder -i` +The E2E harness drives the release binary against a real SwiftUI/AppKit fixture and asserts every effect by independent observation (never the command's own `ok:true`), covering every ref action in **both** headless and `--headed` mode. See `tests/e2e/README.md`. + ## Pre-commit Hook The repo ships a pre-commit hook at `.githooks/pre-commit` that runs `cargo fmt --check`, `cargo clippy --all-targets -- -D warnings`, and `cargo test --lib --workspace` against staged Rust changes. Wire it up once after cloning: @@ -348,7 +351,7 @@ pub trait PlatformAdapter { - `AccessibilityNode` — platform-agnostic tree node: `ref`, `role`, `name`, `value`, `description`, `states`, `available_actions`, `bounds`, `children` - `Action` — Click, DoubleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Scroll(Direction, Amount), PressKey(KeyCombo) -- `ActionRequest` — `{ action, policy }`, where the default `InteractionPolicy` forbids focus stealing and cursor movement +- `ActionRequest` — `{ action, policy }`, where the default headless `InteractionPolicy` forbids focus stealing and cursor movement; the global `--headed` flag selects the headed policy that permits both so physical fallbacks can complete - `NativeHandle` — opaque platform pointer with `PhantomData<*const ()>` to prevent auto-Send/Sync. Inner field is `pub(crate)`. - `RefEntry` — `{ pid, role, name, bounds_hash, available_actions }` - `WindowInfo` — `{ id, title, app_name, pid, bounds }` diff --git a/CONCEPTS.md b/CONCEPTS.md index afe0d0b..1b5f889 100644 --- a/CONCEPTS.md +++ b/CONCEPTS.md @@ -55,12 +55,12 @@ The platform-neutral set of supported action names that core uses to compare com Each adapter maps native primitives into this shared vocabulary before core evaluates actionability. New commands should extend the central vocabulary first, then reuse it from actionability, ref allocation, predicates, FFI tests, and platform adapters. ### Interaction Policy -The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input fallbacks. +The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input fallbacks. Ref commands expose exactly two modes: **headless** (the default — accessibility-only, no cursor, fails closed when the AX path is unavailable) and **headed** (opt-in via the global `--headed` flag — permits focus stealing and cursor movement so the action chain's physical fallbacks can complete). The AX path is always tried first, so headed never regresses headless-capable elements. The `headed` upgrade applies uniformly to every ref command; each command still declares its own headless base policy (most are pure-AX; `type` uses a focus-fallback base because typing requires focus but never moves the cursor). ### Headless Ref Action -A ref-based action that uses semantic accessibility operations without implicit focus stealing, cursor movement, synthetic keyboard input, or pasteboard use. +A ref-based action that uses semantic accessibility operations without implicit focus stealing, cursor movement, synthetic keyboard input, or pasteboard use. This is the default mode. -Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation. A broader interaction policy must be explicit rather than silently substituting physical input. +Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation; they fail closed with `POLICY_DENIED` rather than silently substituting physical input. The broader **headed** policy must be selected explicitly with `--headed`. ### Wait Predicate The condition a wait command polls for before returning, such as element actionability, text presence, window appearance, menu state, or notification arrival. diff --git a/crates/core/src/action_request.rs b/crates/core/src/action_request.rs index f269c41..b5f4155 100644 --- a/crates/core/src/action_request.rs +++ b/crates/core/src/action_request.rs @@ -23,10 +23,10 @@ impl ActionRequest { } } - pub fn physical(action: Action) -> Self { + pub fn headed(action: Action) -> Self { Self { action, - policy: InteractionPolicy::physical(), + policy: InteractionPolicy::headed(), } } } diff --git a/crates/core/src/commands/check.rs b/crates/core/src/commands/check.rs index 582dafa..f05435d 100644 --- a/crates/core/src/commands/check.rs +++ b/crates/core/src/commands/check.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Check), + context.request(Action::Check, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/clear.rs b/crates/core/src/commands/clear.rs index f9cd6fe..60de3a7 100644 --- a/crates/core/src/commands/clear.rs +++ b/crates/core/src/commands/clear.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Clear), + context.request(Action::Clear, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/click.rs b/crates/core/src/commands/click.rs index c8a6c21..61f3c78 100644 --- a/crates/core/src/commands/click.rs +++ b/crates/core/src/commands/click.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Click), + context.request(Action::Click, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/collapse.rs b/crates/core/src/commands/collapse.rs index 5cda85a..b904cbb 100644 --- a/crates/core/src/commands/collapse.rs +++ b/crates/core/src/commands/collapse.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Collapse), + context.request(Action::Collapse, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/double_click.rs b/crates/core/src/commands/double_click.rs index f646bed..2344aac 100644 --- a/crates/core/src/commands/double_click.rs +++ b/crates/core/src/commands/double_click.rs @@ -1,13 +1,16 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; +/// Double-click tries AXOpen (headless) first. Without `--headed` and no +/// AXOpen it fails closed; with `--headed` the chain may perform a physical +/// double-click. pub fn execute( args: RefArgs, adapter: &dyn PlatformAdapter, @@ -16,7 +19,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::DoubleClick), + context.request(Action::DoubleClick, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/expand.rs b/crates/core/src/commands/expand.rs index 4fd9c4f..f3d2fbc 100644 --- a/crates/core/src/commands/expand.rs +++ b/crates/core/src/commands/expand.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Expand), + context.request(Action::Expand, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/focus.rs b/crates/core/src/commands/focus.rs index 1f306e5..cdda016 100644 --- a/crates/core/src/commands/focus.rs +++ b/crates/core/src/commands/focus.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::SetFocus), + context.request(Action::SetFocus, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/press.rs b/crates/core/src/commands/press.rs index ceb27a0..48dfe6a 100644 --- a/crates/core/src/commands/press.rs +++ b/crates/core/src/commands/press.rs @@ -36,8 +36,7 @@ pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result Result { - let request = ActionRequest::headless(Action::RightClick); + let request = context.request(Action::RightClick, InteractionPolicy::headless()); let (entry, result) = execute_ref_action_result_with_context( &args.ref_id, args.snapshot_id.as_deref(), diff --git a/crates/core/src/commands/right_click_tests.rs b/crates/core/src/commands/right_click_tests.rs index 93cba06..701dda2 100644 --- a/crates/core/src/commands/right_click_tests.rs +++ b/crates/core/src/commands/right_click_tests.rs @@ -1,5 +1,6 @@ use super::*; use crate::{ + action_request::ActionRequest, action_result::ActionResult, adapter::{NativeHandle, WindowFilter}, error::{AdapterError, ErrorCode}, diff --git a/crates/core/src/commands/scroll.rs b/crates/core/src/commands/scroll.rs index 5ec5762..ace5f00 100644 --- a/crates/core/src/commands/scroll.rs +++ b/crates/core/src/commands/scroll.rs @@ -1,10 +1,10 @@ use crate::{ action::{Action, Direction}, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::execute_ref_action_result_with_context, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -20,7 +20,10 @@ pub fn execute( adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { - let request = ActionRequest::headless(Action::Scroll(args.direction, args.amount)); + let request = context.request( + Action::Scroll(args.direction, args.amount), + InteractionPolicy::headless(), + ); let (_entry, result) = execute_ref_action_result_with_context( &args.ref_id, args.snapshot_id.as_deref(), diff --git a/crates/core/src/commands/scroll_to.rs b/crates/core/src/commands/scroll_to.rs index d629d8d..3d32c07 100644 --- a/crates/core/src/commands/scroll_to.rs +++ b/crates/core/src/commands/scroll_to.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::ScrollTo), + context.request(Action::ScrollTo, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/select.rs b/crates/core/src/commands/select.rs index b13e7cd..f396d15 100644 --- a/crates/core/src/commands/select.rs +++ b/crates/core/src/commands/select.rs @@ -1,7 +1,7 @@ use crate::{ - action::Action, action_request::ActionRequest, adapter::PlatformAdapter, + action::Action, adapter::PlatformAdapter, commands::helpers::execute_ref_action_result_with_context, context::CommandContext, - error::AppError, + error::AppError, interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { - let request = ActionRequest::headless(Action::Select(args.value)); + let request = context.request(Action::Select(args.value), InteractionPolicy::headless()); let (_entry, result) = execute_ref_action_result_with_context( &args.ref_id, args.snapshot_id.as_deref(), diff --git a/crates/core/src/commands/set_value.rs b/crates/core/src/commands/set_value.rs index aaae824..dc67d2a 100644 --- a/crates/core/src/commands/set_value.rs +++ b/crates/core/src/commands/set_value.rs @@ -1,7 +1,7 @@ use crate::{ - action::Action, action_request::ActionRequest, adapter::PlatformAdapter, + action::Action, adapter::PlatformAdapter, commands::helpers::execute_ref_action_result_with_context, context::CommandContext, - error::AppError, + error::AppError, interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( adapter: &dyn PlatformAdapter, context: &CommandContext, ) -> Result { - let request = ActionRequest::headless(Action::SetValue(args.value)); + let request = context.request(Action::SetValue(args.value), InteractionPolicy::headless()); let (_entry, result) = execute_ref_action_result_with_context( &args.ref_id, args.snapshot_id.as_deref(), diff --git a/crates/core/src/commands/toggle.rs b/crates/core/src/commands/toggle.rs index d2e8e24..c55c8c0 100644 --- a/crates/core/src/commands/toggle.rs +++ b/crates/core/src/commands/toggle.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Toggle), + context.request(Action::Toggle, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/triple_click.rs b/crates/core/src/commands/triple_click.rs index b55c27d..35ff655 100644 --- a/crates/core/src/commands/triple_click.rs +++ b/crates/core/src/commands/triple_click.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::TripleClick), + context.request(Action::TripleClick, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/commands/type_text.rs b/crates/core/src/commands/type_text.rs index 18e1235..7855d84 100644 --- a/crates/core/src/commands/type_text.rs +++ b/crates/core/src/commands/type_text.rs @@ -1,7 +1,7 @@ use crate::{ - action::Action, action_request::ActionRequest, adapter::PlatformAdapter, + action::Action, adapter::PlatformAdapter, commands::helpers::execute_ref_action_result_with_context, context::CommandContext, - error::AppError, + error::AppError, interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -24,7 +24,10 @@ pub fn execute( ))); } - let request = ActionRequest::focus_fallback(Action::TypeText(args.text)); + let request = context.request( + Action::TypeText(args.text), + InteractionPolicy::focus_fallback(), + ); let (_entry, result) = execute_ref_action_result_with_context( &args.ref_id, args.snapshot_id.as_deref(), diff --git a/crates/core/src/commands/uncheck.rs b/crates/core/src/commands/uncheck.rs index 444f4a1..58e6ed8 100644 --- a/crates/core/src/commands/uncheck.rs +++ b/crates/core/src/commands/uncheck.rs @@ -1,10 +1,10 @@ use crate::{ action::Action, - action_request::ActionRequest, adapter::PlatformAdapter, commands::helpers::{RefArgs, execute_ref_action_with_context}, context::CommandContext, error::AppError, + interaction_policy::InteractionPolicy, }; use serde_json::Value; @@ -16,7 +16,7 @@ pub fn execute( execute_ref_action_with_context( args, adapter, - ActionRequest::headless(Action::Uncheck), + context.request(Action::Uncheck, InteractionPolicy::headless()), context, ) } diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index 675d195..fe75a8e 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -1,4 +1,7 @@ -use crate::{error::AppError, trace::TraceConfig}; +use crate::{ + action::Action, action_request::ActionRequest, error::AppError, + interaction_policy::InteractionPolicy, trace::TraceConfig, +}; use serde_json::Value; use std::path::{Path, PathBuf}; @@ -6,6 +9,7 @@ use std::path::{Path, PathBuf}; pub struct CommandContext { session_id: Option, trace: TraceConfig, + headed: bool, } impl CommandContext { @@ -20,9 +24,33 @@ impl CommandContext { Ok(Self { session_id, trace: TraceConfig::new(trace_path, trace_strict)?, + headed: false, }) } + /// Selects headed interaction: ref actions may move the cursor and steal + /// focus, unlocking the physical click/scroll/keypress fallbacks in the + /// action chain. Off by default — the tool is headless-first (Playwright + /// style: headless is the default, headed is opt-in via `--headed`). + pub fn with_headed(mut self, headed: bool) -> Self { + self.headed = headed; + self + } + + /// Builds the action request for a ref command. Headless (default) uses the + /// command's own `base` policy — its minimum viable policy with no cursor + /// movement (most commands are pure-AX `headless`; `type` is `focus_fallback` + /// because typing requires focus). `--headed` upgrades any base to `headed`, + /// unlocking the cursor/OS-input fallbacks instead of failing closed. + pub fn request(&self, action: Action, base: InteractionPolicy) -> ActionRequest { + let policy = if self.headed { + InteractionPolicy::headed() + } else { + base + }; + ActionRequest { action, policy } + } + pub fn for_batch_item(&self, session_id: Option) -> Result { let session_id = session_id.or_else(|| self.session_id.clone()); if let Some(id) = session_id.as_deref() { @@ -31,6 +59,7 @@ impl CommandContext { Ok(Self { session_id, trace: self.trace.clone(), + headed: self.headed, }) } diff --git a/crates/core/src/interaction_policy.rs b/crates/core/src/interaction_policy.rs index db694b0..d612de9 100644 --- a/crates/core/src/interaction_policy.rs +++ b/crates/core/src/interaction_policy.rs @@ -21,7 +21,7 @@ impl InteractionPolicy { } } - pub fn physical() -> Self { + pub fn headed() -> Self { Self { allow_focus_steal: true, allow_cursor_move: true, diff --git a/crates/ffi/include/agent_desktop.h b/crates/ffi/include/agent_desktop.h index 437bf1c..f720d26 100644 --- a/crates/ffi/include/agent_desktop.h +++ b/crates/ffi/include/agent_desktop.h @@ -92,7 +92,7 @@ typedef int32_t AdMouseEventKind; enum AdPolicyKind { AD_POLICY_KIND_HEADLESS = 0, AD_POLICY_KIND_FOCUS_FALLBACK = 1, - AD_POLICY_KIND_PHYSICAL = 2, + AD_POLICY_KIND_HEADED = 2, }; typedef int32_t AdPolicyKind; diff --git a/crates/ffi/src/actions/execute.rs b/crates/ffi/src/actions/execute.rs index 827cedc..665ceb8 100644 --- a/crates/ffi/src/actions/execute.rs +++ b/crates/ffi/src/actions/execute.rs @@ -164,7 +164,7 @@ fn action_request( match policy { AdPolicyKind::Headless => ActionRequest::headless(action), AdPolicyKind::FocusFallback => ActionRequest::focus_fallback(action), - AdPolicyKind::Physical => ActionRequest::physical(action), + AdPolicyKind::Headed => ActionRequest::headed(action), } } @@ -184,8 +184,8 @@ mod tests { InteractionPolicy::focus_fallback() ); assert_eq!( - action_request(AdPolicyKind::Physical, Action::Click).policy, - InteractionPolicy::physical() + action_request(AdPolicyKind::Headed, Action::Click).policy, + InteractionPolicy::headed() ); } } diff --git a/crates/ffi/src/enum_validation.rs b/crates/ffi/src/enum_validation.rs index 5693111..9eca472 100644 --- a/crates/ffi/src/enum_validation.rs +++ b/crates/ffi/src/enum_validation.rs @@ -40,7 +40,7 @@ try_from_c_enum! { try_from_c_enum! { AdPolicyKind { - Headless = 0, FocusFallback = 1, Physical = 2, + Headless = 0, FocusFallback = 1, Headed = 2, } } diff --git a/crates/ffi/src/types/policy_kind.rs b/crates/ffi/src/types/policy_kind.rs index 94fecda..7c0ab6a 100644 --- a/crates/ffi/src/types/policy_kind.rs +++ b/crates/ffi/src/types/policy_kind.rs @@ -3,7 +3,7 @@ pub enum AdPolicyKind { Headless = 0, FocusFallback = 1, - Physical = 2, + Headed = 2, } #[cfg(test)] @@ -14,6 +14,6 @@ mod tests { fn discriminants_are_abi_stable() { assert_eq!(AdPolicyKind::Headless as i32, 0); assert_eq!(AdPolicyKind::FocusFallback as i32, 1); - assert_eq!(AdPolicyKind::Physical as i32, 2); + assert_eq!(AdPolicyKind::Headed as i32, 2); } } diff --git a/crates/ffi/tests/c_abi_actions.rs b/crates/ffi/tests/c_abi_actions.rs index 62f2eb5..6781f09 100644 --- a/crates/ffi/tests/c_abi_actions.rs +++ b/crates/ffi/tests/c_abi_actions.rs @@ -36,7 +36,7 @@ fn invalid_policy_discriminant_rejected_without_ub() { adapter, &handle, &action, - AdPolicyKind::Physical as i32 + 1, + AdPolicyKind::Headed as i32 + 1, &mut out, ); assert!(matches!( diff --git a/skills/agent-desktop/SKILL.md b/skills/agent-desktop/SKILL.md index c56c668..aff5958 100644 --- a/skills/agent-desktop/SKILL.md +++ b/skills/agent-desktop/SKILL.md @@ -94,6 +94,7 @@ Use **progressive skeleton traversal** as the default approach. It reduces token - **Scoped invalidation:** re-drilling `--root @e3` only replaces refs from @e3's previous drill — refs from other regions and the skeleton itself are preserved - **Strict resolution:** stale refs return `STALE_REF`; duplicate plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily. - **Actionability:** ref actions check live visibility, stability, enabled state, supported action, policy, and editability before dispatch. +- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed with `POLICY_DENIED` when only a physical gesture would work. Pass the global `--headed` flag to permit cursor movement and focus stealing so the physical click/double-click/scroll/keypress fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`/`key-up`) are always physical and ignore the mode. - **Sessions:** use `--session ` for concurrent or multi-agent runs that share a latest snapshot pointer; batch entries may override with `"session": "id"`. - **Trace:** use `--trace ` for JSONL diagnostics outside stdout; `--trace-strict` fails on trace setup and pre-action writes. Post-action success traces are best-effort because the desktop mutation already happened. Trace fields whose keys contain `text`, `value`, `expected`, `name`, `description`, `message`, `label`, `query`, `secret`, `token`, or `password` are redacted to `{ "redacted": true, "chars_bucket": "..." }` at every nesting depth — do not expect raw values in trace files. Top-level `--trace` is inherited by every `batch` entry, including entries with a `session` override. diff --git a/skills/agent-desktop/references/commands-interaction.md b/skills/agent-desktop/references/commands-interaction.md index f6bb0ad..2be0a54 100644 --- a/skills/agent-desktop/references/commands-interaction.md +++ b/skills/agent-desktop/references/commands-interaction.md @@ -2,17 +2,26 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and input synthesis. -Ref-based actions are headless by default. They try semantic accessibility operations and do not silently steal focus, move the cursor, synthesize keyboard input, or use the pasteboard. Physical/headed interaction is reserved for explicit `focus`, `press`, `hover`, `drag`, and `mouse-*` commands or an explicit FFI policy. The `type` command has an explicit focus-fallback tier for callers that opt into focus changes while still forbidding cursor movement; the default CLI path remains AX-value-first and headless. +### Headless (default) vs `--headed` + +Ref-based actions run in two modes, Playwright-style: + +- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it **fails closed** with `POLICY_DENIED` rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.) +- **`--headed`.** A global flag (`agent-desktop --headed click @e5`) that upgrades every ref action to permit focus stealing **and** cursor movement, unlocking the physical click/double-click/scroll/keypress fallbacks in the action chain. The AX path is still tried first, so `--headed` never regresses elements that work headlessly — it only adds fallbacks for elements that need a real gesture (e.g. a gesture-only button with no `AXOpen`). + +Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`, `key-up`) are always physical by nature and ignore the mode — they are the explicit low-level escape hatch. + +`--headed` is a global flag and also applies to every `batch` entry. All ref-based interaction commands accept `--snapshot `. Omit it for the active session's latest saved snapshot, or pass the `snapshot_id` returned by `snapshot` to keep scripts pinned to the exact ref map they observed. Explicit snapshot IDs do not require also passing `--session`. Success responses for ref actions include a `steps` array when the activation chain recorded attempts: each entry is `{ "label": "AXPress", "outcome": "attempted" | "skipped" | "succeeded" }` in execution order, showing which activation path produced the result. -When the actionability preflight blocks an action, the error envelope carries the full report in `error.details`: `{ "actionable": false, "checks": [ { "name": "...", "status": "...", "reason": "..." } ] }`. Check names are `visible`, `stable`, `enabled`, `supported_action`, `policy`, and `editable`; statuses are `pass`, `fail`, and `unknown`. Use the failing check's `reason` to pick recovery: `wait --element --predicate actionable`, a fresh snapshot, or an explicit focus/physical command when intended. +When the actionability preflight blocks an action, the error envelope carries the full report in `error.details`: `{ "actionable": false, "checks": [ { "name": "...", "status": "...", "reason": "..." } ] }`. Check names are `visible`, `stable`, `enabled`, `supported_action`, `policy`, and `editable`; statuses are `pass`, `fail`, and `unknown`. Use the failing check's `reason` to pick recovery: `wait --element --predicate actionable`, a fresh snapshot, or `--headed` when a `policy` check failed and a physical gesture is intended. ## Click Actions -Click commands use semantic AX activation first. In the default headless policy, coordinate click fallback is blocked; use `mouse-click` only when physical cursor movement is intended. +Click commands use semantic AX activation first. In the default headless mode, coordinate click fallback is blocked; pass `--headed` to allow the physical click fallback, or use `mouse-click` for a raw coordinate click. ### click ```bash @@ -25,19 +34,19 @@ Primary activation. Tries verified AXPress > AXConfirm > AXOpen > AXPick > child ```bash agent-desktop double-click @e3 ``` -Tries AXOpen. Physical double-click fallback is blocked by default policy; use `mouse-click --xy X,Y --count 2` when a headed physical double-click is intended. +Tries AXOpen (headless). When the element advertises no `AXOpen`, the headless command fails closed with `POLICY_DENIED`; pass `--headed` to perform a real double-click (`agent-desktop --headed double-click @e3`), or use `mouse-click --xy X,Y --count 2` for a raw coordinate double-click. ### triple-click ```bash agent-desktop triple-click @e2 ``` -Physical triple-click requires cursor/focus side effects and is blocked by default policy; use `mouse-click --xy X,Y --count 3` when a headed physical triple-click is intended. +Triple-click requires cursor/focus side effects and is blocked in headless mode; pass `--headed` (`agent-desktop --headed triple-click @e2`), or use `mouse-click --xy X,Y --count 3` for a raw coordinate triple-click. ### right-click ```bash agent-desktop right-click @e5 ``` -Performs a semantic right-click/context-menu action and includes the menu tree when a menu surface can be verified. If the right-click action succeeds but menu probing fails, the command still returns the action result with `menu_probe.ok: false` so callers do not retry and double-open context menus. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked by default policy. +Performs a semantic right-click/context-menu action and includes the menu tree when a menu surface can be verified. If the right-click action succeeds but menu probing fails, the command still returns the action result with `menu_probe.ok: false` so callers do not retry and double-open context menus. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked in headless mode; pass `--headed` to allow them. ## Text Input @@ -46,9 +55,9 @@ Performs a semantic right-click/context-menu action and includes the menu tree w agent-desktop type @e2 "hello@example.com" agent-desktop type @e2 "multi line\ntext" ``` -In the default headless policy, inserts text by mutating the element's AX value when the target has a settable text value. If a target cannot be updated headlessly, the command returns a structured error instead of stealing focus. Physical keyboard synthesis and pasteboard-based insertion are reserved for explicit policy paths. +In headless mode (default), `type` inserts text by mutating the element's AX value and may focus the target field (typing requires focus) but never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs). -When an explicit focus/physical policy is used for non-ASCII text on macOS, the adapter may briefly place the text on the clipboard to paste it. Do not use that path for secrets; prefer the default headless value path or `set-value` when the target supports it. +Under `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer the default value path or `set-value` when the target supports it. ### set-value ```bash diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 54d2640..40edaed 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -60,6 +60,13 @@ pub(crate) struct Cli { )] pub trace_strict: bool, + #[arg( + long, + global = true, + help = "Run ref actions headed: permit cursor movement and focus stealing so physical click/scroll/keypress fallbacks can complete. Default is headless (AX-only, no cursor)." + )] + pub headed: bool, + #[command(subcommand)] pub command: Option, } diff --git a/src/main.rs b/src/main.rs index f5cbb78..1367be9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,7 +36,7 @@ fn main() { init_tracing(cli.verbose); let context = match CommandContext::new(cli.session, cli.trace, cli.trace_strict) { - Ok(context) => context, + Ok(context) => context.with_headed(cli.headed), Err(err) => { finish("unknown", Err(err)); return;