mirror of
https://github.com/lahfir/agent-desktop.git
synced 2026-08-09 00:17:27 +00:00
fix: enforce explicit headed delivery
Keep semantic ref actions strictly headless, make natural headed actions physical-first, and prevent notification observation from opening system UI without headed permission. Gate the native suite on background non-interference and assert delivered mechanisms.
This commit is contained in:
Lahfir
parent
ef5c90cec9
commit
ec247a6f7d
55 changed files with 325 additions and 305 deletions
19
README.md
19
README.md
|
|
@ -190,8 +190,9 @@ agent-desktop list-surfaces --app Notes # list menus, sheets, popovers,
|
|||
### Interaction
|
||||
|
||||
```bash
|
||||
agent-desktop click @s8f3k2p9:e3 # semantic AX-first click
|
||||
agent-desktop double-click @s8f3k2p9:e3 # AXOpen; physical double-click uses --headed mouse-click --count 2
|
||||
agent-desktop click @s8f3k2p9:e3 # strict headless AX click
|
||||
agent-desktop --headed click @s8f3k2p9:e3 # physical click, focus/cursor allowed
|
||||
agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click
|
||||
agent-desktop triple-click @s8f3k2p9:e3 # POLICY_DENIED if physical input is disabled
|
||||
agent-desktop right-click @s8f3k2p9:e3 # open context menu; inspect effect before retrying
|
||||
agent-desktop type @s8f3k2p9:e5 "hello world" # insert text into element
|
||||
|
|
@ -204,11 +205,11 @@ agent-desktop check @s8f3k2p9:e12 # idempotent check
|
|||
agent-desktop uncheck @s8f3k2p9:e12 # idempotent uncheck
|
||||
agent-desktop expand @s8f3k2p9:e15 # expand disclosure/tree item
|
||||
agent-desktop collapse @s8f3k2p9:e15 # collapse disclosure/tree item
|
||||
agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # scroll (AX-first)
|
||||
agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # strict headless AX scroll
|
||||
agent-desktop scroll-to @s8f3k2p9:e20 # scroll element into view
|
||||
```
|
||||
|
||||
> **(macOS, Phase 1)** Pure cursor gestures have no accessibility equivalent, so `triple-click`, `hover`, and `drag` are always physical; `double-click` is headless via `AXOpen` and only needs `--headed` for gesture-only targets. Windows (UIA) and Linux (AT-SPI) adapters may expose different capabilities. See `skills/agent-desktop/references/commands-interaction.md`.
|
||||
> **(macOS, Phase 1)** Default ref actions are strict headless semantic operations. `--headed` prefers physical delivery for natural input commands; double/triple-click, hover, and drag are physical-only. Semantic-only commands remain semantic. See `skills/agent-desktop/references/commands-interaction.md`.
|
||||
|
||||
### Keyboard
|
||||
|
||||
|
|
@ -253,9 +254,9 @@ agent-desktop restore --window-id w-4521
|
|||
### Notifications *(macOS only)*
|
||||
|
||||
```bash
|
||||
agent-desktop list-notifications # list all notifications
|
||||
agent-desktop list-notifications --app "Slack" # filter by app
|
||||
agent-desktop list-notifications --text "deploy" --limit 5 # filter by text
|
||||
agent-desktop --headed list-notifications # open Notification Center if needed, then list
|
||||
agent-desktop --headed list-notifications --app "Slack" # filter by app
|
||||
agent-desktop --headed list-notifications --text "deploy" --limit 5 # filter by text
|
||||
agent-desktop dismiss-notification 1 --expected-app "Slack" --expected-title "Deploy complete"
|
||||
agent-desktop dismiss-all-notifications # dismiss all
|
||||
agent-desktop dismiss-all-notifications --app "Slack" # dismiss all from app
|
||||
|
|
@ -263,8 +264,8 @@ agent-desktop notification-action 1 "Reply" --expected-app "Slack" --expected-ti
|
|||
```
|
||||
|
||||
Single-notification mutations require an app or title fingerprint from the
|
||||
same listing. Semantic dismiss is headless; pass global `--headed` only when
|
||||
Notification Center requires its hover-revealed close control.
|
||||
same listing. Headless listing observes an already-open Notification Center;
|
||||
pass `--headed` to allow opening it and restoring the prior frontmost app.
|
||||
|
||||
### Clipboard
|
||||
|
||||
|
|
|
|||
|
|
@ -95,10 +95,9 @@ impl Action {
|
|||
matches!(self, Self::TypeText(_) | Self::PressKey(_))
|
||||
}
|
||||
|
||||
/// Returns the least-permissive interaction policy that can execute this
|
||||
/// action with the same fallback behavior as its command entrypoint.
|
||||
/// Returns the command's minimum interaction policy.
|
||||
pub fn base_interaction_policy(&self) -> crate::interaction_policy::InteractionPolicy {
|
||||
if self.may_use_focus_fallback() {
|
||||
if matches!(self, Self::PressKey(_)) {
|
||||
crate::interaction_policy::InteractionPolicy::focus_fallback()
|
||||
} else {
|
||||
crate::interaction_policy::InteractionPolicy::headless()
|
||||
|
|
|
|||
|
|
@ -71,21 +71,21 @@ fn pure_ax_actions_base_policy_is_headless() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn press_key_and_type_text_base_policy_is_focus_fallback() {
|
||||
let focus_fallback = InteractionPolicy::focus_fallback();
|
||||
fn type_text_is_headless_but_explicit_press_allows_focus() {
|
||||
let headless = InteractionPolicy::headless();
|
||||
assert_eq!(
|
||||
Action::PressKey(KeyCombo {
|
||||
key: "a".into(),
|
||||
modifiers: vec![Modifier::Meta],
|
||||
})
|
||||
.base_interaction_policy(),
|
||||
focus_fallback,
|
||||
"PressKey must permit the focus fallback needed to target keystrokes"
|
||||
InteractionPolicy::focus_fallback(),
|
||||
"PressKey is an explicit physical-input command"
|
||||
);
|
||||
assert_eq!(
|
||||
Action::TypeText("hello".into()).base_interaction_policy(),
|
||||
focus_fallback,
|
||||
"TypeText must permit the focus fallback needed by physical typing"
|
||||
headless,
|
||||
"TypeText must not gain implicit focus or keyboard permission"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ pub(super) fn check_with_stability(
|
|||
) -> Result<ActionabilityReport, AdapterError> {
|
||||
let requirements = ActionabilityRequirements::for_action(&request.action);
|
||||
let pointer_delivery =
|
||||
requirements.pointer_delivery(&request.action, &evidence.available_actions);
|
||||
requirements.pointer_delivery(&request.action, &evidence.available_actions, request.policy);
|
||||
let mut checks = Vec::new();
|
||||
let mut verified_point = None;
|
||||
if requirements.visible {
|
||||
|
|
|
|||
|
|
@ -44,11 +44,17 @@ impl ActionabilityRequirements {
|
|||
&self,
|
||||
action: &Action,
|
||||
available_actions: &[String],
|
||||
policy: crate::InteractionPolicy,
|
||||
) -> PointerDelivery {
|
||||
if !self.receives_events {
|
||||
return PointerDelivery::NotApplicable;
|
||||
}
|
||||
if crate::capability::supports_direct_semantic_pointer_delivery(action, available_actions) {
|
||||
if policy.is_headed() {
|
||||
PointerDelivery::Physical
|
||||
} else if crate::capability::supports_direct_semantic_pointer_delivery(
|
||||
action,
|
||||
available_actions,
|
||||
) {
|
||||
PointerDelivery::Semantic
|
||||
} else {
|
||||
PointerDelivery::Physical
|
||||
|
|
|
|||
|
|
@ -31,15 +31,31 @@ fn pointer_delivery_is_selected_from_live_semantic_capability_evidence() {
|
|||
let requirements = ActionabilityRequirements::for_action(&Action::Click);
|
||||
|
||||
assert_eq!(
|
||||
requirements.pointer_delivery(&Action::Click, &[crate::capability::CLICK.into()]),
|
||||
requirements.pointer_delivery(
|
||||
&Action::Click,
|
||||
&[crate::capability::CLICK.into()],
|
||||
crate::InteractionPolicy::headless(),
|
||||
),
|
||||
PointerDelivery::Semantic
|
||||
);
|
||||
assert_eq!(
|
||||
requirements.pointer_delivery(&Action::Click, &[]),
|
||||
requirements.pointer_delivery(&Action::Click, &[], crate::InteractionPolicy::headless(),),
|
||||
PointerDelivery::Physical
|
||||
);
|
||||
assert_eq!(
|
||||
requirements.pointer_delivery(&Action::DoubleClick, &[crate::capability::CLICK.into()]),
|
||||
requirements.pointer_delivery(
|
||||
&Action::DoubleClick,
|
||||
&[crate::capability::CLICK.into()],
|
||||
crate::InteractionPolicy::headless(),
|
||||
),
|
||||
PointerDelivery::Physical
|
||||
);
|
||||
assert_eq!(
|
||||
requirements.pointer_delivery(
|
||||
&Action::Click,
|
||||
&[crate::capability::CLICK.into()],
|
||||
crate::InteractionPolicy::headed(),
|
||||
),
|
||||
PointerDelivery::Physical
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
use crate::{
|
||||
AdapterError, AdapterSession, AppInfo, Deadline, DismissAllNotificationsRequest,
|
||||
DismissNotificationRequest, ImageBuffer, InteractionLease, KeyCombo, NotificationActionRequest,
|
||||
NotificationFilter, NotificationInfo, PermissionReport, PermissionState, ProcessIdentity,
|
||||
SessionAffinity, SignalBaseline, SignalFilter, WindowInfo, WindowOp,
|
||||
action_result::ActionResult, display_info::DisplayInfo, screenshot_target::ScreenshotTarget,
|
||||
DismissNotificationRequest, ImageBuffer, InteractionLease, InteractionPolicy, KeyCombo,
|
||||
NotificationActionRequest, NotificationFilter, NotificationInfo, PermissionReport,
|
||||
PermissionState, ProcessIdentity, SessionAffinity, SignalBaseline, SignalFilter, WindowInfo,
|
||||
WindowOp, action_result::ActionResult, display_info::DisplayInfo,
|
||||
screenshot_target::ScreenshotTarget,
|
||||
};
|
||||
|
||||
pub trait SystemOps: Send + Sync {
|
||||
|
|
@ -192,6 +193,7 @@ pub trait SystemOps: Send + Sync {
|
|||
fn list_notifications(
|
||||
&self,
|
||||
_filter: &NotificationFilter,
|
||||
_policy: InteractionPolicy,
|
||||
_deadline: Deadline,
|
||||
) -> Result<Vec<NotificationInfo>, AdapterError> {
|
||||
Err(AdapterError::not_supported("list_notifications"))
|
||||
|
|
|
|||
|
|
@ -31,13 +31,9 @@ pub struct ExecuteByRefArgs<'a> {
|
|||
/// explicit ID must match the qualified ref.
|
||||
///
|
||||
/// The effective `InteractionPolicy` is the join of `caller_policy` and the
|
||||
/// action's CLI base policy, ensuring the result is always at least as
|
||||
/// permissive as what the CLI would use for the same action, while allowing
|
||||
/// FFI callers to opt in to higher-permission policies such as `headed`.
|
||||
///
|
||||
/// Note on PressKey: its base policy is `focus_fallback` (derived from
|
||||
/// `Action::base_interaction_policy`, shared with `TypeText`) because a
|
||||
/// ref-targeted key press may need the target focused for keystrokes to land.
|
||||
/// action's base policy. Semantic actions, including `TypeText`, are strict
|
||||
/// headless; explicit `PressKey` permits focus without cursor movement. FFI
|
||||
/// callers may opt in to `focus_fallback` or `headed`.
|
||||
pub fn execute(
|
||||
args: ExecuteByRefArgs<'_>,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ fn execute_with_timeout_zero_normalizes_to_single_attempt() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn caller_cannot_downgrade_press_key_below_focus_fallback() {
|
||||
fn explicit_press_key_keeps_its_focus_fallback_policy() {
|
||||
let _guard = HomeGuard::new();
|
||||
let snapshot_id = snapshot_with_ref("textfield", &["PressKey"]);
|
||||
let adapter = PolicyCaptureAdapter::new();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use crate::{AppError, NotificationFilter, adapter::PlatformAdapter};
|
||||
use crate::{AppError, NotificationFilter, adapter::PlatformAdapter, context::CommandContext};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub struct ListNotificationsArgs {
|
||||
|
|
@ -10,13 +10,18 @@ pub struct ListNotificationsArgs {
|
|||
pub fn execute(
|
||||
args: ListNotificationsArgs,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
context: &CommandContext,
|
||||
) -> Result<Value, AppError> {
|
||||
let filter = NotificationFilter {
|
||||
app: args.app,
|
||||
text: args.text,
|
||||
limit: args.limit,
|
||||
};
|
||||
let notifications = adapter.list_notifications(&filter, crate::Deadline::standard()?)?;
|
||||
let notifications = adapter.list_notifications(
|
||||
&filter,
|
||||
context.physical_input_policy(),
|
||||
crate::Deadline::standard()?,
|
||||
)?;
|
||||
Ok(json!({
|
||||
"count": notifications.len(),
|
||||
"notifications": notifications,
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ fn default_ref_commands_use_least_permissive_supported_policy() {
|
|||
)
|
||||
.unwrap();
|
||||
let type_request = adapter.requests.lock().unwrap()[before_type].clone();
|
||||
assert_eq!(type_request.policy, InteractionPolicy::focus_fallback());
|
||||
assert_eq!(type_request.policy, InteractionPolicy::headless());
|
||||
scroll::execute(
|
||||
scroll::ScrollArgs {
|
||||
ref_id: "@e1".into(),
|
||||
|
|
@ -251,9 +251,6 @@ fn default_ref_commands_use_least_permissive_supported_policy() {
|
|||
.unwrap();
|
||||
|
||||
for request in adapter.requests.lock().unwrap().iter() {
|
||||
if matches!(request.action, Action::TypeText(_)) {
|
||||
continue;
|
||||
}
|
||||
assert_headless(request);
|
||||
}
|
||||
}
|
||||
|
|
@ -272,7 +269,7 @@ fn focus_command_is_explicit_headless_policy() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn headed_context_preserves_physical_fallback_only_when_semantic_delivery_is_unavailable() {
|
||||
fn headed_context_reaches_every_ref_action_without_policy_downgrade() {
|
||||
let _guard = HomeGuard::new();
|
||||
let snapshot_id = snapshot_id();
|
||||
let adapter = RecordingAdapter::new();
|
||||
|
|
@ -337,13 +334,9 @@ fn headed_context_preserves_physical_fallback_only_when_semantic_delivery_is_una
|
|||
.unwrap();
|
||||
|
||||
for request in adapter.requests.lock().unwrap().iter() {
|
||||
let expected = if matches!(request.action, Action::Click | Action::RightClick) {
|
||||
InteractionPolicy::headless()
|
||||
} else {
|
||||
InteractionPolicy::headed()
|
||||
};
|
||||
assert_eq!(
|
||||
request.policy, expected,
|
||||
request.policy,
|
||||
InteractionPolicy::headed(),
|
||||
"unexpected policy for {:?}",
|
||||
request.action
|
||||
);
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ pub fn execute(
|
|||
}
|
||||
WaitMode::Menu { app, open } => wait_for_menu(app, open, timeout_ms, adapter),
|
||||
WaitMode::Notification { app, text } => {
|
||||
wait_for_notification(app, text, timeout_ms, adapter)
|
||||
wait_for_notification(app, text, timeout_ms, adapter, context)
|
||||
}
|
||||
WaitMode::Element {
|
||||
ref_id,
|
||||
|
|
@ -266,6 +266,7 @@ fn wait_for_notification(
|
|||
text: Option<String>,
|
||||
timeout_ms: u64,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
context: &CommandContext,
|
||||
) -> Result<Value, AppError> {
|
||||
let filter = NotificationFilter {
|
||||
app: app.clone(),
|
||||
|
|
@ -281,7 +282,7 @@ fn wait_for_notification(
|
|||
if deadline.is_expired() {
|
||||
return wait_timeout::notification(app.as_ref(), text.as_ref(), timeout_ms, last_error);
|
||||
}
|
||||
match adapter.list_notifications(&filter, deadline) {
|
||||
match adapter.list_notifications(&filter, context.physical_input_policy(), deadline) {
|
||||
Ok(current) => match &baseline {
|
||||
None => {
|
||||
baseline = Some(notification_counts(¤t));
|
||||
|
|
|
|||
|
|
@ -76,9 +76,7 @@ impl ElementPredicate {
|
|||
fn parse_actionability_action(action: Option<&str>) -> Result<ActionRequest, AppError> {
|
||||
match action.unwrap_or("click") {
|
||||
"click" => Ok(ActionRequest::headless(Action::Click)),
|
||||
"type" => Ok(ActionRequest::focus_fallback(Action::TypeText(
|
||||
String::new(),
|
||||
))),
|
||||
"type" => Ok(ActionRequest::headless(Action::TypeText(String::new()))),
|
||||
"set-value" => Ok(ActionRequest::headless(Action::SetValue(String::new()))),
|
||||
"clear" => Ok(ActionRequest::headless(Action::Clear)),
|
||||
other => Err(AppError::invalid_input_with_suggestion(
|
||||
|
|
|
|||
|
|
@ -315,7 +315,7 @@ fn element_wait_actionable_type_fails_on_uneditable_role() {
|
|||
"@e1".into(),
|
||||
Some(snapshot_id),
|
||||
wait_predicate::ElementPredicate::Actionable(
|
||||
crate::action_request::ActionRequest::focus_fallback(crate::action::Action::TypeText(
|
||||
crate::action_request::ActionRequest::headless(crate::action::Action::TypeText(
|
||||
String::new(),
|
||||
)),
|
||||
),
|
||||
|
|
@ -380,7 +380,7 @@ fn actionable_parse_mirrors_each_real_command_policy() {
|
|||
);
|
||||
assert_eq!(
|
||||
request_for(Some("type")).policy,
|
||||
InteractionPolicy::focus_fallback()
|
||||
InteractionPolicy::headless()
|
||||
);
|
||||
assert_eq!(
|
||||
request_for(Some("set-value")).policy,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ impl SystemOps for NotificationErrorAdapter {
|
|||
fn list_notifications(
|
||||
&self,
|
||||
_filter: &NotificationFilter,
|
||||
_policy: crate::InteractionPolicy,
|
||||
_deadline: crate::Deadline,
|
||||
) -> Result<Vec<NotificationInfo>, AdapterError> {
|
||||
Err(AdapterError::new(
|
||||
|
|
@ -61,6 +62,7 @@ impl SystemOps for FlakyNotificationAdapter {
|
|||
fn list_notifications(
|
||||
&self,
|
||||
_filter: &NotificationFilter,
|
||||
_policy: crate::InteractionPolicy,
|
||||
_deadline: crate::Deadline,
|
||||
) -> Result<Vec<NotificationInfo>, AdapterError> {
|
||||
self.responses
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ fn run_receives_events_error(outcome: Result<HitTestResult, AdapterError>) -> Ad
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_click_skips_inconclusive_screen_hit_testing() {
|
||||
fn headless_semantic_click_skips_inconclusive_screen_hit_testing() {
|
||||
let adapter = HitTestAdapter {
|
||||
outcome: Ok(HitTestResult::Unknown),
|
||||
actions: vec![capability::CLICK.into()],
|
||||
|
|
@ -197,7 +197,7 @@ fn semantic_click_skips_inconclusive_screen_hit_testing() {
|
|||
&clickable_entry(),
|
||||
&NativeHandle::null(),
|
||||
&adapter,
|
||||
&ActionRequest::headed(Action::Click),
|
||||
&ActionRequest::headless(Action::Click),
|
||||
)
|
||||
.expect("a direct semantic click must not depend on screen hit testing");
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ impl InteractionPolicy {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn is_headed(self) -> bool {
|
||||
self.allow_focus_steal && self.allow_cursor_move
|
||||
}
|
||||
|
||||
pub fn join(self, other: InteractionPolicy) -> InteractionPolicy {
|
||||
InteractionPolicy {
|
||||
allow_focus_steal: self.allow_focus_steal || other.allow_focus_steal,
|
||||
|
|
|
|||
|
|
@ -232,14 +232,14 @@ fn successful_action_drops_resolved_payload() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_preflight_revokes_unverified_physical_fallback() {
|
||||
fn headed_preflight_preserves_requested_physical_delivery() {
|
||||
let adapter = SuccessfulAdapter::new();
|
||||
|
||||
execute_entry(&adapter, &entry(), ActionRequest::headed(Action::Click)).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
adapter.dispatched_policies.lock().unwrap().as_slice(),
|
||||
&[crate::InteractionPolicy::headless()]
|
||||
&[crate::InteractionPolicy::headed()]
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1331,12 +1331,10 @@ void ad_app_list_free(struct AdAppList *list);
|
|||
* strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live
|
||||
* actionability preflight → dispatch → owned-handle drop.
|
||||
*
|
||||
* Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type`
|
||||
* command); `PressKey` shares that `focus_fallback` base (a ref-targeted key
|
||||
* press may need the target focused); every other action defaults to
|
||||
* `headless`. An explicit `policy` discriminant may *elevate* to headed but
|
||||
* must not downgrade an action below its base. Base and elevation are computed
|
||||
* by `agent_desktop_core::commands::execute_by_ref::execute` via
|
||||
* Policy: semantic actions, including `TypeText`, default to strict
|
||||
* `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy
|
||||
* discriminant may elevate to focus fallback or headed. Base and elevation
|
||||
* are computed by `agent_desktop_core::commands::execute_by_ref::execute` via
|
||||
* `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and
|
||||
* FFI share a single source of policy truth.
|
||||
*
|
||||
|
|
@ -1349,8 +1347,9 @@ void ad_app_list_free(struct AdAppList *list);
|
|||
*
|
||||
* `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback,
|
||||
* 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)`
|
||||
* accepts the action's own CLI base (so `TypeText` still uses
|
||||
* `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks.
|
||||
* accepts the action's base policy. `FocusFallback (1)` explicitly permits
|
||||
* focus without cursor movement. `Headed (2)` opts in to physical cursor and
|
||||
* keyboard delivery.
|
||||
*
|
||||
* Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before
|
||||
* the actionability preflight, matching the CLI default. Call
|
||||
|
|
|
|||
|
|
@ -9,12 +9,10 @@ use std::ffi::c_char;
|
|||
/// strict element resolution (→ `STALE_REF`/`AMBIGUOUS_TARGET`) → live
|
||||
/// actionability preflight → dispatch → owned-handle drop.
|
||||
///
|
||||
/// Policy: `TypeText` defaults to `focus_fallback` (matching the CLI `type`
|
||||
/// command); `PressKey` shares that `focus_fallback` base (a ref-targeted key
|
||||
/// press may need the target focused); every other action defaults to
|
||||
/// `headless`. An explicit `policy` discriminant may *elevate* to headed but
|
||||
/// must not downgrade an action below its base. Base and elevation are computed
|
||||
/// by `agent_desktop_core::commands::execute_by_ref::execute` via
|
||||
/// Policy: semantic actions, including `TypeText`, default to strict
|
||||
/// `headless`. Explicit `PressKey` defaults to `focus_fallback`. A policy
|
||||
/// discriminant may elevate to focus fallback or headed. Base and elevation
|
||||
/// are computed by `agent_desktop_core::commands::execute_by_ref::execute` via
|
||||
/// `Action::base_interaction_policy` + `InteractionPolicy::join`, so CLI and
|
||||
/// FFI share a single source of policy truth.
|
||||
///
|
||||
|
|
@ -27,8 +25,9 @@ use std::ffi::c_char;
|
|||
///
|
||||
/// `policy` is an `AdPolicyKind` discriminant (0=Headless, 1=FocusFallback,
|
||||
/// 2=Headed). An out-of-range value returns `ErrInvalidArgs`. `Headless (0)`
|
||||
/// accepts the action's own CLI base (so `TypeText` still uses
|
||||
/// `focus_fallback`). `Headed (2)` opts in to cursor-based fallbacks.
|
||||
/// accepts the action's base policy. `FocusFallback (1)` explicitly permits
|
||||
/// focus without cursor movement. `Headed (2)` opts in to physical cursor and
|
||||
/// keyboard delivery.
|
||||
///
|
||||
/// Uses a fixed 5000ms auto-wait budget (`DEFAULT_ACTION_TIMEOUT_MS`) before
|
||||
/// the actionability preflight, matching the CLI default. Call
|
||||
|
|
|
|||
|
|
@ -35,7 +35,11 @@ pub unsafe extern "C" fn ad_list_notifications(
|
|||
};
|
||||
let adapter = crate::adapter::acquire_adapter!(adapter);
|
||||
let deadline = crate::operation::operation_deadline!();
|
||||
match adapter.inner.list_notifications(&core_filter, deadline) {
|
||||
match adapter.inner.list_notifications(
|
||||
&core_filter,
|
||||
agent_desktop_core::InteractionPolicy::headless(),
|
||||
deadline,
|
||||
) {
|
||||
Ok(notifications) => {
|
||||
if let Err(error) =
|
||||
crate::resource::validate_list_len(notifications.len(), "Notification list")
|
||||
|
|
|
|||
|
|
@ -32,12 +32,6 @@ mod imp {
|
|||
|
||||
for (i, step) in def.steps.iter().enumerate() {
|
||||
ctx.ensure_budget()?;
|
||||
if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) {
|
||||
return Err(AdapterError::policy_denied_for_policy(
|
||||
"Physical click fallback is disabled by the current interaction policy",
|
||||
policy,
|
||||
));
|
||||
}
|
||||
let label = step_label(step);
|
||||
let outcome = execute_step(el, step, ctx, policy)?;
|
||||
if record_step_outcome(
|
||||
|
|
@ -109,10 +103,6 @@ mod imp {
|
|||
ChainStep::CGDisclosureClick { .. } => "CGDisclosureClick",
|
||||
}
|
||||
}
|
||||
|
||||
fn physical_click_permitted(policy: InteractionPolicy) -> bool {
|
||||
policy.allow_focus_steal && policy.allow_cursor_move
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ mod imp {
|
|||
|
||||
pub(crate) static CLICK_CHAIN: ChainDef = ChainDef {
|
||||
steps: &[
|
||||
ChainStep::CGClick {
|
||||
button: MouseButton::Left,
|
||||
count: 1,
|
||||
},
|
||||
ChainStep::Action("AXPress"),
|
||||
ChainStep::CGDisclosureClick { expanded: true },
|
||||
],
|
||||
suggestion: "Target an element that advertises Click or use an explicit point click.",
|
||||
continue_after_unverified_delivery: false,
|
||||
|
|
@ -20,6 +23,10 @@ mod imp {
|
|||
|
||||
pub(crate) static RIGHT_CLICK_CHAIN: ChainDef = ChainDef {
|
||||
steps: &[
|
||||
ChainStep::CGClick {
|
||||
button: MouseButton::Right,
|
||||
count: 1,
|
||||
},
|
||||
ChainStep::CustomWithDeadline {
|
||||
label: "show_menu",
|
||||
func: chain_menu_steps::show_menu,
|
||||
|
|
@ -40,10 +47,6 @@ mod imp {
|
|||
label: "ancestor_show_menu",
|
||||
func: chain_menu_steps::show_menu_on_ancestors,
|
||||
},
|
||||
ChainStep::CGClick {
|
||||
button: MouseButton::Right,
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
suggestion: "Try 'mouse-click --button right --xy X,Y'.",
|
||||
continue_after_unverified_delivery: false,
|
||||
|
|
@ -84,13 +87,19 @@ mod imp {
|
|||
|
||||
pub(crate) static CLEAR_CHAIN: ChainDef = ChainDef {
|
||||
steps: &[
|
||||
ChainStep::SetDynamic { attr: "AXValue" },
|
||||
ChainStep::FocusThenClearByKeyboard,
|
||||
ChainStep::SetDynamic { attr: "AXValue" },
|
||||
],
|
||||
suggestion: "Target an editable control or allow the verified keyboard fallback.",
|
||||
continue_after_unverified_delivery: true,
|
||||
};
|
||||
|
||||
pub(crate) static SEMANTIC_CLICK_CHAIN: ChainDef = ChainDef {
|
||||
steps: &[ChainStep::Action("AXPress")],
|
||||
suggestion: "Target an element that advertises Click.",
|
||||
continue_after_unverified_delivery: false,
|
||||
};
|
||||
|
||||
pub(crate) static FOCUS_CHAIN: ChainDef = ChainDef {
|
||||
steps: &[ChainStep::SetBool {
|
||||
attr: "AXFocused",
|
||||
|
|
@ -171,5 +180,5 @@ mod imp {}
|
|||
#[cfg(target_os = "macos")]
|
||||
pub(crate) use imp::{
|
||||
CLEAR_CHAIN, CLICK_CHAIN, COLLAPSE_CHAIN, EXPAND_CHAIN, FOCUS_CHAIN, RIGHT_CLICK_CHAIN,
|
||||
SCROLL_TO_CHAIN, SET_VALUE_CHAIN, double_click, triple_click,
|
||||
SCROLL_TO_CHAIN, SEMANTIC_CLICK_CHAIN, SET_VALUE_CHAIN, double_click, triple_click,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ mod imp {
|
|||
},
|
||||
|
||||
ChainStep::FocusThenClearByKeyboard => {
|
||||
if !policy.allow_focus_steal {
|
||||
if !policy.is_headed() {
|
||||
return Ok(DeliveryOutcome::NotDelivered);
|
||||
}
|
||||
crate::actions::physical_keyboard::press_sequence(
|
||||
|
|
@ -80,6 +80,9 @@ mod imp {
|
|||
ChainStep::CustomWithDeadline { label: _, func } => func(el, ctx.deadline),
|
||||
|
||||
ChainStep::CGClick { button, count } => {
|
||||
if !policy.is_headed() {
|
||||
return Ok(DeliveryOutcome::NotDelivered);
|
||||
}
|
||||
physical_click(el, button.clone(), *count, ctx, policy)?;
|
||||
Ok(DeliveryOutcome::DeliveredUnverified)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use agent_desktop_core::MouseButton;
|
|||
use agent_desktop_core::step_mechanism::StepMechanism;
|
||||
|
||||
#[test]
|
||||
fn right_click_restores_semantic_menu_fallbacks_before_physical_input() {
|
||||
fn right_click_prefers_physical_input_before_semantic_fallbacks() {
|
||||
let labels: Vec<&str> = crate::actions::chain_defs::RIGHT_CLICK_CHAIN
|
||||
.steps
|
||||
.iter()
|
||||
|
|
@ -19,16 +19,31 @@ fn right_click_restores_semantic_menu_fallbacks_before_physical_input() {
|
|||
assert_eq!(
|
||||
labels,
|
||||
[
|
||||
"CGClick",
|
||||
"show_menu",
|
||||
"select_then_show_menu",
|
||||
"selected_items_menu",
|
||||
"child_show_menu",
|
||||
"ancestor_show_menu",
|
||||
"CGClick",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn click_and_clear_prefer_physical_delivery_when_policy_allows_it() {
|
||||
assert!(matches!(
|
||||
crate::actions::chain_defs::CLICK_CHAIN.steps.first(),
|
||||
Some(ChainStep::CGClick {
|
||||
button: MouseButton::Left,
|
||||
count: 1
|
||||
})
|
||||
));
|
||||
assert!(matches!(
|
||||
crate::actions::chain_defs::CLEAR_CHAIN.steps.first(),
|
||||
Some(ChainStep::FocusThenClearByKeyboard)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_mechanism_tags_physical_for_cgclick_and_keyboard_clear() {
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ pub(crate) mod chain_value_write;
|
|||
pub(crate) mod chain_verify;
|
||||
pub(crate) mod dispatch;
|
||||
pub(crate) mod extras;
|
||||
mod mutation_delivery;
|
||||
mod physical_click;
|
||||
mod physical_keyboard;
|
||||
pub(crate) mod post_state;
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
use agent_desktop_core::{AdapterError, ErrorCode};
|
||||
|
||||
pub(crate) fn fallback_is_safe(error: &AdapterError) -> bool {
|
||||
error.code == ErrorCode::ActionFailed
|
||||
&& error.disposition == agent_desktop_core::DeliverySemantics::not_delivered()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use agent_desktop_core::{AdapterError, ErrorCode};
|
||||
|
||||
use super::fallback_is_safe;
|
||||
|
||||
#[test]
|
||||
fn rejects_uncertain_or_non_action_failures() {
|
||||
let uncertain = AdapterError::new(ErrorCode::ActionFailed, "uncertain")
|
||||
.with_disposition(agent_desktop_core::DeliverySemantics::uncertain());
|
||||
assert!(!fallback_is_safe(&uncertain));
|
||||
assert!(!fallback_is_safe(&AdapterError::permission_denied()));
|
||||
assert!(!fallback_is_safe(&AdapterError::new(
|
||||
ErrorCode::AppUnresponsive,
|
||||
"unresponsive",
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_definite_non_delivery() {
|
||||
let not_delivered = AdapterError::new(ErrorCode::ActionFailed, "not delivered")
|
||||
.with_disposition(agent_desktop_core::DeliverySemantics::not_delivered());
|
||||
assert!(fallback_is_safe(¬_delivered));
|
||||
}
|
||||
}
|
||||
|
|
@ -46,19 +46,6 @@ pub(crate) fn type_text(
|
|||
})
|
||||
}
|
||||
|
||||
pub(crate) fn repeat_keycode(
|
||||
element: &AXElement,
|
||||
key_code: u16,
|
||||
repeats: u32,
|
||||
policy: InteractionPolicy,
|
||||
deadline: Deadline,
|
||||
) -> Result<(), AdapterError> {
|
||||
let identity = prepare_target(element, policy, deadline)?;
|
||||
let pid = identity.pid();
|
||||
verify_delivery_target(element, identity, deadline)?;
|
||||
crate::input::keyboard::synthesize_keycode(key_code, repeats, Some(pid), deadline)
|
||||
}
|
||||
|
||||
fn prepare_target(
|
||||
element: &AXElement,
|
||||
policy: InteractionPolicy,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ pub(crate) fn ax_scroll(
|
|||
validate_amount(amount)?;
|
||||
let scroll_area = find_scroll_area(element, deadline)?;
|
||||
let target = scroll_area.as_ref().unwrap_or(element);
|
||||
if policy.is_headed() {
|
||||
physical_wheel(target, direction, amount, deadline)?;
|
||||
return Ok((StepMechanism::PhysicalSynthetic, false));
|
||||
}
|
||||
accept_optional_visibility_result(try_action(element, "AXScrollToVisible", deadline))?;
|
||||
let (bar_attribute, increment_action) = scroll_bar_action(direction);
|
||||
|
||||
|
|
@ -33,26 +37,6 @@ pub(crate) fn ax_scroll(
|
|||
if perform_repeated_action(target, page_action(direction), amount, deadline)? {
|
||||
return Ok((StepMechanism::SemanticApi, false));
|
||||
}
|
||||
if policy.allow_focus_steal {
|
||||
let keycode = direction_keycode(direction);
|
||||
match crate::actions::physical_keyboard::repeat_keycode(
|
||||
target, keycode, amount, policy, deadline,
|
||||
) {
|
||||
Ok(()) => return Ok((StepMechanism::PhysicalSynthetic, false)),
|
||||
Err(error) if crate::actions::mutation_delivery::fallback_is_safe(&error) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
if policy.allow_focus_steal && policy.allow_cursor_move {
|
||||
physical_wheel(target, direction, amount, deadline)?;
|
||||
return Ok((StepMechanism::PhysicalSynthetic, false));
|
||||
}
|
||||
if policy.allow_focus_steal {
|
||||
return Err(AdapterError::policy_denied_for_policy(
|
||||
"Cursor-moving scroll fallback is disabled by the current interaction policy",
|
||||
policy,
|
||||
));
|
||||
}
|
||||
Err(AdapterError::new(
|
||||
ErrorCode::ActionNotSupported,
|
||||
"No scroll mechanism found on element",
|
||||
|
|
@ -107,15 +91,6 @@ fn page_action(direction: &Direction) -> &'static str {
|
|||
}
|
||||
}
|
||||
|
||||
fn direction_keycode(direction: &Direction) -> u16 {
|
||||
match direction {
|
||||
Direction::Down => 121,
|
||||
Direction::Up => 116,
|
||||
Direction::Right => 124,
|
||||
Direction::Left => 123,
|
||||
}
|
||||
}
|
||||
|
||||
fn perform_repeated_action(
|
||||
element: &AXElement,
|
||||
action: &'static str,
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ pub(crate) fn toggle(
|
|||
verified_point: None,
|
||||
deadline,
|
||||
};
|
||||
let mut steps = execute_chain(el, &chain_defs::CLICK_CHAIN, &ctx, policy)?;
|
||||
let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?;
|
||||
let verified = if let Some(before) = before {
|
||||
wait_for_value_change(el, &before, deadline).map_err(after_delivery)?;
|
||||
true
|
||||
|
|
@ -93,7 +93,7 @@ pub(crate) fn check_uncheck(
|
|||
verified_point: None,
|
||||
deadline,
|
||||
};
|
||||
let mut steps = execute_chain(el, &chain_defs::CLICK_CHAIN, &ctx, policy)?;
|
||||
let mut steps = execute_chain(el, &chain_defs::SEMANTIC_CLICK_CHAIN, &ctx, policy)?;
|
||||
wait_for_checked_state(el, want_checked, deadline).map_err(after_delivery)?;
|
||||
mark_last_verified(&mut steps, true);
|
||||
Ok(steps)
|
||||
|
|
|
|||
|
|
@ -17,20 +17,15 @@ pub(crate) fn execute_type(
|
|||
"Type requires a text field, secure text field, or combo box",
|
||||
));
|
||||
}
|
||||
match insert_selected_text(element, text, deadline) {
|
||||
Ok(()) => {
|
||||
return Ok(ActionStep::succeeded("AXSelectedText")
|
||||
.with_mechanism(StepMechanism::SemanticApi)
|
||||
.with_verified(false));
|
||||
}
|
||||
Err(error)
|
||||
if policy.allow_focus_steal
|
||||
&& crate::actions::mutation_delivery::fallback_is_safe(&error) => {}
|
||||
Err(error) => return Err(error),
|
||||
if policy.is_headed() {
|
||||
crate::actions::physical_keyboard::type_text(element, text, policy, deadline)?;
|
||||
return Ok(ActionStep::succeeded("PIDTargetedUnicodeText")
|
||||
.with_mechanism(StepMechanism::PhysicalSynthetic)
|
||||
.with_verified(false));
|
||||
}
|
||||
crate::actions::physical_keyboard::type_text(element, text, policy, deadline)?;
|
||||
Ok(ActionStep::succeeded("PIDTargetedUnicodeText")
|
||||
.with_mechanism(StepMechanism::PhysicalSynthetic)
|
||||
insert_selected_text(element, text, deadline)?;
|
||||
Ok(ActionStep::succeeded("AXSelectedText")
|
||||
.with_mechanism(StepMechanism::SemanticApi)
|
||||
.with_verified(false))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -190,9 +190,10 @@ impl SystemOps for MacOSAdapter {
|
|||
fn list_notifications(
|
||||
&self,
|
||||
filter: &NotificationFilter,
|
||||
policy: agent_desktop_core::InteractionPolicy,
|
||||
deadline: Deadline,
|
||||
) -> Result<Vec<NotificationInfo>, AdapterError> {
|
||||
crate::notifications::list::list_notifications(filter, deadline)
|
||||
crate::notifications::list::list_notifications(filter, policy, deadline)
|
||||
}
|
||||
|
||||
fn dismiss_notification(
|
||||
|
|
|
|||
|
|
@ -60,33 +60,6 @@ pub(crate) fn preflight_text(text: &str, deadline: Deadline) -> Result<(), Adapt
|
|||
crate::input::keyboard_event::preflight_text(text, deadline)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) fn synthesize_keycode(
|
||||
key_code: u16,
|
||||
repeats: u32,
|
||||
target_pid: Option<i32>,
|
||||
deadline: Deadline,
|
||||
) -> Result<(), AdapterError> {
|
||||
const MAX_REPEATS: u32 = 1_000;
|
||||
|
||||
if repeats == 0 || repeats > MAX_REPEATS {
|
||||
return Err(AdapterError::new(
|
||||
ErrorCode::InvalidArgs,
|
||||
"Key repeat count must be between 1 and 1000",
|
||||
));
|
||||
}
|
||||
for delivered in 0..repeats {
|
||||
crate::input::keyboard_event::post_key(
|
||||
key_code,
|
||||
core_graphics::event::CGEventFlags::empty(),
|
||||
target_pid,
|
||||
deadline,
|
||||
(delivered as usize, repeats as usize),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub(crate) fn synthesize_key(
|
||||
_combo: &KeyCombo,
|
||||
|
|
@ -119,16 +92,6 @@ pub(crate) fn preflight_text(_text: &str, _deadline: Deadline) -> Result<(), Ada
|
|||
Err(AdapterError::not_supported("synthesize_text"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub(crate) fn synthesize_keycode(
|
||||
_key_code: u16,
|
||||
_repeats: u32,
|
||||
_target_pid: Option<i32>,
|
||||
_deadline: Deadline,
|
||||
) -> Result<(), AdapterError> {
|
||||
Err(AdapterError::not_supported("synthesize_keycode"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ pub fn dismiss_notification(
|
|||
deadline: Deadline,
|
||||
) -> Result<NotificationInfo, AdapterError> {
|
||||
require_foreground_policy(policy)?;
|
||||
let session = NcSession::open(deadline)?;
|
||||
let session = NcSession::open(policy, deadline)?;
|
||||
let result = dismiss_impl(index, app_filter, identity, policy, session.pid(), deadline);
|
||||
close_session(session, result)
|
||||
}
|
||||
|
|
@ -28,7 +28,7 @@ pub fn dismiss_all(
|
|||
deadline: Deadline,
|
||||
) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> {
|
||||
require_foreground_policy(policy)?;
|
||||
let session = NcSession::open(deadline)?;
|
||||
let session = NcSession::open(policy, deadline)?;
|
||||
let result = dismiss_all_impl(app_filter, policy, session.pid(), deadline);
|
||||
close_session(session, result)
|
||||
}
|
||||
|
|
@ -41,7 +41,7 @@ pub fn notification_action(
|
|||
deadline: Deadline,
|
||||
) -> Result<ActionResult, AdapterError> {
|
||||
require_foreground_policy(policy)?;
|
||||
let session = NcSession::open(deadline)?;
|
||||
let session = NcSession::open(policy, deadline)?;
|
||||
let result = action_impl(index, identity, action_name, session.pid(), deadline);
|
||||
close_session(session, result)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
use agent_desktop_core::{AdapterError, Deadline, NotificationFilter, NotificationInfo};
|
||||
use agent_desktop_core::{
|
||||
AdapterError, Deadline, InteractionPolicy, NotificationFilter, NotificationInfo,
|
||||
};
|
||||
|
||||
use super::nc_session::{NcSession, close_session};
|
||||
|
||||
pub fn list_notifications(
|
||||
filter: &NotificationFilter,
|
||||
policy: InteractionPolicy,
|
||||
deadline: Deadline,
|
||||
) -> Result<Vec<NotificationInfo>, AdapterError> {
|
||||
let session = NcSession::open(deadline)?;
|
||||
let session = NcSession::open(policy, deadline)?;
|
||||
let result = list_from_nc(filter, session.pid(), deadline);
|
||||
close_session(session, result)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use agent_desktop_core::{AdapterError, Deadline, KeyCombo};
|
||||
use agent_desktop_core::{AdapterError, Deadline, InteractionPolicy, KeyCombo};
|
||||
|
||||
pub(crate) fn close_session<T>(
|
||||
session: NcSession,
|
||||
|
|
@ -24,18 +24,30 @@ pub(crate) struct NcSession {
|
|||
}
|
||||
|
||||
impl NcSession {
|
||||
pub(crate) fn open(deadline: Deadline) -> Result<Self, AdapterError> {
|
||||
pub(crate) fn open(
|
||||
policy: InteractionPolicy,
|
||||
deadline: Deadline,
|
||||
) -> Result<Self, AdapterError> {
|
||||
if let Some(pid) = nc_pid(deadline)?
|
||||
&& is_nc_open(pid, deadline)
|
||||
{
|
||||
return Ok(Self {
|
||||
pid,
|
||||
was_already_open: true,
|
||||
previous_app: None,
|
||||
closed: false,
|
||||
deadline,
|
||||
});
|
||||
}
|
||||
if !policy.is_headed() {
|
||||
return Err(closed_center_policy_error(policy));
|
||||
}
|
||||
let previous_app = frontmost_app(deadline);
|
||||
let (was_already_open, pid) = match nc_pid(deadline)? {
|
||||
Some(pid) if is_nc_open(pid, deadline) => (true, pid),
|
||||
_ => {
|
||||
open_nc(deadline)?;
|
||||
(false, wait_for_nc_ready(deadline)?)
|
||||
}
|
||||
};
|
||||
open_nc(deadline)?;
|
||||
let pid = wait_for_nc_ready(deadline)?;
|
||||
Ok(Self {
|
||||
pid,
|
||||
was_already_open,
|
||||
was_already_open: false,
|
||||
previous_app,
|
||||
closed: false,
|
||||
deadline,
|
||||
|
|
@ -60,6 +72,16 @@ impl NcSession {
|
|||
}
|
||||
}
|
||||
|
||||
fn closed_center_policy_error(policy: InteractionPolicy) -> AdapterError {
|
||||
AdapterError::policy_denied_for_policy(
|
||||
"Notification Center is closed and observation cannot open it in headless mode",
|
||||
policy,
|
||||
)
|
||||
.with_suggestion(
|
||||
"Open Notification Center yourself or pass --headed to allow opening and restoring desktop focus.",
|
||||
)
|
||||
}
|
||||
|
||||
impl Drop for NcSession {
|
||||
fn drop(&mut self) {
|
||||
if self.closed {
|
||||
|
|
@ -221,7 +243,7 @@ fn close_nc(deadline: Deadline) -> Result<(), AdapterError> {
|
|||
|
||||
#[cfg(all(test, target_os = "macos"))]
|
||||
mod tests {
|
||||
use super::{applescript_string, nc_pid_from_output};
|
||||
use super::{applescript_string, closed_center_policy_error, nc_pid_from_output};
|
||||
use agent_desktop_core::AdapterError;
|
||||
|
||||
#[test]
|
||||
|
|
@ -247,6 +269,20 @@ mod tests {
|
|||
|
||||
assert_eq!(error.code, agent_desktop_core::ErrorCode::Timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closed_notification_center_is_policy_denied_headlessly() {
|
||||
let error = closed_center_policy_error(agent_desktop_core::InteractionPolicy::headless());
|
||||
|
||||
assert_eq!(error.code, agent_desktop_core::ErrorCode::PolicyDenied);
|
||||
assert!(error.message.contains("headless"));
|
||||
assert!(
|
||||
error
|
||||
.suggestion
|
||||
.as_deref()
|
||||
.is_some_and(|value| value.contains("--headed"))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ crates/macos/src/
|
|||
│ ├── extras.rs # select_value helpers
|
||||
│ ├── post_state.rs # Post-action state reads
|
||||
│ ├── scroll.rs # scroll semantics and explicit physical policy paths
|
||||
│ └── type_text.rs # focus-fallback text insertion and physical typing
|
||||
│ └── type_text.rs # strict semantic text insertion and headed physical typing
|
||||
├── input/
|
||||
│ ├── mod.rs # re-exports
|
||||
│ ├── keyboard.rs # CGEventCreateKeyboardEvent, key synthesis, text typing
|
||||
|
|
@ -285,7 +285,7 @@ crates/macos/src/
|
|||
**Action execution:**
|
||||
- Ref actions take `ActionRequest`, not bare `Action`
|
||||
- Default policy forbids focus stealing and cursor movement
|
||||
- Click/right-click/scroll chains run semantic AX steps first and return structured errors instead of silently using physical/headed paths
|
||||
- Click/right-click/scroll use semantic AX delivery headlessly and prefer physical delivery only under explicit `--headed`
|
||||
- Type uses the focus-fallback policy floor; SetValue/Clear are the pure headless AX value-mutation paths
|
||||
- SetValue/Clear: `AXUIElementSetAttributeValue(kAXValueAttribute, value)`
|
||||
- SetFocus/Press/Hover/Drag/Mouse: explicit focus/cursor/physical commands
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ high-level ref path can reproduce CLI observation-to-action behavior.
|
|||
`ad_execute_by_ref` and its timeout variant use the core command path. They
|
||||
load the ref map, resolve strictly, apply actionability, and compute policy
|
||||
through `Action::base_interaction_policy` joined with the caller's explicit
|
||||
policy. Headless callers therefore retain an action's required focus fallback;
|
||||
headed is an opt-in elevation.
|
||||
policy. Semantic actions, including `type`, stay strictly headless; explicit
|
||||
`press` retains its focus fallback and headed is an opt-in elevation.
|
||||
|
||||
`ad_execute_action` and struct-based direct action entrypoints are deliberately
|
||||
lower-level escape hatches. They operate on a caller-held native handle or
|
||||
|
|
|
|||
|
|
@ -25,11 +25,12 @@ future adapters copy macOS assumptions.
|
|||
## Guidance
|
||||
|
||||
Core creates an `ActionRequest` with the action's least-permissive base policy.
|
||||
Most actions start headless. `type` and ref-targeted `press` may use focus
|
||||
fallback. Explicit headed mode can elevate that policy but may not weaken it.
|
||||
Semantic actions, including `type`, start strictly headless. Explicit `press`
|
||||
may use focus fallback. Headed mode can elevate policy but may not weaken it.
|
||||
|
||||
The adapter chooses the best legal implementation: semantic accessibility APIs
|
||||
first, then a policy-gated physical fallback when appropriate. `hover` and
|
||||
The adapter chooses the requested legal implementation: strict headless uses
|
||||
semantic accessibility APIs, while headed natural-input commands prefer
|
||||
physical delivery. `hover` and
|
||||
`drag` are physical pointer commands by definition and require a cursor-moving
|
||||
policy before resolving or moving the pointer. A semantic reorder capability,
|
||||
if a future platform offers one, is a distinct action contract rather than a
|
||||
|
|
|
|||
|
|
@ -42,9 +42,10 @@ is `AMBIGUOUS_TARGET`. Mutable field values are not stable identity.
|
|||
### 3. Separate actionability from dispatch
|
||||
|
||||
Semantic ref actions use the shared auto-wait and live actionability checks.
|
||||
The command owns the base interaction policy: most actions are headless;
|
||||
typing and ref-targeted key presses may use focus fallback; headed mode can
|
||||
only elevate policy. A failed preflight must say why and preserve retry safety.
|
||||
The command owns the base interaction policy: semantic actions, including
|
||||
typing, are strictly headless; explicit ref-targeted key presses may use focus
|
||||
fallback, and headed mode can only elevate policy. A failed preflight must say
|
||||
why and preserve retry safety.
|
||||
|
||||
Pointer commands are a separate physical family. They resolve a live point,
|
||||
verify visibility, geometry stability, and hit-test receipt, then require an
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ Use **progressive skeleton traversal** as the default approach. It reduces token
|
|||
- **Scoped invalidation:** re-drilling a qualified root ref only replaces refs from that root's previous drill — refs from other regions and the skeleton itself are preserved
|
||||
- **Strict resolution:** stale refs return `STALE_REF`; duplicate plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily.
|
||||
- **Actionability:** every ref-addressed action checks its applicable live visibility, stability, enabled, editability, policy, supported-action, and hit-test requirements under one bounded budget before a single dispatch. Pointer actions focus before their final geometry read, re-resolve moving endpoints, and return `TIMEOUT` with `details.kind: "actionability_timeout"` instead of sending input after the deadline.
|
||||
- **Headless vs headed:** ref actions are headless by default (AX-only, no cursor) and fail closed when only a physical gesture would work. `type` uses a focus-fallback base policy because typing needs focus but never moves the cursor. Pass the global `--headed` flag to permit cursor movement and focus stealing so physical fallbacks can complete; the AX path is still tried first, so `--headed` never regresses headless-capable elements. Raw cursor commands (`hover`, `drag`, `mouse-*`) are physical and require `--headed`; keyboard commands (`press`, `key-down`, `key-up`) are explicit low-level input.
|
||||
- **Headless vs headed:** ref actions are strictly headless by default: semantic accessibility APIs only, with no focus stealing, cursor movement, or synthesized keyboard input. Pass global `--headed` to prefer physical delivery for natural input commands (`click`, `right-click`, `type`, `clear`, `expand`, `collapse`, and `scroll`); semantic-only commands such as `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` remain semantic. Raw input commands (`press`, `hover`, `drag`, `mouse-*`) are explicit physical input; cursor commands require `--headed`.
|
||||
- **Sessions and tracing:** run `session start` once per agent run to create a manifest with `trace: on` (default), then pass its returned ID with `--session` or `AGENT_DESKTOP_SESSION`. Use `session start --screenshots` when you need replay artifacts (`artifacts: full`): pre/post-action PNGs and refmap copies under the session trace directory (sensitive — treat exports like screenshots). Commands in that explicit scope record JSONL automatically to per-process segments under `~/.agent-desktop/sessions/<id>/trace/<pid>-<procTs>.jsonl` — no `--trace` on every call. Read traces back with `trace show` (bounded JSON for agents) or `trace export` (single-file HTML for humans). A session owns both its trace and its latest-snapshot namespace. Snapshot lookup never searches another namespace. **`--session <id>` alone** (no manifest from `session start`) selects only the snapshot namespace — existing callers see no surprise trace files. **`--trace <path>`** still overrides to one atomic file for CI or one-offs. Activation precedence is `--session` > `AGENT_DESKTOP_SESSION` > no session; `session start` does not activate later processes. Multi-agent shared sessions: each agent acts on qualified refs from its own snapshot — implicit latest is not a cross-agent guarantee. Run `status` to see `session_id` and `tracing`. Trace lines include `ts_ms`, monotonic per-process `seq`, and redacted sensitive fields (`text`, `value`, `expected`, `name`, `username`, `description`, `label`, `query`, `secret`, `token`, `password`, `title`, `url`, `help`, `placeholder` → `{ "redacted": true }`). `--trace-strict` fails on trace setup and pre-action writes; post-action success traces are best-effort.
|
||||
|
||||
## JSON Output Contract
|
||||
|
|
@ -151,7 +151,7 @@ agent-desktop list-surfaces --app "App" # Available surfaces
|
|||
### Interaction
|
||||
```
|
||||
agent-desktop click @e5 --snapshot <snapshot_id> # AX-first click, no cursor move by default
|
||||
agent-desktop double-click @s8f3k2p9:e3 # AXOpen; physical double-click uses --headed mouse-click --count 2
|
||||
agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click
|
||||
agent-desktop triple-click @s8f3k2p9:e2 # Physical triple-click uses mouse-click --count 3
|
||||
agent-desktop right-click @s8f3k2p9:e5 # Right-click; inspect the resulting menu/effect separately
|
||||
agent-desktop type @e2 --snapshot <snapshot_id> "hello" # Headless AX text insertion when supported
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and
|
|||
|
||||
Ref-based actions run in two modes, Playwright-style:
|
||||
|
||||
- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the AX path cannot perform the action it fails closed rather than reaching for OS input synthesis. (`type` is the one exception: its base tier may focus the target field — required for reliable typing — but still never moves the cursor.)
|
||||
- **`--headed`.** A global flag (`agent-desktop --headed click @s8f3k2p9:e5`) that upgrades every ref action to permit focus stealing **and** cursor movement, unlocking the physical click/double-click/scroll/keypress fallbacks in the action chain. The AX path is still tried first, so `--headed` never regresses elements that work headlessly — it only adds fallbacks for elements that need a real gesture (e.g. a gesture-only button with no `AXOpen`).
|
||||
- **Headless (default).** Semantic accessibility operations only. The action never silently steals focus, moves the cursor, synthesizes keyboard input, or uses the pasteboard. When the semantic path cannot perform the action it fails closed.
|
||||
- **`--headed`.** A global flag (`agent-desktop --headed click @s8f3k2p9:e5`) that permits focus/cursor side effects and prefers physical delivery for natural input commands: `click`, `right-click`, `double-click`, `triple-click`, `type`, `clear`, `expand`, `collapse`, and `scroll`. Semantic-only commands (`set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, `scroll-to`) stay semantic.
|
||||
|
||||
Raw-input commands (`press`, `hover`, `drag`, `mouse-*`, `key-down`, `key-up`) are physical by nature. Cursor-moving commands (`hover`, `drag`, `mouse-*`) require `--headed`; keyboard commands are explicit low-level input.
|
||||
|
||||
|
|
@ -47,9 +47,9 @@ The command surface is platform-agnostic: every ref action builds an `Action` an
|
|||
|
||||
| Command | Headless path (macOS) | Notes |
|
||||
|---------|---------------|-------|
|
||||
| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions; the default and most reliable surface |
|
||||
| `type` | focus fallback | CLI `type` may focus the target field but never moves the cursor; use `set-value` for pure headless value mutation when supported |
|
||||
| `double-click` | partial | `AXOpen` works headless on items that advertise it (Finder/list/outline rows, table cells). Falls back to `--headed` only for gesture-only targets with no `AXOpen`. |
|
||||
| `click`, `set-value`, `check`, `select`, `scroll`, `expand`, … | yes | semantic AX actions in strict headless mode |
|
||||
| `type` | yes | uses `AXSelectedText` headlessly; `--headed` synthesizes keyboard input |
|
||||
| `double-click` | no | a real two-click gesture; requires `--headed` |
|
||||
| `triple-click` | no | macOS exposes no triple-click action; it is purely 3 physical clicks → `--headed` only |
|
||||
| `hover` | no | hovering *is* moving the cursor over an element; no AX equivalent |
|
||||
| `drag` / drop | no | dragging *is* a cursor press-move-release; no general AX drag. Native cross-app drop needs the OS dragging-session/pasteboard protocol that synthetic events cannot start (works for same-view source-tracked gestures and web/Electron mouse-DnD) |
|
||||
|
|
@ -69,20 +69,20 @@ Every ref-resolving action accepts `--timeout-ms` (default `5000`), but it budge
|
|||
|
||||
## Click Actions
|
||||
|
||||
Click commands use semantic AX activation first. In the default headless mode, coordinate click fallback is blocked; pass `--headed` to allow the physical click fallback, or use `agent-desktop --headed mouse-click` for a raw coordinate click.
|
||||
Click commands use semantic AX activation in strict headless mode. Pass `--headed` to prefer a physical click, or use `agent-desktop --headed mouse-click` for a raw coordinate click.
|
||||
|
||||
### click
|
||||
```bash
|
||||
agent-desktop click @s8f3k2p9:e5
|
||||
agent-desktop click @e5 --snapshot <snapshot_id>
|
||||
```
|
||||
Primary activation. Tries verified AXPress > AXConfirm > AXOpen > AXPick > child activation > selection/value relays > custom actions > ancestor activation. Focus-stealing and coordinate fallback steps are not used by the default ref command path.
|
||||
Primary activation. Headless uses `AXPress`; `--headed` performs a physical click first and reports `physical_synthetic` in `data.steps`.
|
||||
|
||||
### double-click
|
||||
```bash
|
||||
agent-desktop double-click @s8f3k2p9:e3
|
||||
```
|
||||
Tries AXOpen (headless). When the element advertises no `AXOpen`, the headless command fails closed with `POLICY_DENIED`; pass `--headed` to perform a real double-click (`agent-desktop --headed double-click @s8f3k2p9:e3`), or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for a raw coordinate double-click.
|
||||
Double-click is a physical gesture and fails closed in headless mode. Pass `--headed` to perform it, or use `agent-desktop --headed mouse-click --xy X,Y --count 2` for raw coordinates.
|
||||
|
||||
### triple-click
|
||||
```bash
|
||||
|
|
@ -94,7 +94,7 @@ Triple-click requires cursor/focus side effects and is blocked in headless mode;
|
|||
```bash
|
||||
agent-desktop right-click @s8f3k2p9:e5
|
||||
```
|
||||
Performs a semantic right-click/context-menu action. On macOS, `AXShowMenu` can return `APP_UNRESPONSIVE` with `delivery: delivery_uncertain` and `retry: unsafe` after opening a modal context menu; inspect the resulting menu or target effect before deciding what to do, and never retry that outcome blindly. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked in headless mode; pass `--headed` to allow them.
|
||||
Headless uses semantic context-menu actions. `--headed` performs a physical right-click first. On macOS, a semantic `AXShowMenu` can return `APP_UNRESPONSIVE` with uncertain delivery after opening a modal menu; inspect the effect and never retry blindly. Use `select` for combo boxes and menu buttons.
|
||||
|
||||
## Text Input
|
||||
|
||||
|
|
@ -103,9 +103,7 @@ Performs a semantic right-click/context-menu action. On macOS, `AXShowMenu` can
|
|||
agent-desktop type @s8f3k2p9:e2 "hello@example.com"
|
||||
agent-desktop type @s8f3k2p9:e2 "multi line\ntext"
|
||||
```
|
||||
`type` uses the focus-fallback policy floor: it may focus the target field because typing requires focus, but it never moves the cursor. If the field cannot be updated and the focused-insert path is unavailable, it returns a structured error. Pass `--headed` to unlock physical keyboard synthesis and pasteboard-based insertion for fields that ignore AX value writes (common in web/Electron inputs).
|
||||
|
||||
Under focus-fallback or `--headed`, non-ASCII text on macOS may be briefly placed on the clipboard to paste it. Do not use that path for secrets; prefer `set-value` when the target supports it.
|
||||
Headless `type` uses `AXSelectedText` without focusing the app or synthesizing keys. Pass `--headed` to focus the target and synthesize keyboard input. Use `set-value` when direct semantic value assignment is the intended interaction.
|
||||
|
||||
### set-value
|
||||
```bash
|
||||
|
|
@ -117,7 +115,7 @@ Sets the value directly via the AX value attribute. Faster than `type` but may n
|
|||
```bash
|
||||
agent-desktop clear @s8f3k2p9:e2
|
||||
```
|
||||
Clears the element's value to an empty string. Equivalent to `set-value @s8f3k2p9:e2 ""`.
|
||||
Headless clears through `AXValue`. With `--headed`, it performs focus + Select All + Delete first.
|
||||
|
||||
### focus
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -79,11 +79,10 @@ The default activation-chain deadline is 10 seconds. Set `AGENT_DESKTOP_CHAIN_TI
|
|||
|
||||
Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. macOS actions split semantic AX steps from explicit physical/headed paths:
|
||||
|
||||
- `click`, `right-click`, `scroll`, `set-value`, `clear`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, and `scroll-to` try AX-first semantics and fail clearly when the headless path is unavailable.
|
||||
- `type` uses focus fallback in the CLI/ref-action path. It may focus the target field, never moves the cursor, and can use the pasteboard for non-ASCII insertion. Use `set-value` for pure headless value mutation when supported.
|
||||
- `focus`, `press`, `hover`, `drag`, and `mouse-*` are explicit physical/focus/cursor commands.
|
||||
- FFI ref-action callers should use focus fallback for `type` to match CLI behavior; direct-handle `ad_execute_action` is lower-level and defaults to headless.
|
||||
- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Use `set-value` for sensitive text when possible.
|
||||
- `click`, `right-click`, `type`, `clear`, `scroll`, `expand`, and `collapse` use semantic AX delivery headlessly and prefer physical input with `--headed`.
|
||||
- `set-value`, `select`, `toggle`, `check`, `uncheck`, `focus`, and `scroll-to` are semantic-only even under `--headed`.
|
||||
- `press`, `hover`, `drag`, and `mouse-*` are explicit physical input; cursor-moving commands require `--headed`.
|
||||
- FFI ref-action callers get the same strict headless `type` default; direct-handle `ad_execute_action` is lower-level and applies the supplied policy verbatim.
|
||||
- If a command would need a forbidden physical path, it returns a structured error with a recovery hint.
|
||||
|
||||
### Surfaces
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ OBSERVATION
|
|||
|
||||
INTERACTION
|
||||
click <ref> Click element (kAXPress)
|
||||
double-click <ref> Open via AXOpen; physical double-click uses mouse-click
|
||||
double-click <ref> Physical double-click; requires --headed
|
||||
triple-click <ref> Triple-click element; POLICY_DENIED if physical input is disabled
|
||||
right-click <ref> Right-click; includes menu when verified
|
||||
type <ref> <text> Insert text; may use focus fallback only for explicit policy paths
|
||||
|
|
|
|||
|
|
@ -42,13 +42,13 @@ pub(crate) enum Commands {
|
|||
Is(IsArgs),
|
||||
#[command(about = "Click element via accessibility press action")]
|
||||
Click(RefArgs),
|
||||
#[command(about = "Open element via AXOpen; physical double-click uses mouse-click")]
|
||||
#[command(about = "Physically double-click element; requires --headed")]
|
||||
DoubleClick(RefArgs),
|
||||
#[command(
|
||||
about = "Triple-click element; returns POLICY_DENIED when physical input is disabled"
|
||||
)]
|
||||
TripleClick(RefArgs),
|
||||
#[command(about = "Right-click and include menu/menu_probe details when available")]
|
||||
#[command(about = "Open a context menu semantically, or physically with --headed")]
|
||||
RightClick(RefArgs),
|
||||
#[command(about = "Insert text into a text target")]
|
||||
Type(TypeArgs),
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ pub(crate) struct Cli {
|
|||
#[arg(
|
||||
long,
|
||||
global = true,
|
||||
help = "Permit cursor movement and focus stealing for physical input commands and fallbacks. Default is headless (AX-only, no cursor)."
|
||||
help = "Prefer physical delivery for natural input commands and permit focus/cursor side effects. Default is strict headless semantic delivery."
|
||||
)]
|
||||
pub headed: bool,
|
||||
#[command(flatten)]
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ pub(crate) fn dispatch(
|
|||
Commands::Maximize(args) => app_window::maximize(args, adapter),
|
||||
Commands::Restore(args) => app_window::restore(args, adapter),
|
||||
Commands::ListSurfaces(args) => app_window::list_surfaces(args, adapter),
|
||||
Commands::ListNotifications(args) => notifications::list(args, adapter),
|
||||
Commands::ListNotifications(args) => notifications::list(args, adapter, context),
|
||||
Commands::DismissNotification(args) => notifications::dismiss(args, adapter, context),
|
||||
Commands::DismissAllNotifications(args) => {
|
||||
notifications::dismiss_all(args, adapter, context)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ use crate::cli_args::notifications::{
|
|||
pub(super) fn list(
|
||||
args: ListNotificationsCliArgs,
|
||||
adapter: &dyn PlatformAdapter,
|
||||
context: &CommandContext,
|
||||
) -> Result<Value, AppError> {
|
||||
list_notifications::execute(
|
||||
list_notifications::ListNotificationsArgs {
|
||||
|
|
@ -24,6 +25,7 @@ pub(super) fn list(
|
|||
limit: args.limit,
|
||||
},
|
||||
adapter,
|
||||
context,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -287,6 +287,19 @@ def command_match_value():
|
|||
print("" if match["value"] is None else match["value"])
|
||||
|
||||
|
||||
def command_delivered_mechanism():
|
||||
data = read_json()
|
||||
steps = data.get("data", {}).get("steps", [])
|
||||
delivered = [
|
||||
step.get("mechanism")
|
||||
for step in steps
|
||||
if isinstance(step, dict) and step.get("outcome") == "succeeded"
|
||||
]
|
||||
if data.get("ok") is not True or not delivered or not delivered[0]:
|
||||
raise SystemExit(2)
|
||||
print(delivered[0])
|
||||
|
||||
|
||||
def command_tree(name, mode):
|
||||
data = read_json()
|
||||
node = find_node(data.get("data", {}).get("tree", {}), name)
|
||||
|
|
@ -377,6 +390,8 @@ def main():
|
|||
command_target()
|
||||
elif command == "match-value":
|
||||
command_match_value()
|
||||
elif command == "delivered-mechanism":
|
||||
command_delivered_mechanism()
|
||||
elif command == "tree":
|
||||
command_tree(sys.argv[2], sys.argv[3])
|
||||
elif command == "duplicate-ids":
|
||||
|
|
|
|||
|
|
@ -202,17 +202,18 @@ wait_target() {
|
|||
}
|
||||
|
||||
verify() {
|
||||
local label="$1" status="$2" expected="$3" target="$4" command="$5"
|
||||
shift 5
|
||||
local before after output command_ok error
|
||||
local label="$1" status="$2" expected="$3" expected_mechanism="$4" target="$5" command="$6"
|
||||
shift 6
|
||||
local before after output command_ok error mechanism
|
||||
require_value before "$status"
|
||||
output="$(act_target "$target" "$command" "$@" 2>&1)"
|
||||
sleep 0.35
|
||||
require_value after "$status"
|
||||
command_ok="$(json_field "$output" ok)"
|
||||
error="$(json_field "$output" error.code)"
|
||||
assert "$label" "$([ "$after" = "$expected" ] && [ "$command_ok" = "True" ] && echo 1 || echo 0)" \
|
||||
"before='$before' after='$after' expected='$expected' ok=$command_ok${error:+ error=$error}"
|
||||
mechanism="$(printf '%s' "$output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)"
|
||||
assert "$label" "$([ "$after" = "$expected" ] && [ "$command_ok" = "True" ] && [ "$mechanism" = "$expected_mechanism" ] && echo 1 || echo 0)" \
|
||||
"before='$before' after='$after' expected='$expected' ok=$command_ok mechanism=$mechanism${error:+ error=$error}"
|
||||
}
|
||||
|
||||
run_timed() {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ if [ ! -x "$release_bin" ]; then
|
|||
echo "release binary missing at $release_bin; run 'cargo build --release'" >&2
|
||||
exit 2
|
||||
fi
|
||||
note "Strict headless non-interference gate"
|
||||
AGENT_DESKTOP_E2E_RELEASE_BIN="$release_bin" bash "$here/safe-semantic.sh"
|
||||
safe_semantic_status=$?
|
||||
if [ "$safe_semantic_status" -ne 0 ]; then
|
||||
exit "$safe_semantic_status"
|
||||
fi
|
||||
if ! prepare_native_harness; then
|
||||
exit 2
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
interaction_suite() {
|
||||
local mode="$1" selection slider stepper direction
|
||||
local mode="$1" selection slider stepper direction natural_mechanism
|
||||
MODE_FLAG=""
|
||||
if [ "$mode" = "headed" ]; then
|
||||
MODE_FLAG="--headed"
|
||||
|
|
@ -7,16 +7,18 @@ interaction_suite() {
|
|||
slider="60"
|
||||
stepper="6"
|
||||
direction="up"
|
||||
natural_mechanism="physical_synthetic"
|
||||
"$bin" focus-window --app "$app" >/dev/null 2>&1
|
||||
else
|
||||
selection="Beta"
|
||||
slider="50"
|
||||
stepper="4"
|
||||
direction="down"
|
||||
natural_mechanism="semantic_api"
|
||||
fi
|
||||
"$bin" focus-window --app "$app" >/dev/null 2>&1
|
||||
|
||||
note "[$mode] exact-once click and text actions"
|
||||
local primary text_field click_before click_after before_number after_number click_output click_ok
|
||||
local primary text_field click_before click_after before_number after_number click_output click_ok click_mechanism
|
||||
require_target primary button primary-button
|
||||
require_value click_before click-status
|
||||
click_output="$(act_target "$primary" click 2>&1)"
|
||||
|
|
@ -27,45 +29,47 @@ interaction_suite() {
|
|||
case "$before_number" in *[!0-9]*|'') before_number=0 ;; esac
|
||||
case "$after_number" in *[!0-9]*|'') after_number=0 ;; esac
|
||||
click_ok="$(json_field "$click_output" ok)"
|
||||
click_mechanism="$(printf '%s' "$click_output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)"
|
||||
assert "[$mode] click dispatches exactly once" \
|
||||
"$([ "$click_ok" = "True" ] && [ "$after_number" -eq $((before_number + 1)) ] && echo 1 || echo 0)" \
|
||||
"before=$click_before after=$click_after ok=$click_ok"
|
||||
"$([ "$click_ok" = "True" ] && [ "$after_number" -eq $((before_number + 1)) ] && [ "$click_mechanism" = "$natural_mechanism" ] && echo 1 || echo 0)" \
|
||||
"before=$click_before after=$click_after ok=$click_ok mechanism=$click_mechanism"
|
||||
|
||||
require_target text_field textfield text-input
|
||||
act_target "$text_field" clear >/dev/null 2>&1
|
||||
sleep 0.2
|
||||
require_target text_field textfield text-input
|
||||
verify "[$mode] type sets field" text-echo "typed-$mode" "$text_field" type "typed-$mode"
|
||||
verify "[$mode] type sets field" text-echo "typed-$mode" "$natural_mechanism" "$text_field" type "typed-$mode"
|
||||
require_target text_field textfield text-input
|
||||
verify "[$mode] set-value sets field" text-echo "set-$mode" "$text_field" set-value "set-$mode"
|
||||
verify "[$mode] set-value sets field" text-echo "set-$mode" semantic_api "$text_field" set-value "set-$mode"
|
||||
require_target text_field textfield text-input
|
||||
verify "[$mode] clear empties field" text-echo "" "$text_field" clear
|
||||
verify "[$mode] clear empties field" text-echo "" "$natural_mechanism" "$text_field" clear
|
||||
|
||||
note "[$mode] state and value controls"
|
||||
local toggle picker native_slider native_stepper scroll_area scroll_before scroll_after
|
||||
local toggle picker native_slider native_stepper scroll_area scroll_before scroll_after scroll_output scroll_mechanism
|
||||
require_target toggle checkbox toggle-box
|
||||
act_target "$toggle" uncheck >/dev/null 2>&1
|
||||
sleep 0.2
|
||||
require_target toggle checkbox toggle-box
|
||||
verify "[$mode] check turns toggle on" toggle-status on "$toggle" check
|
||||
verify "[$mode] check turns toggle on" toggle-status on semantic_api "$toggle" check
|
||||
require_target toggle checkbox toggle-box
|
||||
verify "[$mode] uncheck turns toggle off" toggle-status off "$toggle" uncheck
|
||||
verify "[$mode] uncheck turns toggle off" toggle-status off semantic_api "$toggle" uncheck
|
||||
|
||||
require_target_by_id picker combobox option-picker
|
||||
verify "[$mode] select combobox" picker-status "$selection" "$picker" select "$selection"
|
||||
verify "[$mode] select combobox" picker-status "$selection" semantic_api "$picker" select "$selection"
|
||||
require_target native_slider slider value-slider
|
||||
verify "[$mode] set slider value" slider-status "$slider" "$native_slider" set-value "$slider"
|
||||
verify "[$mode] set slider value" slider-status "$slider" semantic_api "$native_slider" set-value "$slider"
|
||||
require_target native_stepper incrementor value-stepper
|
||||
verify "[$mode] set stepper value" stepper-status "$stepper" "$native_stepper" set-value "$stepper"
|
||||
verify "[$mode] set stepper value" stepper-status "$stepper" semantic_api "$native_stepper" set-value "$stepper"
|
||||
|
||||
require_target scroll_area scrollarea scroll-area
|
||||
require_value scroll_before scroll-offset
|
||||
scroll_output="$(act_target "$scroll_area" scroll --direction "$direction" --amount 10 2>&1)"
|
||||
sleep 0.4
|
||||
require_value scroll_after scroll-offset
|
||||
scroll_mechanism="$(printf '%s' "$scroll_output" | python3 "$json_tool" delivered-mechanism 2>/dev/null)"
|
||||
assert "[$mode] scroll moves content" \
|
||||
"$([ "$scroll_before" != "$scroll_after" ] && echo 1 || echo 0)" \
|
||||
"before=$scroll_before after=$scroll_after direction=$direction cmd_ok=$(json_field "$scroll_output" ok) cmd_err=$(json_field "$scroll_output" error.code) mechanism=$(json_field "$scroll_output" data.steps.0.mechanism)"
|
||||
"$([ "$scroll_before" != "$scroll_after" ] && [ "$scroll_mechanism" = "$natural_mechanism" ] && echo 1 || echo 0)" \
|
||||
"before=$scroll_before after=$scroll_after direction=$direction cmd_ok=$(json_field "$scroll_output" ok) cmd_err=$(json_field "$scroll_output" error.code) mechanism=$scroll_mechanism"
|
||||
}
|
||||
|
||||
interaction_suite headless
|
||||
|
|
@ -74,11 +78,11 @@ MODE_FLAG=""
|
|||
|
||||
note "Radio and tab selection"
|
||||
require_target radio_two radiobutton Two
|
||||
verify "click radio option Two" radio-status Two "$radio_two" click
|
||||
verify "click radio option Two" radio-status Two semantic_api "$radio_two" click
|
||||
require_target tab_two radiobutton "Tab Two"
|
||||
verify "select Tab Two" tab-status 1 "$tab_two" click
|
||||
verify "select Tab Two" tab-status 1 semantic_api "$tab_two" click
|
||||
require_target tab_one radiobutton "Tab One"
|
||||
verify "select Tab One" tab-status 0 "$tab_one" click
|
||||
verify "select Tab One" tab-status 0 semantic_api "$tab_one" click
|
||||
|
||||
note "Headed gesture fallback"
|
||||
require_target double_target button double-target
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ if [ -n "$nc_posted" ]; then
|
|||
nc_list=""
|
||||
nc_found=""
|
||||
for _ in $(seq 1 20); do
|
||||
nc_list="$("$bin" list-notifications --text "$nc_title_a" 2>/dev/null)"
|
||||
nc_list="$("$bin" --headed list-notifications --text "$nc_title_a" 2>/dev/null)"
|
||||
if [ "$(json_field "$nc_list" data.count)" = "1" ]; then
|
||||
nc_found=1
|
||||
break
|
||||
|
|
@ -51,7 +51,7 @@ if [ -n "$nc_posted" ]; then
|
|||
nc_gone=""
|
||||
nc_after=""
|
||||
for _ in $(seq 1 12); do
|
||||
nc_after="$("$bin" list-notifications --text "$nc_title_a" 2>/dev/null)"
|
||||
nc_after="$("$bin" --headed list-notifications --text "$nc_title_a" 2>/dev/null)"
|
||||
if [ "$(json_field "$nc_after" data.count)" = "0" ]; then
|
||||
nc_gone=1
|
||||
break
|
||||
|
|
@ -65,7 +65,7 @@ if [ -n "$nc_posted" ]; then
|
|||
osascript -e "display notification \"$nc_body\" with title \"$nc_title_b\"" >/dev/null 2>&1
|
||||
nc_list_b=""
|
||||
for _ in $(seq 1 10); do
|
||||
nc_list_b="$("$bin" list-notifications --text "$nc_title_b" 2>/dev/null)"
|
||||
nc_list_b="$("$bin" --headed list-notifications --text "$nc_title_b" 2>/dev/null)"
|
||||
if [ "$(json_field "$nc_list_b" data.count)" = "1" ]; then
|
||||
break
|
||||
fi
|
||||
|
|
@ -77,7 +77,7 @@ if [ -n "$nc_posted" ]; then
|
|||
nc_cleared=""
|
||||
nc_left=""
|
||||
for _ in $(seq 1 12); do
|
||||
nc_left="$("$bin" list-notifications --text "$nc_prefix" 2>/dev/null)"
|
||||
nc_left="$("$bin" --headed list-notifications --text "$nc_prefix" 2>/dev/null)"
|
||||
if [ "$(json_field "$nc_left" data.count)" = "0" ]; then
|
||||
nc_cleared=1
|
||||
break
|
||||
|
|
|
|||
|
|
@ -63,6 +63,13 @@ class HarnessContractTests(unittest.TestCase):
|
|||
self.assertIn("require_value actionable_before click-status", source)
|
||||
self.assertIn("require_value actionable_after click-status", source)
|
||||
|
||||
def test_native_runner_gates_on_strict_headless_non_interference(self):
|
||||
source = (E2E_ROOT / "run.sh").read_text()
|
||||
|
||||
safe_gate = source.index('bash "$here/safe-semantic.sh"')
|
||||
focused_fixture = source.index("prepare_native_harness")
|
||||
self.assertLess(safe_gate, focused_fixture)
|
||||
|
||||
def test_fixture_status_oracles_use_stable_native_identifiers(self):
|
||||
source = (E2E_ROOT / "lib.sh").read_text()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,10 +5,27 @@ import tempfile
|
|||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from json_tool import run_bounded
|
||||
from json_tool import command_delivered_mechanism, run_bounded
|
||||
|
||||
|
||||
class RunBoundedTests(unittest.TestCase):
|
||||
def test_delivered_mechanism_reports_the_first_successful_step(self):
|
||||
payload = {
|
||||
"ok": True,
|
||||
"data": {
|
||||
"steps": [
|
||||
{"outcome": "skipped", "mechanism": "physical_synthetic"},
|
||||
{"outcome": "succeeded", "mechanism": "semantic_api"},
|
||||
]
|
||||
},
|
||||
}
|
||||
with mock.patch("json_tool.read_json", return_value=payload), mock.patch(
|
||||
"builtins.print"
|
||||
) as output:
|
||||
command_delivered_mechanism()
|
||||
|
||||
output.assert_called_once_with("semantic_api")
|
||||
|
||||
def test_marked_child_receives_the_canonical_interaction_lease_fd(self):
|
||||
with tempfile.TemporaryFile() as lease:
|
||||
os.set_inheritable(lease.fileno(), True)
|
||||
|
|
|
|||
Loading…
Reference in a new issue