From 1e4667ad3a2eab58b4dd6ebdc095e48945922666 Mon Sep 17 00:00:00 2001 From: Lahfir Date: Tue, 12 May 2026 00:42:25 -0700 Subject: [PATCH] refactor!: unify command execution contracts BREAKING CHANGE: CLI and batch execution now share the typed command path and current command argument contracts. BREAKING CHANGE: Ref-consuming commands use snapshot-scoped refs; deterministic consumers should pass snapshot_id and handle SNAPSHOT_NOT_FOUND. BREAKING CHANGE: permissions and status now return PermissionReport fields for accessibility, screen_recording, and automation instead of a single boolean status. BREAKING CHANGE: PermissionState gains NotRequired; macOS automation now reports not_required instead of unknown. BREAKING CHANGE: right-click now separates action success from menu verification; consumers should inspect menu or menu_probe instead of assuming every right-click returns an inline menu. BREAKING CHANGE: focus-window now confirms OS focus and returns ACTION_FAILED when focus does not settle; data.focused.is_focused is true on success. BREAKING CHANGE: PlatformAdapter::execute_action now takes ActionRequest, and permission probing uses permission_report/request_permissions. BREAKING CHANGE: FFI ad_execute_action now defaults to headless policy. Consumers that need focus fallback or cursor-moving behavior must call ad_execute_action_with_policy with AD_POLICY_KIND_FOCUS_FALLBACK or AD_POLICY_KIND_PHYSICAL. BREAKING CHANGE: FFI ad_check_permissions no longer treats unknown accessibility permission as success; stub-style unknown probes return ERR_PLATFORM_NOT_SUPPORTED and macOS ambiguous unknown returns ERR_INTERNAL with last-error detail. BREAKING CHANGE: JSON response envelopes now report version 2.0; parsers pinned to 1.0 must branch or update. BREAKING CHANGE: focus now uses accessibility focus without cursor movement; callers that need physical focus must use explicit mouse or physical-policy paths. BREAKING CHANGE: chain execution deadlines now return TIMEOUT instead of ACTION_FAILED when the target app does not respond before the chain deadline. --- CHANGELOG.md | 2 +- CLAUDE.md | 44 +- Cargo.toml | 1 + README.md | 69 ++- crates/core/Cargo.toml | 1 + crates/core/src/action.rs | 89 +++ crates/core/src/adapter.rs | 55 +- crates/core/src/commands/check.rs | 15 +- crates/core/src/commands/clear.rs | 15 +- crates/core/src/commands/click.rs | 15 +- crates/core/src/commands/collapse.rs | 8 +- crates/core/src/commands/double_click.rs | 15 +- crates/core/src/commands/drag.rs | 22 +- crates/core/src/commands/expand.rs | 8 +- crates/core/src/commands/find.rs | 300 ++++++++-- crates/core/src/commands/focus.rs | 8 +- crates/core/src/commands/focus_window.rs | 217 +++++++- crates/core/src/commands/get.rs | 5 +- crates/core/src/commands/helpers.rs | 226 +++++++- crates/core/src/commands/hover.rs | 5 +- crates/core/src/commands/is_check.rs | 266 ++++++++- crates/core/src/commands/key_down.rs | 7 +- crates/core/src/commands/key_up.rs | 7 +- crates/core/src/commands/list_apps.rs | 54 +- crates/core/src/commands/maximize.rs | 33 +- crates/core/src/commands/minimize.rs | 33 +- crates/core/src/commands/mod.rs | 5 + crates/core/src/commands/move_window.rs | 21 +- crates/core/src/commands/permissions.rs | 39 +- crates/core/src/commands/press.rs | 5 +- crates/core/src/commands/ref_policy_tests.rs | 158 ++++++ crates/core/src/commands/resize_window.rs | 21 +- crates/core/src/commands/resolved_element.rs | 22 + crates/core/src/commands/restore.rs | 33 +- crates/core/src/commands/right_click.rs | 231 +++++++- crates/core/src/commands/scroll.rs | 10 +- crates/core/src/commands/scroll_to.rs | 15 +- crates/core/src/commands/search_text.rs | 31 ++ crates/core/src/commands/select.rs | 13 +- crates/core/src/commands/set_value.rs | 13 +- crates/core/src/commands/skills.rs | 2 +- crates/core/src/commands/snapshot.rs | 10 +- crates/core/src/commands/status.rs | 69 ++- crates/core/src/commands/toggle.rs | 8 +- crates/core/src/commands/triple_click.rs | 15 +- crates/core/src/commands/type_text.rs | 14 +- crates/core/src/commands/uncheck.rs | 15 +- crates/core/src/commands/wait.rs | 185 +++++-- crates/core/src/commands/wait_tests.rs | 191 +++++++ crates/core/src/error.rs | 29 +- crates/core/src/lib.rs | 19 +- crates/core/src/node.rs | 5 + crates/core/src/output.rs | 6 +- crates/core/src/permission_report.rs | 80 +++ crates/core/src/permission_state.rs | 8 + crates/core/src/ref_alloc.rs | 100 +++- crates/core/src/refs.rs | 511 +++++------------- crates/core/src/refs_lock.rs | 296 ++++++++++ crates/core/src/refs_store.rs | 196 +++++++ crates/core/src/refs_test_support.rs | 51 ++ crates/core/src/refs_tests.rs | 309 +++++++++++ crates/core/src/roles.rs | 104 ++++ crates/core/src/snapshot.rs | 32 +- crates/core/src/snapshot_ref.rs | 39 +- crates/core/src/snapshot_ref_alloc_tests.rs | 107 ++++ crates/core/src/snapshot_ref_tests.rs | 188 +++---- crates/core/src/snapshot_tests.rs | 4 + crates/core/src/window_lookup.rs | 27 + crates/ffi/cbindgen.toml | 1 + crates/ffi/include/agent_desktop.h | 23 + crates/ffi/src/actions/execute.rs | 44 +- crates/ffi/src/actions/resolve.rs | 2 + crates/ffi/src/adapter.rs | 80 ++- crates/ffi/src/enum_validation.rs | 17 +- crates/ffi/src/error.rs | 102 +--- crates/ffi/src/error_tests.rs | 78 +++ crates/ffi/src/lib.rs | 1 + crates/ffi/src/observation/find.rs | 2 + crates/ffi/src/observation/is.rs | 1 + crates/ffi/src/observation/walk.rs | 1 + crates/ffi/src/tree/flatten.rs | 1 + crates/ffi/src/types/mod.rs | 2 + crates/ffi/src/types/policy_kind.rs | 19 + crates/ffi/tests/c_abi_actions.rs | 62 +++ crates/ffi/tests/c_abi_harness.rs | 468 ---------------- crates/ffi/tests/c_abi_inputs.rs | 59 ++ crates/ffi/tests/c_abi_layout.rs | 23 + crates/ffi/tests/c_abi_lifecycle.rs | 176 ++++++ crates/ffi/tests/common/mod.rs | 100 ++++ crates/ffi/tests/error_lifetime.rs | 5 +- crates/macos/src/actions/ax_helpers.rs | 76 ++- crates/macos/src/actions/chain.rs | 238 +++++--- crates/macos/src/actions/chain_context.rs | 4 + crates/macos/src/actions/chain_def.rs | 7 + crates/macos/src/actions/chain_defs.rs | 110 ++-- crates/macos/src/actions/chain_menu_steps.rs | 249 +++++++++ crates/macos/src/actions/chain_step.rs | 34 ++ crates/macos/src/actions/chain_steps.rs | 420 ++++++-------- crates/macos/src/actions/chain_steps_tests.rs | 47 ++ crates/macos/src/actions/chain_web_steps.rs | 118 ++++ crates/macos/src/actions/dispatch.rs | 197 +++---- crates/macos/src/actions/extras.rs | 431 +++++---------- crates/macos/src/actions/mod.rs | 28 +- crates/macos/src/actions/post_state.rs | 51 ++ crates/macos/src/actions/scroll.rs | 328 +++++++++++ crates/macos/src/actions/toggle_state.rs | 195 +++++++ crates/macos/src/actions/type_text.rs | 157 ++++++ crates/macos/src/adapter.rs | 193 +++---- crates/macos/src/input/keyboard.rs | 377 ++++++------- crates/macos/src/input/keyboard_map.rs | 174 ++++++ crates/macos/src/input/mod.rs | 1 + crates/macos/src/notifications/nc_session.rs | 113 ++-- crates/macos/src/system/app_list.rs | 363 +++++++++++++ crates/macos/src/system/app_list_tests.rs | 64 +++ crates/macos/src/system/app_ops.rs | 115 +--- crates/macos/src/system/key_dispatch.rs | 22 +- crates/macos/src/system/mod.rs | 3 + crates/macos/src/system/permissions.rs | 75 ++- crates/macos/src/system/process.rs | 119 ++++ crates/macos/src/system/screenshot.rs | 182 +++++-- crates/macos/src/system/wait.rs | 18 +- crates/macos/src/system/window_list.rs | 196 +++++++ crates/macos/src/system/window_ops.rs | 33 +- crates/macos/src/tree/action_list.rs | 68 +++ crates/macos/src/tree/ax_element.rs | 41 ++ crates/macos/src/tree/build_context.rs | 18 + crates/macos/src/tree/builder.rs | 140 +++-- crates/macos/src/tree/builder_tests.rs | 29 + crates/macos/src/tree/capabilities.rs | 62 +++ crates/macos/src/tree/element.rs | 155 ++---- crates/macos/src/tree/element_bounds.rs | 80 +++ crates/macos/src/tree/mod.rs | 16 +- crates/macos/src/tree/resolve.rs | 314 ++++++++--- crates/macos/src/tree/roles.rs | 64 ++- crates/macos/src/tree/surfaces.rs | 46 +- docs/phases.md | 263 ++++----- ...velope-version-bump-contract-2026-05-13.md | 62 +++ ...tion-policy-aligned-with-cli-2026-05-12.md | 44 ++ ...cy-semantics-during-refactor-2026-05-12.md | 86 +++ ...ve-snapshot-review-contract-2026-04-16.md} | 6 +- skills/agent-desktop-ffi/SKILL.md | 11 + .../references/error-handling.md | 11 +- skills/agent-desktop/SKILL.md | 59 +- .../references/commands-interaction.md | 33 +- .../references/commands-observation.md | 22 +- .../references/commands-system.md | 49 +- skills/agent-desktop/references/macos.md | 69 ++- skills/agent-desktop/references/workflows.md | 68 +-- src/batch.rs | 309 +++++++++++ src/batch_dispatch.rs | 348 ------------ src/batch_dispatch_ext.rs | 208 ------- src/cli.rs | 42 +- src/cli_args.rs | 328 +++-------- src/cli_args_actions.rs | 164 ++++++ src/cli_args_notifications.rs | 13 +- src/cli_args_system.rs | 158 ++++++ src/command_policy.rs | 228 ++++++++ src/dispatch.rs | 187 ++----- src/dispatch_parse.rs | 129 +++++ src/help_after.txt | 31 +- src/main.rs | 106 ++-- 161 files changed, 10293 insertions(+), 4536 deletions(-) create mode 100644 crates/core/src/commands/ref_policy_tests.rs create mode 100644 crates/core/src/commands/resolved_element.rs create mode 100644 crates/core/src/commands/search_text.rs create mode 100644 crates/core/src/commands/wait_tests.rs create mode 100644 crates/core/src/permission_report.rs create mode 100644 crates/core/src/permission_state.rs create mode 100644 crates/core/src/refs_lock.rs create mode 100644 crates/core/src/refs_store.rs create mode 100644 crates/core/src/refs_test_support.rs create mode 100644 crates/core/src/refs_tests.rs create mode 100644 crates/core/src/roles.rs create mode 100644 crates/core/src/snapshot_ref_alloc_tests.rs create mode 100644 crates/core/src/window_lookup.rs create mode 100644 crates/ffi/src/error_tests.rs create mode 100644 crates/ffi/src/types/policy_kind.rs create mode 100644 crates/ffi/tests/c_abi_actions.rs delete mode 100644 crates/ffi/tests/c_abi_harness.rs create mode 100644 crates/ffi/tests/c_abi_inputs.rs create mode 100644 crates/ffi/tests/c_abi_layout.rs create mode 100644 crates/ffi/tests/c_abi_lifecycle.rs create mode 100644 crates/ffi/tests/common/mod.rs create mode 100644 crates/macos/src/actions/chain_context.rs create mode 100644 crates/macos/src/actions/chain_def.rs create mode 100644 crates/macos/src/actions/chain_menu_steps.rs create mode 100644 crates/macos/src/actions/chain_step.rs create mode 100644 crates/macos/src/actions/chain_steps_tests.rs create mode 100644 crates/macos/src/actions/chain_web_steps.rs create mode 100644 crates/macos/src/actions/post_state.rs create mode 100644 crates/macos/src/actions/scroll.rs create mode 100644 crates/macos/src/actions/toggle_state.rs create mode 100644 crates/macos/src/actions/type_text.rs create mode 100644 crates/macos/src/input/keyboard_map.rs create mode 100644 crates/macos/src/system/app_list.rs create mode 100644 crates/macos/src/system/app_list_tests.rs create mode 100644 crates/macos/src/system/process.rs create mode 100644 crates/macos/src/system/window_list.rs create mode 100644 crates/macos/src/tree/action_list.rs create mode 100644 crates/macos/src/tree/ax_element.rs create mode 100644 crates/macos/src/tree/build_context.rs create mode 100644 crates/macos/src/tree/builder_tests.rs create mode 100644 crates/macos/src/tree/capabilities.rs create mode 100644 crates/macos/src/tree/element_bounds.rs create mode 100644 docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md create mode 100644 docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md create mode 100644 docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md rename docs/solutions/logic-errors/{progressive-snapshot-review-hardening-2026-04-16.md => progressive-snapshot-review-contract-2026-04-16.md} (96%) create mode 100644 src/batch.rs delete mode 100644 src/batch_dispatch.rs delete mode 100644 src/batch_dispatch_ext.rs create mode 100644 src/cli_args_actions.rs create mode 100644 src/cli_args_system.rs create mode 100644 src/command_policy.rs create mode 100644 src/dispatch_parse.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 55286a5..3fa8341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Features -* **ffi:** ship C-ABI cdylib with review hardening and release pipeline ([#26](https://github.com/lahfir/agent-desktop/issues/26)) ([3cffbd6](https://github.com/lahfir/agent-desktop/commit/3cffbd67f6b27f42001643bef9fd2530cb7f9003)) +* **ffi:** ship C-ABI cdylib with review fixes and release pipeline ([#26](https://github.com/lahfir/agent-desktop/issues/26)) ([3cffbd6](https://github.com/lahfir/agent-desktop/commit/3cffbd67f6b27f42001643bef9fd2530cb7f9003)) ## [0.1.12](https://github.com/lahfir/agent-desktop/compare/v0.1.11...v0.1.12) (2026-04-16) diff --git a/CLAUDE.md b/CLAUDE.md index c1a8b0f..2da5344 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,8 +79,10 @@ agent-desktop/ │ ├── main.rs # entry point, permission check, JSON envelope │ ├── cli.rs # clap derive enum (Commands) │ ├── cli_args.rs # all command argument structs +│ ├── batch.rs # batch JSON → typed Commands +│ ├── command_policy.rs # command permission/ref/side-effect policy │ ├── dispatch.rs # command dispatcher + parse helpers -│ └── batch_dispatch.rs # batch command execution +│ └── dispatch_notifications.rs ├── docs/ │ └── solutions/ # documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (module, tags, problem_type); relevant when implementing or debugging in documented areas └── tests/ @@ -135,7 +137,11 @@ agent-desktop-linux = { path = "crates/linux" } Direct `match` in the binary crate. No `Command` trait, no `CommandRegistry`. Each command is a standalone `execute()` function under `crates/core/src/commands/`. ```rust -pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result { +pub fn dispatch( + cmd: Commands, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, +) -> Result { match cmd { Commands::Snapshot(args) => commands::snapshot::execute(args, adapter), Commands::Click(args) => commands::click::execute(args, adapter), @@ -144,14 +150,16 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result Result, AdapterError>; fn list_apps(&self) -> Result, AdapterError>; fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions) -> Result; fn get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions) -> Result; - fn execute_action(&self, handle: &NativeHandle, action: Action) -> Result; + fn execute_action(&self, handle: &NativeHandle, request: ActionRequest) -> Result; fn resolve_element(&self, entry: &RefEntry) -> Result; - fn check_permissions(&self) -> PermissionStatus; + fn permission_report(&self) -> PermissionReport; + fn request_permissions(&self) -> PermissionReport; fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError>; fn launch_app(&self, id: &str, wait: bool) -> Result; fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError>; @@ -331,12 +343,14 @@ pub trait PlatformAdapter: Send + Sync { ## Key Types -- `AccessibilityNode` — platform-agnostic tree node: `ref`, `role`, `name`, `value`, `description`, `states`, `bounds`, `children` +- `AccessibilityNode` — platform-agnostic tree node: `ref`, `role`, `name`, `value`, `description`, `states`, `available_actions`, `bounds`, `children` - `Action` — Click, DoubleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Scroll(Direction, Amount), PressKey(KeyCombo) +- `ActionRequest` — `{ action, policy }`, where the default `InteractionPolicy` forbids focus stealing and cursor movement - `NativeHandle` — opaque platform pointer with `PhantomData<*const ()>` to prevent auto-Send/Sync. Inner field is `pub(crate)`. - `RefEntry` — `{ pid, role, name, bounds_hash, available_actions }` - `WindowInfo` — `{ id, title, app_name, pid, bounds }` -- `ErrorCode` — 11-variant enum with `#[serde(rename_all = "SCREAMING_SNAKE_CASE")]` +- `PermissionReport` — `{ accessibility, screen_recording, automation }`, each `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, or `{ "state": "unknown" }` +- `ErrorCode` — machine-readable enum with `#[serde(rename_all = "SCREAMING_SNAKE_CASE")]` - `AdapterError` — struct with `code`, `message`, `suggestion`, `platform_detail` - `AppError` — enum with `#[from]` impls for `AdapterError`, `std::io::Error`, `serde_json::Error` @@ -355,7 +369,7 @@ pub trait PlatformAdapter: Send + Sync { - SetFocus: `AXUIElementSetAttributeValue(kAXFocusedAttribute, true)` - Keyboard/Mouse: `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` - Clipboard: `NSPasteboard.generalPasteboard` via Cocoa FFI -- Screenshot: `CGWindowListCreateImage` +- Screenshot: `ScreenshotBackend` boundary with secure `screencapture` temp files ### Permission Detection - Call `AXIsProcessTrusted()` on startup diff --git a/Cargo.toml b/Cargo.toml index 470026f..ce75023 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } base64 = "0.22" rustc-hash = "2.1" +libc = "0.2" agent-desktop-core = { path = "crates/core" } [profile.release] diff --git a/README.md b/README.md index 1fcbdbf..04d52d8 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ - **C-ABI cdylib** (`libagent_desktop_ffi`): Load once from Python / Swift / Go / Ruby / Node / C instead of forking the CLI per call - **54 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, plus a bundled `skills` doc loader - **Progressive skeleton traversal**: 78–96% token reduction on dense apps via shallow overview + targeted drill-down -- **Snapshot & refs**: AI-optimized workflow using deterministic element references (`@e1`, `@e2`) -- **AX-first interactions**: Every action exhausts pure accessibility API strategies before falling back to mouse events +- **Snapshot & refs**: AI-optimized workflow using compact snapshot IDs and deterministic element references (`@e1`, `@e2`) +- **Headless-by-default interactions**: Ref actions use accessibility APIs and block silent focus, cursor, keyboard, or pasteboard side effects - **Structured JSON output**: Machine-readable responses with error codes and recovery hints - **Works with any app**: Finder, Safari, System Settings, Xcode, Slack — anything with an accessibility tree @@ -71,10 +71,20 @@ Requires Rust 1.78+ and macOS 13.0+. ### Permissions -macOS requires Accessibility permission. Grant it in **System Settings > Privacy & Security > Accessibility** by adding your terminal app, or: +macOS requires Accessibility permission. Screenshots also require Screen Recording permission. Grant them in **System Settings > Privacy & Security** by adding the app that launches agent-desktop, or: ```bash -agent-desktop permissions --request # trigger system dialog +agent-desktop permissions --request # trigger platform permission request path +``` + +Permission fields are explicit objects, for example: + +```json +{ + "accessibility": { "state": "granted" }, + "screen_recording": { "state": "denied", "suggestion": "Grant Screen Recording permission" }, + "automation": { "state": "not_required" } +} ``` ## Language bindings (FFI) @@ -116,23 +126,24 @@ For dense apps (Slack, VS Code, Notion), use **progressive skeleton traversal** ```bash # 1. Shallow overview — depth-3 map, truncated containers show children_count agent-desktop snapshot --skeleton --app Slack -i --compact +# Keep snapshot_id, for example s8f3k2p9 # 2. Drill into a region of interest (named containers get refs as drill targets) -agent-desktop snapshot --root @e3 -i --compact +agent-desktop snapshot --root @e3 --snapshot s8f3k2p9 -i --compact # 3. Act on an element found in the drill-down -agent-desktop click @e12 +agent-desktop click @e12 --snapshot s8f3k2p9 # 4. Re-drill the same region to verify the state change -agent-desktop snapshot --root @e3 -i --compact +agent-desktop snapshot --root @e3 --snapshot s8f3k2p9 -i --compact ``` For simple apps, a full snapshot is fine: ```bash -agent-desktop snapshot --app Finder -i # get interactive elements with refs -agent-desktop click @e3 # click a button by ref -agent-desktop type @e5 "quarterly report" # type into a text field +agent-desktop snapshot --app Finder -i # get interactive elements with refs and snapshot_id +agent-desktop click @e3 --snapshot s8f3k2p9 # click a button by ref +agent-desktop type @e5 --snapshot s8f3k2p9 "quarterly report" # insert text into a field agent-desktop press cmd+s # keyboard shortcut agent-desktop snapshot -i # re-observe after UI changes ``` @@ -150,29 +161,29 @@ agent-desktop snapshot --app Safari -i # accessibility tree with refs agent-desktop snapshot --surface menu # capture open menu agent-desktop screenshot --app Finder # PNG screenshot agent-desktop find --role button --app TextEdit # search by role, name, value, text -agent-desktop get @e3 value # read element property -agent-desktop is @e7 checked # check boolean state +agent-desktop get @e3 --snapshot s8f3k2p9 --property value # read element property +agent-desktop is @e7 --snapshot s8f3k2p9 --property checked # check boolean state agent-desktop list-surfaces --app Notes # list menus, sheets, popovers, alerts ``` ### Interaction ```bash -agent-desktop click @e3 # smart AX-first click (15-step chain) -agent-desktop double-click @e3 # open files, select words -agent-desktop triple-click @e3 # select lines/paragraphs -agent-desktop right-click @e3 # context menu (returns menu tree inline) -agent-desktop type @e5 "hello world" # type text into element +agent-desktop click @e3 # semantic AX-first click +agent-desktop double-click @e3 # AXOpen; physical double-click uses mouse-click --count 2 +agent-desktop triple-click @e3 # POLICY_DENIED if physical input is disabled +agent-desktop right-click @e3 # open verified context menu +agent-desktop type @e5 "hello world" # insert text into element agent-desktop set-value @e5 "new value" # set value directly via AX agent-desktop clear @e5 # clear element value agent-desktop focus @e5 # set keyboard focus -agent-desktop select @e9 "Option B" # select option in dropdown/list +agent-desktop select @e9 "Option B" # select verified dropdown/list option agent-desktop toggle @e12 # flip checkbox or switch agent-desktop check @e12 # idempotent check agent-desktop uncheck @e12 # idempotent uncheck agent-desktop expand @e15 # expand disclosure/tree item agent-desktop collapse @e15 # collapse disclosure/tree item -agent-desktop scroll @e1 down 3 # scroll (AX-first, 10-step chain) +agent-desktop scroll @e1 --direction down --amount 3 # scroll (AX-first) agent-desktop scroll-to @e20 # scroll element into view ``` @@ -250,8 +261,8 @@ agent-desktop wait --menu --timeout 3000 # wait for menu ```bash agent-desktop batch '[ - {"command": "click", "args": {"ref_id": "@e2"}}, - {"command": "type", "args": {"ref_id": "@e5", "text": "hello"}}, + {"command": "click", "args": {"ref_id": "@e2", "snapshot": "s8f3k2p9"}}, + {"command": "type", "args": {"ref_id": "@e5", "snapshot": "s8f3k2p9", "text": "hello"}}, {"command": "press", "args": {"combo": "return"}} ]' --stop-on-error ``` @@ -259,9 +270,9 @@ agent-desktop batch '[ ### System ```bash -agent-desktop status # platform, permission state -agent-desktop permissions # check accessibility permission -agent-desktop permissions --request # trigger system dialog +agent-desktop status # platform, permission report, latest snapshot +agent-desktop permissions # check accessibility/screen-recording/automation +agent-desktop permissions --request # invoke platform request path agent-desktop version # version string ``` @@ -281,6 +292,7 @@ agent-desktop snapshot [OPTIONS] | `--max-depth ` | 10 | Maximum tree depth | | `--skeleton` | off | Shallow 3-level overview; truncated containers show `children_count` and get refs as drill targets | | `--root ` | - | Start traversal from this ref; merges into existing refmap with scoped invalidation | +| `--snapshot ` | latest | Snapshot ID to use when resolving `--root` | | `--surface ` | window | `window`, `focused`, `menu`, `menubar`, `sheet`, `popover`, `alert` | ## JSON Output @@ -289,7 +301,7 @@ Every command returns structured JSON: ```json { - "version": "1.0", + "version": "2.0", "ok": true, "command": "click", "data": { "action": "click" } @@ -300,7 +312,7 @@ Errors include machine-readable codes and recovery hints: ```json { - "version": "1.0", + "version": "2.0", "ok": false, "command": "click", "error": { @@ -319,7 +331,10 @@ Errors include machine-readable codes and recovery hints: | `ELEMENT_NOT_FOUND` | No element matched the ref or query | | `APP_NOT_FOUND` | Application not running or no windows | | `STALE_REF` | Ref is from a previous snapshot | +| `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired | +| `POLICY_DENIED` | Physical/headed path blocked by policy | | `ACTION_FAILED` | The OS rejected the action | +| `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform | | `TIMEOUT` | Wait condition expired | | `INVALID_ARGS` | Invalid argument values | @@ -329,7 +344,7 @@ Errors include machine-readable codes and recovery hints: ## Ref System -`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are valid until the next snapshot replaces them. +`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are scoped to a compact `snapshot_id` such as `s8f3k2p9`. Commands can omit `--snapshot` to use the latest snapshot pointer, but passing the ID is more deterministic in multi-step flows. Interactive roles that receive refs: `button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`. diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2a6b1a9..becfa7d 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -11,3 +11,4 @@ thiserror.workspace = true tracing.workspace = true rustc-hash.workspace = true base64.workspace = true +libc.workspace = true diff --git a/crates/core/src/action.rs b/crates/core/src/action.rs index 499377a..a5ea375 100644 --- a/crates/core/src/action.rs +++ b/crates/core/src/action.rs @@ -26,6 +26,95 @@ pub enum Action { Drag(DragParams), } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ActionRequest { + pub action: Action, + pub policy: InteractionPolicy, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct InteractionPolicy { + pub allow_focus_steal: bool, + pub allow_cursor_move: bool, +} + +impl ActionRequest { + pub fn headless(action: Action) -> Self { + Self { + action, + policy: InteractionPolicy::headless(), + } + } + + pub fn focus_fallback(action: Action) -> Self { + Self { + action, + policy: InteractionPolicy::focus_fallback(), + } + } + + pub fn physical(action: Action) -> Self { + Self { + action, + policy: InteractionPolicy::physical(), + } + } +} + +impl InteractionPolicy { + pub fn headless() -> Self { + Self { + allow_focus_steal: false, + allow_cursor_move: false, + } + } + + pub fn focus_fallback() -> Self { + Self { + allow_focus_steal: true, + allow_cursor_move: false, + } + } + + pub fn physical() -> Self { + Self { + allow_focus_steal: true, + allow_cursor_move: true, + } + } +} + +impl Default for InteractionPolicy { + fn default() -> Self { + Self::headless() + } +} + +#[cfg(test)] +mod policy_tests { + use super::*; + + #[test] + fn default_policy_is_headless() { + let policy = InteractionPolicy::default(); + assert!(!policy.allow_focus_steal); + assert!(!policy.allow_cursor_move); + } + + #[test] + fn headless_request_blocks_physical_side_effects() { + let request = ActionRequest::headless(Action::Click); + assert_eq!(request.policy, InteractionPolicy::headless()); + } + + #[test] + fn focus_fallback_policy_never_moves_cursor() { + let request = ActionRequest::focus_fallback(Action::Scroll(Direction::Down, 1)); + assert!(request.policy.allow_focus_steal); + assert!(!request.policy.allow_cursor_move); + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Direction { Up, diff --git a/crates/core/src/adapter.rs b/crates/core/src/adapter.rs index 28b0395..18f47e4 100644 --- a/crates/core/src/adapter.rs +++ b/crates/core/src/adapter.rs @@ -1,9 +1,12 @@ use crate::{ - action::{Action, ActionResult, DragParams, MouseEvent, WindowOp}, + action::{ + ActionRequest, ActionResult, DragParams, ElementState, KeyCombo, MouseEvent, WindowOp, + }, error::AdapterError, node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo}, notification::{NotificationFilter, NotificationIdentity, NotificationInfo}, refs::RefEntry, + PermissionReport, PermissionState, }; use std::marker::PhantomData; @@ -53,11 +56,6 @@ pub enum ScreenshotTarget { FullScreen, } -pub enum PermissionStatus { - Granted, - Denied { suggestion: String }, -} - pub struct NativeHandle { pub(crate) ptr: *const std::ffi::c_void, _not_send_sync: PhantomData<*const ()>, @@ -87,12 +85,6 @@ impl NativeHandle { } } -// SAFETY: Phase 1 is single-threaded CLI. NativeHandle is never sent across thread -// boundaries. The unsafe impls are required for use with dyn PlatformAdapter (which -// is Send + Sync). Remove in Phase 4 when async daemon is introduced. -unsafe impl Send for NativeHandle {} -unsafe impl Sync for NativeHandle {} - pub struct ImageBuffer { pub data: Vec, pub format: ImageFormat, @@ -134,7 +126,7 @@ pub trait PlatformAdapter: Send + Sync { fn execute_action( &self, _handle: &NativeHandle, - _action: Action, + _request: ActionRequest, ) -> Result { Err(AdapterError::not_supported("execute_action")) } @@ -144,20 +136,33 @@ pub trait PlatformAdapter: Send + Sync { } /// Releases a platform-specific element handle returned from - /// `resolve_element`. macOS implementations must `CFRelease` the - /// underlying `AXUIElementRef` to balance the `CFRetain` that - /// happened during resolve. Windows/Linux consumers can leave this - /// as the default `not_supported` no-op. + /// `resolve_element`. Adapter methods that receive `&NativeHandle` + /// borrow it only; they must not consume or release it. macOS + /// implementations must `CFRelease` here to balance the `CFRetain` + /// that happened during resolve. Windows/Linux consumers can leave + /// this as the default no-op. fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { - Err(AdapterError::not_supported("release_handle")) + Ok(()) } - fn check_permissions(&self) -> PermissionStatus { - PermissionStatus::Denied { - suggestion: "Platform adapter not available".into(), + fn permission_report(&self) -> PermissionReport { + PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "Platform adapter not available".into(), + }, + screen_recording: PermissionState::Unknown, + automation: PermissionState::NotRequired, } } + fn unknown_accessibility_means_unsupported(&self) -> bool { + true + } + + fn request_permissions(&self) -> PermissionReport { + self.permission_report() + } + fn focus_window(&self, _win: &WindowInfo) -> Result<(), AdapterError> { Err(AdapterError::not_supported("focus_window")) } @@ -190,6 +195,10 @@ pub trait PlatformAdapter: Send + Sync { Err(AdapterError::not_supported("get_live_value")) } + fn get_live_state(&self, _handle: &NativeHandle) -> Result, AdapterError> { + Err(AdapterError::not_supported("get_live_state")) + } + fn press_key_for_app( &self, _app_name: &str, @@ -218,6 +227,10 @@ pub trait PlatformAdapter: Send + Sync { Err(AdapterError::not_supported("mouse_event")) } + fn key_event(&self, _combo: &KeyCombo, _down: bool) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("key_event")) + } + fn drag(&self, _params: DragParams) -> Result<(), AdapterError> { Err(AdapterError::not_supported("drag")) } diff --git a/crates/core/src/commands/check.rs b/crates/core/src/commands/check.rs index 6a6fd6f..44e9c0c 100644 --- a/crates/core/src/commands/check.rs +++ b/crates/core/src/commands/check.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct CheckArgs { - pub ref_id: String, -} - -pub fn execute(args: CheckArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Check)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::Check)) } diff --git a/crates/core/src/commands/clear.rs b/crates/core/src/commands/clear.rs index e6a8583..5a5c4df 100644 --- a/crates/core/src/commands/clear.rs +++ b/crates/core/src/commands/clear.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct ClearArgs { - pub ref_id: String, -} - -pub fn execute(args: ClearArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Clear)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::Clear)) } diff --git a/crates/core/src/commands/click.rs b/crates/core/src/commands/click.rs index 1e079c2..4d2fb3d 100644 --- a/crates/core/src/commands/click.rs +++ b/crates/core/src/commands/click.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct ClickArgs { - pub ref_id: String, -} - -pub fn execute(args: ClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Click)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::Click)) } diff --git a/crates/core/src/commands/collapse.rs b/crates/core/src/commands/collapse.rs index 19d181c..5413026 100644 --- a/crates/core/src/commands/collapse.rs +++ b/crates/core/src/commands/collapse.rs @@ -1,13 +1,11 @@ use crate::{ - action::Action, + action::{Action, ActionRequest}, adapter::PlatformAdapter, - commands::helpers::{resolve_ref, RefArgs}, + commands::helpers::{execute_ref_action, RefArgs}, error::AppError, }; use serde_json::Value; pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Collapse)?; - Ok(serde_json::to_value(result)?) + execute_ref_action(args, adapter, ActionRequest::headless(Action::Collapse)) } diff --git a/crates/core/src/commands/double_click.rs b/crates/core/src/commands/double_click.rs index 50c80a5..7e61e2d 100644 --- a/crates/core/src/commands/double_click.rs +++ b/crates/core/src/commands/double_click.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct DoubleClickArgs { - pub ref_id: String, -} - -pub fn execute(args: DoubleClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::DoubleClick)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::DoubleClick)) } diff --git a/crates/core/src/commands/drag.rs b/crates/core/src/commands/drag.rs index edd8446..5a0e680 100644 --- a/crates/core/src/commands/drag.rs +++ b/crates/core/src/commands/drag.rs @@ -11,12 +11,25 @@ pub struct DragArgs { pub from_xy: Option<(f64, f64)>, pub to_ref: Option, pub to_xy: Option<(f64, f64)>, + pub snapshot_id: Option, pub duration_ms: Option, } pub fn execute(args: DragArgs, adapter: &dyn PlatformAdapter) -> Result { - let from = resolve_point(&args.from_ref, args.from_xy, "from", adapter)?; - let to = resolve_point(&args.to_ref, args.to_xy, "to", adapter)?; + let from = resolve_point( + &args.from_ref, + args.from_xy, + "from", + args.snapshot_id.as_deref(), + adapter, + )?; + let to = resolve_point( + &args.to_ref, + args.to_xy, + "to", + args.snapshot_id.as_deref(), + adapter, + )?; let params = DragParams { from: from.clone(), to: to.clone(), @@ -34,12 +47,13 @@ fn resolve_point( ref_id: &Option, xy: Option<(f64, f64)>, label: &str, + snapshot_id: Option<&str>, adapter: &dyn PlatformAdapter, ) -> Result { if let Some(ref_id) = ref_id { - let (_entry, handle) = resolve_ref(ref_id, adapter)?; + let (_entry, handle) = resolve_ref(ref_id, snapshot_id, adapter)?; let bounds = adapter - .get_element_bounds(&handle)? + .get_element_bounds(handle.handle())? .ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?; Ok(Point { x: bounds.x + bounds.width / 2.0, diff --git a/crates/core/src/commands/expand.rs b/crates/core/src/commands/expand.rs index 37cdaf7..0f6bd5b 100644 --- a/crates/core/src/commands/expand.rs +++ b/crates/core/src/commands/expand.rs @@ -1,13 +1,11 @@ use crate::{ - action::Action, + action::{Action, ActionRequest}, adapter::PlatformAdapter, - commands::helpers::{resolve_ref, RefArgs}, + commands::helpers::{execute_ref_action, RefArgs}, error::AppError, }; use serde_json::Value; pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Expand)?; - Ok(serde_json::to_value(result)?) + execute_ref_action(args, adapter, ActionRequest::headless(Action::Expand)) } diff --git a/crates/core/src/commands/find.rs b/crates/core/src/commands/find.rs index 93fd275..97eb266 100644 --- a/crates/core/src/commands/find.rs +++ b/crates/core/src/commands/find.rs @@ -1,6 +1,11 @@ -use crate::{adapter::PlatformAdapter, error::AppError, node::AccessibilityNode, snapshot}; +use crate::{ + adapter::PlatformAdapter, commands::search_text, error::AppError, node::AccessibilityNode, + snapshot, +}; use serde_json::{json, Value}; +const DEFAULT_LIMIT: usize = 50; + pub struct FindArgs { pub app: Option, pub role: Option, @@ -11,19 +16,29 @@ pub struct FindArgs { pub first: bool, pub last: bool, pub nth: Option, + pub limit: Option, } pub fn execute(args: FindArgs, adapter: &dyn PlatformAdapter) -> Result { + validate_find_mode(&args)?; let opts = crate::adapter::TreeOptions::default(); let result = snapshot::run(adapter, &opts, args.app.as_deref(), None)?; - - let mut matches = Vec::new(); - search_tree(&result.tree, &args, &mut Vec::new(), &mut matches); + let query = FindQuery::from_args(&args); if args.count { - return Ok(json!({ "count": matches.len() })); + return Ok(json!({ "count": count_matches(&result.tree, &query) })); } + let mut matches = Vec::new(); + let max_matches = max_matches_for_args(&args); + search_tree( + &result.tree, + &query, + &mut Vec::new(), + &mut matches, + max_matches, + ); + if args.first { return Ok(json!({ "match": matches.into_iter().next() })); } @@ -39,45 +54,70 @@ pub fn execute(args: FindArgs, adapter: &dyn PlatformAdapter) -> Result Option { + if args.count || args.last { + return None; + } + if args.first { + return Some(1); + } + if let Some(n) = args.nth { + return Some(n.saturating_add(1)); + } + match args.limit.unwrap_or(DEFAULT_LIMIT) { + 0 => None, + limit => Some(limit), + } +} + +fn validate_find_mode(args: &FindArgs) -> Result<(), AppError> { + let selector_count = [args.count, args.first, args.last, args.nth.is_some()] + .into_iter() + .filter(|selected| *selected) + .count(); + if selector_count > 1 || (selector_count == 1 && args.limit.is_some()) { + return Err(AppError::invalid_input_with_suggestion( + "find accepts only one result-shaping mode", + "Use one of --count, --first, --last, --nth, or --limit.", + )); + } + Ok(()) +} + +struct FindQuery<'a> { + role: Option<&'a str>, + name: Option, + value: Option, + text: Option, +} + +impl<'a> FindQuery<'a> { + fn from_args(args: &'a FindArgs) -> Self { + Self { + role: args.role.as_deref(), + name: args.name.as_deref().map(search_text::normalize), + value: args.value.as_deref().map(search_text::normalize), + text: args.text.as_deref().map(search_text::normalize), + } + } +} + fn search_tree( node: &AccessibilityNode, - args: &FindArgs, + query: &FindQuery, path: &mut Vec, matches: &mut Vec, -) { - let role_match = args.role.as_deref().is_none_or(|r| node.role == r); - let name_match = args.name.as_deref().is_none_or(|n| { - node.name - .as_deref() - .is_some_and(|name| name.to_lowercase().contains(&n.to_lowercase())) - }); - let value_match = args.value.as_deref().is_none_or(|v| { - node.value - .as_deref() - .is_some_and(|val| val.to_lowercase().contains(&v.to_lowercase())) - }); - let text_match = args.text.as_deref().is_none_or(|t| { - let t_lower = t.to_lowercase(); - let in_name = node - .name - .as_deref() - .is_some_and(|n| n.to_lowercase().contains(&t_lower)); - let in_value = node - .value - .as_deref() - .is_some_and(|v| v.to_lowercase().contains(&t_lower)); - let in_desc = node - .description - .as_deref() - .is_some_and(|d| d.to_lowercase().contains(&t_lower)); - in_name || in_value || in_desc - }); - - if role_match && name_match && value_match && text_match { + max_matches: Option, +) -> bool { + if max_matches.is_some_and(|limit| matches.len() >= limit) { + return true; + } + if node_matches(node, query) { let interactive = node.ref_id.is_some(); let display_name = node .name .as_deref() + .or(node.value.as_deref()) .or(node.description.as_deref()) .map(String::from) .unwrap_or_else(|| format!("(unnamed {})", node.role)); @@ -89,18 +129,194 @@ fn search_tree( "interactive": interactive, "path": path.clone() })); + if max_matches.is_some_and(|limit| matches.len() >= limit) { + return true; + } } - let label = if let Some(name) = &node.name { - format!("{}:{}", node.role, name) - } else { - node.role.clone() - }; + let label = node + .name + .as_deref() + .or(node.value.as_deref()) + .map(|label| format!("{}:{label}", node.role)) + .unwrap_or_else(|| node.role.clone()); path.push(label); for child in &node.children { - search_tree(child, args, path, matches); + if search_tree(child, query, path, matches, max_matches) { + path.pop(); + return true; + } } path.pop(); + false +} + +fn count_matches(node: &AccessibilityNode, query: &FindQuery) -> usize { + usize::from(node_matches(node, query)) + + node + .children + .iter() + .map(|child| count_matches(child, query)) + .sum::() +} + +fn node_matches(node: &AccessibilityNode, query: &FindQuery) -> bool { + let role_match = query.role.is_none_or(|r| node.role == r); + let name_match = query.name.as_deref().is_none_or(|n| { + node.name + .as_deref() + .is_some_and(|text| search_text::contains(text, n)) + }); + let value_match = query.value.as_deref().is_none_or(|v| { + node.value + .as_deref() + .is_some_and(|val| search_text::contains(val, v)) + }); + let text_match = query + .text + .as_deref() + .is_none_or(|t| search_text::node_contains(node, t)); + role_match && name_match && value_match && text_match +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node( + name: Option<&str>, + value: Option<&str>, + description: Option<&str>, + ) -> AccessibilityNode { + AccessibilityNode { + ref_id: Some("@e1".into()), + role: "textfield".into(), + name: name.map(String::from), + value: value.map(String::from), + description: description.map(String::from), + hint: None, + states: vec![], + available_actions: vec![], + bounds: None, + children_count: None, + children: vec![], + } + } + + #[test] + fn display_name_prefers_value_before_description() { + let root = node(None, Some("current value"), Some("help text")); + let query = FindQuery { + role: None, + name: None, + value: None, + text: None, + }; + let mut matches = Vec::new(); + + search_tree(&root, &query, &mut Vec::new(), &mut matches, None); + + assert_eq!(matches[0]["name"], "current value"); + } + + #[test] + fn search_tree_matches_text_across_fields() { + let root = node(None, Some("Primary"), Some("Secondary")); + let query = FindQuery { + role: None, + name: None, + value: None, + text: Some(search_text::normalize("secondary")), + }; + let mut matches = Vec::new(); + + search_tree(&root, &query, &mut Vec::new(), &mut matches, None); + + assert_eq!(matches.len(), 1); + } + + #[test] + fn default_limit_caps_materialized_matches() { + let root = AccessibilityNode { + ref_id: None, + role: "window".into(), + name: None, + value: None, + description: None, + hint: None, + states: vec![], + available_actions: vec![], + bounds: None, + children_count: None, + children: (0..60) + .map(|i| node(Some(&format!("Button {i}")), None, None)) + .collect(), + }; + let query = FindQuery { + role: None, + name: None, + value: None, + text: Some(search_text::normalize("button")), + }; + let mut matches = Vec::new(); + + search_tree( + &root, + &query, + &mut Vec::new(), + &mut matches, + Some(DEFAULT_LIMIT), + ); + + assert_eq!(matches.len(), DEFAULT_LIMIT); + } + + #[test] + fn limit_conflicts_with_single_result_modes_for_batch_too() { + let err = validate_find_mode(&FindArgs { + app: None, + role: None, + name: None, + value: None, + text: None, + count: false, + first: true, + last: false, + nth: None, + limit: Some(10), + }) + .unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + } + + #[test] + fn count_matches_does_not_build_result_json() { + let root = AccessibilityNode { + ref_id: None, + role: "window".into(), + name: None, + value: None, + description: None, + hint: None, + states: vec![], + available_actions: vec![], + bounds: None, + children_count: None, + children: vec![ + node(Some("Save"), None, None), + node(Some("Cancel"), None, None), + ], + }; + let query = FindQuery { + role: None, + name: None, + value: None, + text: Some(search_text::normalize("a")), + }; + + assert_eq!(count_matches(&root, &query), 2); + } } diff --git a/crates/core/src/commands/focus.rs b/crates/core/src/commands/focus.rs index db8587f..b8c4f9e 100644 --- a/crates/core/src/commands/focus.rs +++ b/crates/core/src/commands/focus.rs @@ -1,13 +1,11 @@ use crate::{ - action::Action, + action::{Action, ActionRequest}, adapter::PlatformAdapter, - commands::helpers::{resolve_ref, RefArgs}, + commands::helpers::{execute_ref_action, RefArgs}, error::AppError, }; use serde_json::Value; pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::SetFocus)?; - Ok(serde_json::to_value(result)?) + execute_ref_action(args, adapter, ActionRequest::headless(Action::SetFocus)) } diff --git a/crates/core/src/commands/focus_window.rs b/crates/core/src/commands/focus_window.rs index 72f87e7..1ba4cb1 100644 --- a/crates/core/src/commands/focus_window.rs +++ b/crates/core/src/commands/focus_window.rs @@ -1,8 +1,16 @@ use crate::{ adapter::{PlatformAdapter, WindowFilter}, - error::AppError, + error::{AdapterError, AppError, ErrorCode}, + node::WindowInfo, }; use serde_json::{json, Value}; +use std::time::{Duration, Instant}; + +#[cfg(not(test))] +const FOCUS_SETTLE_TIMEOUT_MS: u64 = 750; +#[cfg(test)] +const FOCUS_SETTLE_TIMEOUT_MS: u64 = 250; +const FOCUS_CONFIRMATIONS: u8 = 2; pub struct FocusWindowArgs { pub window_id: Option, @@ -43,6 +51,211 @@ pub fn execute(args: FocusWindowArgs, adapter: &dyn PlatformAdapter) -> Result, +) -> Result { + let deadline = Instant::now() + Duration::from_millis(FOCUS_SETTLE_TIMEOUT_MS); + let mut confirmations = 0; + loop { + match observed_focused_window(adapter, app.as_deref())? { + Some(window) if window.id == window_id => { + confirmations += 1; + if confirmations >= FOCUS_CONFIRMATIONS { + return Ok(window); + } + } + _ => { + confirmations = 0; + } + } + + if Instant::now() >= deadline { + return Err(AppError::Adapter( + AdapterError::new( + ErrorCode::ActionFailed, + "Window focus did not settle on the requested window", + ) + .with_suggestion("Run 'list-windows' to refresh window IDs, then retry."), + )); + } + + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn observed_focused_window( + adapter: &dyn PlatformAdapter, + app: Option<&str>, +) -> Result, AppError> { + match adapter.focused_window() { + Ok(window) => Ok(window), + Err(err) if err.code == ErrorCode::PlatformNotSupported => adapter + .list_windows(&WindowFilter { + focused_only: true, + app: app.map(str::to_string), + }) + .map(|windows| windows.into_iter().next()) + .map_err(AppError::Adapter), + Err(err) => Err(AppError::Adapter(err)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + struct FocusAdapter { + windows: Vec, + focused_windows: Mutex>, + focused_window_calls: Mutex, + focused_window_supported: bool, + } + + impl PlatformAdapter for FocusAdapter { + fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { + if filter.focused_only { + Ok(self.focused_windows.lock().unwrap().clone()) + } else { + Ok(self.windows.clone()) + } + } + + fn focus_window(&self, _win: &WindowInfo) -> Result<(), AdapterError> { + Ok(()) + } + + fn focused_window(&self) -> Result, AdapterError> { + *self.focused_window_calls.lock().unwrap() += 1; + if !self.focused_window_supported { + return Err(AdapterError::not_supported("focused_window")); + } + let mut focused = self.focused_windows.lock().unwrap(); + if focused.len() > 1 { + Ok(Some(focused.remove(0))) + } else { + Ok(focused.first().cloned()) + } + } + } + + fn window(id: &str, focused: bool) -> WindowInfo { + WindowInfo { + id: id.into(), + title: "Main".into(), + app: "TextEdit".into(), + pid: 42, + bounds: None, + is_focused: focused, + } + } + + #[test] + fn reports_focused_window_after_os_confirms_focus() { + let target = window("w1", false); + let adapter = FocusAdapter { + windows: vec![target.clone()], + focused_windows: Mutex::new(vec![window("w1", true)]), + focused_window_calls: Mutex::new(0), + focused_window_supported: true, + }; + + let value = execute( + FocusWindowArgs { + window_id: Some(target.id), + app: None, + title: None, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(value["focused"]["id"], "w1"); + assert_eq!(value["focused"]["is_focused"], true); + assert_eq!(*adapter.focused_window_calls.lock().unwrap(), 2); + } + + #[test] + fn errors_when_focus_does_not_settle_on_requested_window() { + let target = window("w1", false); + let adapter = FocusAdapter { + windows: vec![target.clone()], + focused_windows: Mutex::new(Vec::new()), + focused_window_calls: Mutex::new(0), + focused_window_supported: true, + }; + + let err = execute( + FocusWindowArgs { + window_id: Some(target.id), + app: None, + title: None, + }, + &adapter, + ) + .unwrap_err(); + + assert_eq!(err.code(), "ACTION_FAILED"); + } + + #[test] + fn falls_back_to_focused_window_list_when_direct_observation_is_unsupported() { + let target = window("w1", false); + let adapter = FocusAdapter { + windows: vec![target.clone()], + focused_windows: Mutex::new(vec![window("w1", true)]), + focused_window_calls: Mutex::new(0), + focused_window_supported: false, + }; + + let value = execute( + FocusWindowArgs { + window_id: Some(target.id), + app: None, + title: None, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(value["focused"]["id"], "w1"); + assert_eq!(*adapter.focused_window_calls.lock().unwrap(), 2); + } + + #[test] + fn focus_confirmation_resets_after_transient_wrong_window() { + let target = window("w1", false); + let adapter = FocusAdapter { + windows: vec![target.clone()], + focused_windows: Mutex::new(vec![ + window("w1", true), + window("w2", true), + window("w1", true), + window("w1", true), + ]), + focused_window_calls: Mutex::new(0), + focused_window_supported: true, + }; + + let value = execute( + FocusWindowArgs { + window_id: Some(target.id), + app: None, + title: None, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(value["focused"]["id"], "w1"); + assert_eq!(*adapter.focused_window_calls.lock().unwrap(), 4); + } } diff --git a/crates/core/src/commands/get.rs b/crates/core/src/commands/get.rs index ee09410..956470c 100644 --- a/crates/core/src/commands/get.rs +++ b/crates/core/src/commands/get.rs @@ -3,6 +3,7 @@ use serde_json::{json, Value}; pub struct GetArgs { pub ref_id: String, + pub snapshot_id: Option, pub property: GetProperty, } @@ -16,13 +17,13 @@ pub enum GetProperty { } pub fn execute(args: GetArgs, adapter: &dyn PlatformAdapter) -> Result { - let (entry, handle) = resolve_ref(&args.ref_id, adapter)?; + let (entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; let value = match args.property { GetProperty::Role => json!(entry.role), GetProperty::Title => json!(entry.name), GetProperty::Text | GetProperty::Value => { - let live = adapter.get_live_value(&handle).ok().flatten(); + let live = adapter.get_live_value(handle.handle()).ok().flatten(); json!(live.or(entry.value)) } GetProperty::Bounds => json!(entry.bounds), diff --git a/crates/core/src/commands/helpers.rs b/crates/core/src/commands/helpers.rs index e30fb56..d68a257 100644 --- a/crates/core/src/commands/helpers.rs +++ b/crates/core/src/commands/helpers.rs @@ -1,19 +1,32 @@ use crate::{ - adapter::{NativeHandle, PlatformAdapter, WindowFilter}, + action::{ActionRequest, WindowOp}, + adapter::{PlatformAdapter, WindowFilter}, + commands::resolved_element::ResolvedElement, error::AppError, - refs::{RefEntry, RefMap}, + node::WindowInfo, + refs::RefEntry, + refs_store::RefStore, + window_lookup, }; +use serde_json::{json, Value}; + +pub struct AppArgs { + pub app: Option, +} pub struct RefArgs { pub ref_id: String, + pub snapshot_id: Option, } -pub fn resolve_ref( +pub fn resolve_ref<'a>( ref_id: &str, - adapter: &dyn PlatformAdapter, -) -> Result<(RefEntry, NativeHandle), AppError> { + snapshot_id: Option<&str>, + adapter: &'a dyn PlatformAdapter, +) -> Result<(RefEntry, ResolvedElement<'a>), AppError> { validate_ref_id(ref_id)?; - let refmap = RefMap::load().map_err(|e| { + let store = RefStore::new()?; + let refmap = store.load(snapshot_id).map_err(|e| { tracing::debug!("refmap load failed: {e}"); AppError::stale_ref(ref_id) })?; @@ -30,14 +43,15 @@ pub fn resolve_ref( ); let handle = adapter.resolve_element(&entry)?; tracing::debug!("resolve: {} resolved successfully", ref_id); - Ok((entry, handle)) + Ok((entry, ResolvedElement::new(adapter, handle))) } pub fn validate_ref_id(ref_id: &str) -> Result<(), AppError> { let valid = ref_id.starts_with("@e") && ref_id.len() >= 3 && ref_id.len() <= 12 - && ref_id[2..].chars().all(|c| c.is_ascii_digit()); + && ref_id[2..].chars().all(|c| c.is_ascii_digit()) + && ref_id[2..].parse::().is_ok_and(|n| n > 0); if !valid { return Err(AppError::invalid_input(format!( "Invalid ref_id '{ref_id}': must match @e{{N}} where N is a positive integer" @@ -66,9 +80,204 @@ pub fn resolve_app_pid(app: Option<&str>, adapter: &dyn PlatformAdapter) -> Resu } } +pub fn execute_ref_action( + args: RefArgs, + adapter: &dyn PlatformAdapter, + request: ActionRequest, +) -> Result { + let (_entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let result = adapter.execute_action(handle.handle(), request)?; + Ok(serde_json::to_value(result)?) +} + +pub fn window_op_command( + args: AppArgs, + adapter: &dyn PlatformAdapter, + op: WindowOp, + response_key: &'static str, +) -> Result { + let pid = resolve_app_pid(args.app.as_deref(), adapter)?; + let win = match find_window_for_pid(pid, adapter) { + Ok(win) => win, + Err(_) if matches!(op, WindowOp::Restore) => WindowInfo { + id: String::new(), + title: String::new(), + app: args.app.unwrap_or_default(), + pid, + bounds: None, + is_focused: false, + }, + Err(err) => return Err(err), + }; + adapter.window_op(&win, op)?; + Ok(json!({ response_key: true })) +} + +pub fn find_window_for_pid( + pid: i32, + adapter: &dyn PlatformAdapter, +) -> Result { + window_lookup::find_window_for_pid(pid, adapter) +} + +pub fn resolve_window_for_app( + app: Option<&str>, + adapter: &dyn PlatformAdapter, +) -> Result { + let pid = resolve_app_pid(app, adapter)?; + find_window_for_pid(pid, adapter) +} + #[cfg(test)] mod tests { use super::*; + use crate::action::{Action, ActionResult, InteractionPolicy}; + use crate::adapter::NativeHandle; + use crate::error::{AdapterError, ErrorCode}; + use crate::node::AppInfo; + use crate::refs::RefMap; + use crate::refs_test_support::HomeGuard; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Mutex; + + struct ReleaseCountingAdapter { + releases: AtomicU32, + } + + impl PlatformAdapter for ReleaseCountingAdapter { + fn resolve_element(&self, _entry: &RefEntry) -> Result { + Ok(NativeHandle::null()) + } + + fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> { + self.releases.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + struct RecordingAdapter { + request: Mutex>, + } + + impl PlatformAdapter for RecordingAdapter { + fn resolve_element(&self, _entry: &RefEntry) -> Result { + Ok(NativeHandle::null()) + } + + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + ) -> Result { + *self.request.lock().unwrap() = Some(request); + Ok(ActionResult::new("ok")) + } + } + + struct RestoreWithoutWindowAdapter { + op_count: AtomicU32, + } + + impl PlatformAdapter for RestoreWithoutWindowAdapter { + fn list_apps(&self) -> Result, AdapterError> { + Ok(vec![AppInfo { + name: "TextEdit".into(), + pid: 42, + bundle_id: None, + }]) + } + + fn list_windows( + &self, + _filter: &crate::adapter::WindowFilter, + ) -> Result, AdapterError> { + Err(AdapterError::new(ErrorCode::WindowNotFound, "no windows")) + } + + fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { + assert_eq!(win.pid, 42); + assert!(matches!(op, WindowOp::Restore)); + self.op_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + fn entry() -> RefEntry { + RefEntry { + pid: 1, + role: "button".into(), + name: Some("OK".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + } + } + + #[test] + fn resolved_element_releases_handle_once_on_drop() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = ReleaseCountingAdapter { + releases: AtomicU32::new(0), + }; + + { + let (_entry, resolved) = resolve_ref("@e1", Some(&snapshot_id), &adapter).unwrap(); + let _handle = resolved.handle(); + assert_eq!(adapter.releases.load(Ordering::SeqCst), 0); + } + + assert_eq!(adapter.releases.load(Ordering::SeqCst), 1); + } + + #[test] + fn execute_ref_action_preserves_action_and_policy() { + let _guard = HomeGuard::new(); + let mut refmap = RefMap::new(); + refmap.allocate(entry()); + let snapshot_id = RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap(); + let adapter = RecordingAdapter { + request: Mutex::new(None), + }; + let args = RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + }; + + execute_ref_action(args, &adapter, ActionRequest::headless(Action::Clear)).unwrap(); + + let request = adapter.request.lock().unwrap().clone().unwrap(); + assert!(matches!(request.action, Action::Clear)); + assert_eq!(request.policy, InteractionPolicy::headless()); + } + + #[test] + fn restore_can_run_when_no_window_is_currently_listed() { + let adapter = RestoreWithoutWindowAdapter { + op_count: AtomicU32::new(0), + }; + + let value = window_op_command( + AppArgs { + app: Some("TextEdit".into()), + }, + &adapter, + WindowOp::Restore, + "restored", + ) + .unwrap(); + + assert_eq!(value["restored"], true); + assert_eq!(adapter.op_count.load(Ordering::SeqCst), 1); + } #[test] fn test_valid_refs() { @@ -82,6 +291,7 @@ mod tests { assert!(validate_ref_id("@").is_err()); assert!(validate_ref_id("e1").is_err()); assert!(validate_ref_id("@e").is_err()); + assert!(validate_ref_id("@e0").is_err()); assert!(validate_ref_id("@e0abc").is_err()); assert!(validate_ref_id("1").is_err()); assert!(validate_ref_id("").is_err()); diff --git a/crates/core/src/commands/hover.rs b/crates/core/src/commands/hover.rs index d4b0a90..a5097f6 100644 --- a/crates/core/src/commands/hover.rs +++ b/crates/core/src/commands/hover.rs @@ -8,6 +8,7 @@ use serde_json::{json, Value}; pub struct HoverArgs { pub ref_id: Option, + pub snapshot_id: Option, pub xy: Option<(f64, f64)>, pub duration_ms: Option, } @@ -27,9 +28,9 @@ pub fn execute(args: HoverArgs, adapter: &dyn PlatformAdapter) -> Result Result { if let Some(ref_id) = &args.ref_id { - let (_entry, handle) = resolve_ref(ref_id, adapter)?; + let (_entry, handle) = resolve_ref(ref_id, args.snapshot_id.as_deref(), adapter)?; let bounds = adapter - .get_element_bounds(&handle)? + .get_element_bounds(handle.handle())? .ok_or_else(|| AppError::invalid_input(format!("Element {ref_id} has no bounds")))?; Ok(Point { x: bounds.x + bounds.width / 2.0, diff --git a/crates/core/src/commands/is_check.rs b/crates/core/src/commands/is_check.rs index ae46db3..13a0481 100644 --- a/crates/core/src/commands/is_check.rs +++ b/crates/core/src/commands/is_check.rs @@ -1,8 +1,12 @@ -use crate::{adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError}; +use crate::{ + action::ElementState, adapter::PlatformAdapter, commands::helpers::resolve_ref, + error::AppError, refs::RefEntry, +}; use serde_json::{json, Value}; pub struct IsArgs { pub ref_id: String, + pub snapshot_id: Option, pub property: IsProperty, } @@ -14,11 +18,14 @@ pub enum IsProperty { Expanded, } -/// States are read from the last snapshot's RefMap. `resolve_ref` verifies the element -/// is still live before returning, but the state values themselves are not re-queried -/// from the AX API. Run `snapshot` to refresh state before calling `is`. +/// State is read live when the platform supports it, then falls back to snapshot state. pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result { - let (entry, _handle) = resolve_ref(&args.ref_id, adapter)?; + let (entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let state = adapter + .get_live_state(handle.handle()) + .ok() + .flatten() + .unwrap_or_else(|| state_from_ref_entry(&entry)); let prop_name = match args.property { IsProperty::Visible => "visible", @@ -28,14 +35,14 @@ pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result "expanded", }; - let applicable = is_applicable(&args.property, &entry.role); + let applicable = is_applicable(&args.property, &entry, &state); let result = match args.property { - IsProperty::Visible => !entry.states.contains(&"hidden".to_string()), - IsProperty::Enabled => !entry.states.contains(&"disabled".to_string()), - IsProperty::Checked => entry.states.contains(&"checked".to_string()), - IsProperty::Focused => entry.states.contains(&"focused".to_string()), - IsProperty::Expanded => entry.states.contains(&"expanded".to_string()), + IsProperty::Visible => !has_state(&state, "hidden"), + IsProperty::Enabled => !has_state(&state, "disabled"), + IsProperty::Checked => has_state(&state, "checked"), + IsProperty::Focused => has_state(&state, "focused"), + IsProperty::Expanded => has_state(&state, "expanded"), }; Ok( @@ -43,21 +50,226 @@ pub fn execute(args: IsArgs, adapter: &dyn PlatformAdapter) -> Result bool { - match property { - IsProperty::Visible | IsProperty::Enabled | IsProperty::Focused => true, - IsProperty::Checked => matches!( - role, - "checkbox" - | "switch" - | "radiobutton" - | "togglebutton" - | "menuitemcheckbox" - | "menuitemradio" - ), - IsProperty::Expanded => matches!( - role, - "disclosuretriangle" | "treeitem" | "combobox" | "popupbutton" | "outline" | "row" - ), +fn state_from_ref_entry(entry: &RefEntry) -> ElementState { + ElementState { + role: entry.role.clone(), + states: entry.states.clone(), + value: entry.value.clone(), + } +} + +fn has_state(state: &ElementState, name: &str) -> bool { + state.states.iter().any(|s| s == name) +} + +fn is_applicable(property: &IsProperty, entry: &RefEntry, state: &ElementState) -> bool { + match property { + IsProperty::Visible | IsProperty::Enabled | IsProperty::Focused => true, + IsProperty::Checked => { + crate::roles::is_toggleable_role(&entry.role) + || has_state(state, "checked") + || has_available_action(entry, "Toggle") + || has_available_action(entry, "Check") + || has_available_action(entry, "Uncheck") + } + IsProperty::Expanded => { + crate::roles::is_expandable_role(&entry.role) + || has_state(state, "expanded") + || has_available_action(entry, "Expand") + || has_available_action(entry, "Collapse") + } + } +} + +fn has_available_action(entry: &RefEntry, action: &str) -> bool { + entry.available_actions.iter().any(|a| a == action) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + adapter::NativeHandle, error::AdapterError, refs::RefMap, refs_store::RefStore, + refs_test_support::HomeGuard, + }; + use std::sync::Mutex; + + struct LiveStateAdapter { + state: Mutex>, + } + + impl PlatformAdapter for LiveStateAdapter { + fn resolve_element(&self, _entry: &RefEntry) -> Result { + Ok(NativeHandle::null()) + } + + fn get_live_state( + &self, + _handle: &NativeHandle, + ) -> Result, AdapterError> { + Ok(self.state.lock().unwrap().clone()) + } + } + + fn save_entry(entry: RefEntry) -> String { + let mut refmap = RefMap::new(); + refmap.allocate(entry); + RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() + } + + fn entry(states: Vec, value: Option<&str>, actions: Vec<&str>) -> RefEntry { + RefEntry { + pid: 1, + role: "checkbox".into(), + name: Some("Target".into()), + value: value.map(str::to_string), + states, + bounds: None, + bounds_hash: None, + available_actions: actions.into_iter().map(str::to_string).collect(), + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + } + } + + #[test] + fn checked_uses_live_canonical_state() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec!["Toggle"])); + let adapter = LiveStateAdapter { + state: Mutex::new(Some(ElementState { + role: "checkbox".into(), + states: vec!["checked".into()], + value: Some("1".into()), + })), + }; + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Checked, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(result["result"], true); + assert_eq!(result["applicable"], true); + } + + #[test] + fn checked_does_not_infer_platform_values_in_core() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], Some("1"), vec!["Toggle"])); + let adapter = LiveStateAdapter { + state: Mutex::new(None), + }; + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Checked, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(result["result"], false); + assert_eq!(result["applicable"], true); + } + + #[test] + fn checked_falls_back_to_snapshot_state_when_live_state_is_missing() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec!["checked".into()], None, vec!["Toggle"])); + let adapter = LiveStateAdapter { + state: Mutex::new(None), + }; + + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + property: IsProperty::Checked, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(result["result"], true); + assert_eq!(result["applicable"], true); + } + + #[test] + fn basic_state_properties_use_live_state() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(entry(vec![], None, vec![])); + let adapter = LiveStateAdapter { + state: Mutex::new(Some(ElementState { + role: "button".into(), + states: vec!["focused".into(), "expanded".into()], + value: None, + })), + }; + + for (property, expected) in [ + (IsProperty::Visible, true), + (IsProperty::Enabled, true), + (IsProperty::Focused, true), + (IsProperty::Expanded, true), + ] { + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + property, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(result["result"], expected); + assert_eq!(result["applicable"], true); + } + } + + #[test] + fn action_availability_makes_toggle_and_expand_applicable() { + let _guard = HomeGuard::new(); + let snapshot_id = save_entry(RefEntry { + pid: 1, + role: "cell".into(), + name: Some("Disclosure".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Check".into(), "Expand".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + let adapter = LiveStateAdapter { + state: Mutex::new(None), + }; + + for property in [IsProperty::Checked, IsProperty::Expanded] { + let result = execute( + IsArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + property, + }, + &adapter, + ) + .unwrap(); + + assert_eq!(result["applicable"], true); + } } } diff --git a/crates/core/src/commands/key_down.rs b/crates/core/src/commands/key_down.rs index 743884c..b3f192e 100644 --- a/crates/core/src/commands/key_down.rs +++ b/crates/core/src/commands/key_down.rs @@ -1,6 +1,4 @@ -use crate::{ - action::Action, adapter::PlatformAdapter, commands::press::parse_combo, error::AppError, -}; +use crate::{adapter::PlatformAdapter, commands::press::parse_combo, error::AppError}; use serde_json::{json, Value}; pub struct KeyDownArgs { @@ -9,7 +7,6 @@ pub struct KeyDownArgs { pub fn execute(args: KeyDownArgs, adapter: &dyn PlatformAdapter) -> Result { let combo = parse_combo(&args.combo)?; - let handle = crate::adapter::NativeHandle::null(); - adapter.execute_action(&handle, Action::KeyDown(combo))?; + adapter.key_event(&combo, true)?; Ok(json!({ "key_down": args.combo })) } diff --git a/crates/core/src/commands/key_up.rs b/crates/core/src/commands/key_up.rs index df8b88d..f8d42ed 100644 --- a/crates/core/src/commands/key_up.rs +++ b/crates/core/src/commands/key_up.rs @@ -1,6 +1,4 @@ -use crate::{ - action::Action, adapter::PlatformAdapter, commands::press::parse_combo, error::AppError, -}; +use crate::{adapter::PlatformAdapter, commands::press::parse_combo, error::AppError}; use serde_json::{json, Value}; pub struct KeyUpArgs { @@ -9,7 +7,6 @@ pub struct KeyUpArgs { pub fn execute(args: KeyUpArgs, adapter: &dyn PlatformAdapter) -> Result { let combo = parse_combo(&args.combo)?; - let handle = crate::adapter::NativeHandle::null(); - adapter.execute_action(&handle, Action::KeyUp(combo))?; + adapter.key_event(&combo, false)?; Ok(json!({ "key_up": args.combo })) } diff --git a/crates/core/src/commands/list_apps.rs b/crates/core/src/commands/list_apps.rs index 0e18005..1fe997d 100644 --- a/crates/core/src/commands/list_apps.rs +++ b/crates/core/src/commands/list_apps.rs @@ -1,7 +1,55 @@ -use crate::{adapter::PlatformAdapter, error::AppError}; +use crate::{adapter::PlatformAdapter, commands::search_text, error::AppError}; use serde_json::{json, Value}; -pub fn execute(adapter: &dyn PlatformAdapter) -> Result { - let apps = adapter.list_apps()?; +pub struct ListAppsArgs { + pub app: Option, +} + +pub fn execute(args: ListAppsArgs, adapter: &dyn PlatformAdapter) -> Result { + let mut apps = adapter.list_apps()?; + if let Some(app) = args.app { + let needle = search_text::normalize(&app); + apps.retain(|candidate| search_text::contains(&candidate.name, &needle)); + } Ok(json!({ "apps": apps })) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{adapter::PlatformAdapter, error::AdapterError, node::AppInfo}; + + struct AppsAdapter; + + impl PlatformAdapter for AppsAdapter { + fn list_apps(&self) -> Result, AdapterError> { + Ok(vec![ + AppInfo { + name: "Finder".into(), + pid: 1, + bundle_id: Some("com.apple.finder".into()), + }, + AppInfo { + name: "TextEdit".into(), + pid: 2, + bundle_id: Some("com.apple.TextEdit".into()), + }, + ]) + } + } + + #[test] + fn app_filter_matches_by_name_case_insensitively() { + let value = execute( + ListAppsArgs { + app: Some("text".into()), + }, + &AppsAdapter, + ) + .unwrap(); + + let apps = value["apps"].as_array().unwrap(); + assert_eq!(apps.len(), 1); + assert_eq!(apps[0]["name"], "TextEdit"); + } +} diff --git a/crates/core/src/commands/maximize.rs b/crates/core/src/commands/maximize.rs index 09d9b74..774fd35 100644 --- a/crates/core/src/commands/maximize.rs +++ b/crates/core/src/commands/maximize.rs @@ -1,30 +1,11 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError, + action::WindowOp, + adapter::PlatformAdapter, + commands::helpers::{window_op_command, AppArgs}, + error::AppError, }; -use serde_json::{json, Value}; +use serde_json::Value; -pub struct MaximizeArgs { - pub app: Option, -} - -pub fn execute(args: MaximizeArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = find_window(pid, adapter)?; - adapter.window_op(&win, WindowOp::Maximize)?; - Ok(json!({ "maximized": true })) -} - -fn find_window( - pid: i32, - adapter: &dyn PlatformAdapter, -) -> Result { - let filter = crate::adapter::WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .into_iter() - .find(|w| w.pid == pid) - .ok_or_else(|| AppError::invalid_input("No window found for this application")) +pub fn execute(args: AppArgs, adapter: &dyn PlatformAdapter) -> Result { + window_op_command(args, adapter, WindowOp::Maximize, "maximized") } diff --git a/crates/core/src/commands/minimize.rs b/crates/core/src/commands/minimize.rs index d081c1a..b67d374 100644 --- a/crates/core/src/commands/minimize.rs +++ b/crates/core/src/commands/minimize.rs @@ -1,30 +1,11 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError, + action::WindowOp, + adapter::PlatformAdapter, + commands::helpers::{window_op_command, AppArgs}, + error::AppError, }; -use serde_json::{json, Value}; +use serde_json::Value; -pub struct MinimizeArgs { - pub app: Option, -} - -pub fn execute(args: MinimizeArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = find_window(pid, adapter)?; - adapter.window_op(&win, WindowOp::Minimize)?; - Ok(json!({ "minimized": true })) -} - -fn find_window( - pid: i32, - adapter: &dyn PlatformAdapter, -) -> Result { - let filter = crate::adapter::WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .into_iter() - .find(|w| w.pid == pid) - .ok_or_else(|| AppError::invalid_input("No window found for this application")) +pub fn execute(args: AppArgs, adapter: &dyn PlatformAdapter) -> Result { + window_op_command(args, adapter, WindowOp::Minimize, "minimized") } diff --git a/crates/core/src/commands/mod.rs b/crates/core/src/commands/mod.rs index 239e33a..6dfc37f 100644 --- a/crates/core/src/commands/mod.rs +++ b/crates/core/src/commands/mod.rs @@ -37,11 +37,13 @@ pub mod notification_action; pub mod permissions; pub mod press; pub mod resize_window; +pub mod resolved_element; pub mod restore; pub mod right_click; pub mod screenshot; pub mod scroll; pub mod scroll_to; +pub(crate) mod search_text; pub mod select; pub mod set_value; pub mod skills; @@ -53,3 +55,6 @@ pub mod type_text; pub mod uncheck; pub mod version; pub mod wait; + +#[cfg(test)] +mod ref_policy_tests; diff --git a/crates/core/src/commands/move_window.rs b/crates/core/src/commands/move_window.rs index ec6d1fd..612b220 100644 --- a/crates/core/src/commands/move_window.rs +++ b/crates/core/src/commands/move_window.rs @@ -1,5 +1,6 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError, + action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_window_for_app, + error::AppError, }; use serde_json::{json, Value}; @@ -10,8 +11,7 @@ pub struct MoveWindowArgs { } pub fn execute(args: MoveWindowArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = find_window(pid, adapter)?; + let win = resolve_window_for_app(args.app.as_deref(), adapter)?; adapter.window_op( &win, WindowOp::Move { @@ -21,18 +21,3 @@ pub fn execute(args: MoveWindowArgs, adapter: &dyn PlatformAdapter) -> Result Result { - let filter = crate::adapter::WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .into_iter() - .find(|w| w.pid == pid) - .ok_or_else(|| AppError::invalid_input("No window found for this application")) -} diff --git a/crates/core/src/commands/permissions.rs b/crates/core/src/commands/permissions.rs index 8b2c06c..a48c79c 100644 --- a/crates/core/src/commands/permissions.rs +++ b/crates/core/src/commands/permissions.rs @@ -1,26 +1,27 @@ -use crate::{ - adapter::{PermissionStatus, PlatformAdapter}, - error::AppError, -}; +use crate::{adapter::PlatformAdapter, error::AppError, PermissionReport}; use serde_json::{json, Value}; pub struct PermissionsArgs { pub request: bool, } -pub fn execute(args: PermissionsArgs, adapter: &dyn PlatformAdapter) -> Result { - if args.request { - return Ok(json!({ - "requested": true, - "note": "Permission dialog triggered via --request flag" - })); - } - - match adapter.check_permissions() { - PermissionStatus::Granted => Ok(json!({ "granted": true })), - PermissionStatus::Denied { suggestion } => Ok(json!({ - "granted": false, - "suggestion": suggestion - })), - } +pub fn execute_with_report( + args: PermissionsArgs, + adapter: &dyn PlatformAdapter, + report: &PermissionReport, +) -> Result { + let report = if args.request { + adapter.request_permissions() + } else { + report.clone() + }; + Ok(render(report)) +} + +fn render(report: PermissionReport) -> Value { + json!({ + "accessibility": report.accessibility, + "screen_recording": report.screen_recording, + "automation": report.automation + }) } diff --git a/crates/core/src/commands/press.rs b/crates/core/src/commands/press.rs index 6368d1f..f3fdca5 100644 --- a/crates/core/src/commands/press.rs +++ b/crates/core/src/commands/press.rs @@ -1,5 +1,5 @@ use crate::{ - action::{Action, KeyCombo, Modifier}, + action::{Action, ActionRequest, KeyCombo, Modifier}, adapter::PlatformAdapter, error::AppError, }; @@ -35,7 +35,8 @@ pub fn execute(args: PressArgs, adapter: &dyn PlatformAdapter) -> Result>, +} + +impl RecordingAdapter { + fn new() -> Self { + Self { + requests: Mutex::new(Vec::new()), + } + } + + fn last_request(&self) -> ActionRequest { + self.requests.lock().unwrap().last().cloned().unwrap() + } +} + +impl PlatformAdapter for RecordingAdapter { + fn resolve_element(&self, _entry: &RefEntry) -> Result { + Ok(NativeHandle::null()) + } + + fn execute_action( + &self, + _handle: &NativeHandle, + request: ActionRequest, + ) -> Result { + self.requests.lock().unwrap().push(request); + Ok(ActionResult::new("ok")) + } +} + +fn snapshot_id() -> String { + let mut refmap = RefMap::new(); + refmap.allocate(RefEntry { + pid: 1, + role: "textfield".into(), + name: Some("Target".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() +} + +fn ref_args(snapshot_id: &str) -> RefArgs { + RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.to_owned()), + } +} + +fn assert_headless(request: &ActionRequest) { + assert_eq!(request.policy, InteractionPolicy::headless()); +} + +#[test] +fn default_ref_commands_are_headless() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = RecordingAdapter::new(); + + click::execute(ref_args(&snapshot_id), &adapter).unwrap(); + double_click::execute(ref_args(&snapshot_id), &adapter).unwrap(); + triple_click::execute(ref_args(&snapshot_id), &adapter).unwrap(); + let before_right_click = adapter.requests.lock().unwrap().len(); + let _ = right_click::execute(ref_args(&snapshot_id), &adapter); + assert_eq!( + adapter.requests.lock().unwrap().len(), + before_right_click + 1 + ); + clear::execute(ref_args(&snapshot_id), &adapter).unwrap(); + toggle::execute(ref_args(&snapshot_id), &adapter).unwrap(); + check::execute(ref_args(&snapshot_id), &adapter).unwrap(); + uncheck::execute(ref_args(&snapshot_id), &adapter).unwrap(); + expand::execute(ref_args(&snapshot_id), &adapter).unwrap(); + collapse::execute(ref_args(&snapshot_id), &adapter).unwrap(); + scroll_to::execute(ref_args(&snapshot_id), &adapter).unwrap(); + set_value::execute( + set_value::SetValueArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + value: "value".into(), + }, + &adapter, + ) + .unwrap(); + select::execute( + select::SelectArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + value: "choice".into(), + }, + &adapter, + ) + .unwrap(); + let before_type = adapter.requests.lock().unwrap().len(); + type_text::execute( + type_text::TypeArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id.clone()), + text: "text".into(), + }, + &adapter, + ) + .unwrap(); + let type_request = adapter.requests.lock().unwrap()[before_type].clone(); + assert_eq!(type_request.policy, InteractionPolicy::focus_fallback()); + scroll::execute( + scroll::ScrollArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + direction: Direction::Down, + amount: 1, + }, + &adapter, + ) + .unwrap(); + + for request in adapter.requests.lock().unwrap().iter() { + if matches!(request.action, Action::TypeText(_)) { + continue; + } + assert_headless(request); + } +} + +#[test] +fn focus_command_is_explicit_headless_policy() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_id(); + let adapter = RecordingAdapter::new(); + + focus::execute(ref_args(&snapshot_id), &adapter).unwrap(); + + let request = adapter.last_request(); + assert!(matches!(request.action, Action::SetFocus)); + assert_eq!(request.policy, InteractionPolicy::headless()); +} diff --git a/crates/core/src/commands/resize_window.rs b/crates/core/src/commands/resize_window.rs index 343ce3a..067d0f3 100644 --- a/crates/core/src/commands/resize_window.rs +++ b/crates/core/src/commands/resize_window.rs @@ -1,5 +1,6 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError, + action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_window_for_app, + error::AppError, }; use serde_json::{json, Value}; @@ -10,8 +11,7 @@ pub struct ResizeWindowArgs { } pub fn execute(args: ResizeWindowArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = find_window(pid, adapter)?; + let win = resolve_window_for_app(args.app.as_deref(), adapter)?; adapter.window_op( &win, WindowOp::Resize { @@ -21,18 +21,3 @@ pub fn execute(args: ResizeWindowArgs, adapter: &dyn PlatformAdapter) -> Result< )?; Ok(json!({ "resized": true, "width": args.width, "height": args.height })) } - -fn find_window( - pid: i32, - adapter: &dyn PlatformAdapter, -) -> Result { - let filter = crate::adapter::WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .into_iter() - .find(|w| w.pid == pid) - .ok_or_else(|| AppError::invalid_input("No window found for this application")) -} diff --git a/crates/core/src/commands/resolved_element.rs b/crates/core/src/commands/resolved_element.rs new file mode 100644 index 0000000..60ecb96 --- /dev/null +++ b/crates/core/src/commands/resolved_element.rs @@ -0,0 +1,22 @@ +use crate::adapter::{NativeHandle, PlatformAdapter}; + +pub struct ResolvedElement<'a> { + adapter: &'a dyn PlatformAdapter, + handle: NativeHandle, +} + +impl<'a> ResolvedElement<'a> { + pub fn new(adapter: &'a dyn PlatformAdapter, handle: NativeHandle) -> Self { + Self { adapter, handle } + } + + pub fn handle(&self) -> &NativeHandle { + &self.handle + } +} + +impl Drop for ResolvedElement<'_> { + fn drop(&mut self) { + let _ = self.adapter.release_handle(&self.handle); + } +} diff --git a/crates/core/src/commands/restore.rs b/crates/core/src/commands/restore.rs index 5fee550..fccafc5 100644 --- a/crates/core/src/commands/restore.rs +++ b/crates/core/src/commands/restore.rs @@ -1,30 +1,11 @@ use crate::{ - action::WindowOp, adapter::PlatformAdapter, commands::helpers::resolve_app_pid, error::AppError, + action::WindowOp, + adapter::PlatformAdapter, + commands::helpers::{window_op_command, AppArgs}, + error::AppError, }; -use serde_json::{json, Value}; +use serde_json::Value; -pub struct RestoreArgs { - pub app: Option, -} - -pub fn execute(args: RestoreArgs, adapter: &dyn PlatformAdapter) -> Result { - let pid = resolve_app_pid(args.app.as_deref(), adapter)?; - let win = find_window(pid, adapter)?; - adapter.window_op(&win, WindowOp::Restore)?; - Ok(json!({ "restored": true })) -} - -fn find_window( - pid: i32, - adapter: &dyn PlatformAdapter, -) -> Result { - let filter = crate::adapter::WindowFilter { - focused_only: false, - app: None, - }; - let windows = adapter.list_windows(&filter)?; - windows - .into_iter() - .find(|w| w.pid == pid) - .ok_or_else(|| AppError::invalid_input("No window found for this application")) +pub fn execute(args: AppArgs, adapter: &dyn PlatformAdapter) -> Result { + window_op_command(args, adapter, WindowOp::Restore, "restored") } diff --git a/crates/core/src/commands/right_click.rs b/crates/core/src/commands/right_click.rs index e1e8b5c..50ad158 100644 --- a/crates/core/src/commands/right_click.rs +++ b/crates/core/src/commands/right_click.rs @@ -1,48 +1,231 @@ use crate::{ - action::Action, - adapter::{PlatformAdapter, TreeOptions}, - commands::helpers::resolve_ref, + action::{Action, ActionRequest}, + adapter::{PlatformAdapter, SnapshotSurface, TreeOptions, WindowFilter}, + commands::helpers::{resolve_ref, RefArgs}, error::AppError, - node::AccessibilityNode, + refs::RefEntry, snapshot, }; -use serde_json::Value; +use serde_json::{json, Value}; -pub struct RightClickArgs { - pub ref_id: String, -} - -pub fn execute(args: RightClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::RightClick)?; +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + let (entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let result = + adapter.execute_action(handle.handle(), ActionRequest::headless(Action::RightClick))?; let mut response = serde_json::to_value(&result)?; std::thread::sleep(std::time::Duration::from_millis(200)); let opts = TreeOptions { interactive_only: true, + surface: SnapshotSurface::Menu, ..Default::default() }; - if let Ok(snap) = snapshot::build(adapter, &opts, entry.source_app.as_deref(), None) { - snap.refmap.save().ok(); - if let Some(menu) = find_context_menu(&snap.tree) { - if let Ok(menu_json) = serde_json::to_value(menu) { + let probe_app = probe_app_name(adapter, &entry); + match snapshot::run(adapter, &opts, probe_app.as_deref(), None) { + Ok(snap) => match serde_json::to_value(&snap.tree) { + Ok(menu_json) => { response["menu"] = menu_json; + if let Some(snapshot_id) = snap.snapshot_id { + response["menu_snapshot_id"] = json!(snapshot_id); + } } - } + Err(err) => { + response["menu_probe"] = json!({ + "ok": false, + "error": { + "code": "INTERNAL", + "message": err.to_string(), + } + }) + } + }, + Err(err) => response["menu_probe"] = probe_error_json(&err), } Ok(response) } -fn find_context_menu(node: &AccessibilityNode) -> Option<&AccessibilityNode> { - if node.role == "menu" && node.children.iter().any(|c| c.role == "menuitem") { - return Some(node); +fn probe_app_name(adapter: &dyn PlatformAdapter, entry: &RefEntry) -> Option { + if entry.source_app.is_some() { + return entry.source_app.clone(); } - for child in &node.children { - if let Some(menu) = find_context_menu(child) { - return Some(menu); + adapter + .list_windows(&WindowFilter { + focused_only: false, + app: None, + }) + .ok() + .and_then(|windows| { + windows + .into_iter() + .find(|window| window.pid == entry.pid) + .map(|window| window.app) + }) +} + +fn probe_error_json(err: &AppError) -> Value { + if err.code() == "ELEMENT_NOT_FOUND" { + return json!({ + "ok": false, + "error": { + "code": "ELEMENT_NOT_FOUND", + "message": "Right-click action was accepted, but no menu accessibility tree was exposed for capture.", + "suggestion": "Use 'snapshot --surface menu' only when the app exposes the context menu through accessibility." + } + }); + } + + let mut error = json!({ + "code": err.code(), + "message": err.to_string(), + }); + if let Some(suggestion) = err.suggestion() { + error["suggestion"] = json!(suggestion); + } + json!({ + "ok": false, + "error": error, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + action::ActionResult, + adapter::NativeHandle, + error::{AdapterError, ErrorCode}, + node::WindowInfo, + refs::{RefEntry, RefMap}, + refs_store::RefStore, + refs_test_support::HomeGuard, + }; + + struct ProbeFailingAdapter { + tree_error: Option, + } + + impl PlatformAdapter for ProbeFailingAdapter { + fn resolve_element(&self, _entry: &RefEntry) -> Result { + Ok(NativeHandle::null()) + } + + fn execute_action( + &self, + _handle: &NativeHandle, + _request: ActionRequest, + ) -> Result { + Ok(ActionResult::new("right_click")) + } + + fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { + if filter.app.is_some() && self.tree_error.is_none() { + return Err(AdapterError::new( + ErrorCode::WindowNotFound, + "menu probe failed", + )); + } + if filter.focused_only { + return Err(AdapterError::new( + ErrorCode::WindowNotFound, + "no focused menu", + )); + } + Ok(vec![WindowInfo { + id: "w1".into(), + title: "Main".into(), + app: "TargetApp".into(), + pid: 7, + bounds: None, + is_focused: true, + }]) + } + + fn get_tree( + &self, + _win: &WindowInfo, + _opts: &TreeOptions, + ) -> Result { + if let Some(code) = self.tree_error.clone() { + return Err(AdapterError::new(code, "menu tree unavailable")); + } + Ok(crate::node::AccessibilityNode { + ref_id: None, + role: "menu".into(), + name: None, + value: None, + description: None, + hint: None, + states: Vec::new(), + available_actions: Vec::new(), + bounds: None, + children_count: None, + children: Vec::new(), + }) } } - None + + fn save_refmap(source_app: Option) -> String { + let mut refmap = RefMap::new(); + refmap.allocate(RefEntry { + pid: 7, + role: "button".into(), + name: Some("Open".into()), + value: None, + states: Vec::new(), + bounds: None, + bounds_hash: None, + available_actions: vec!["RightClick".into()], + source_app, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() + } + + #[test] + fn returns_action_success_when_menu_probe_fails() { + let _guard = HomeGuard::new(); + let snapshot_id = save_refmap(None); + + let value = execute( + RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + }, + &ProbeFailingAdapter { tree_error: None }, + ) + .unwrap(); + + assert_eq!(value["action"], "right_click"); + assert_eq!(value["menu_probe"]["ok"], false); + assert_eq!(value["menu_probe"]["error"]["code"], "WINDOW_NOT_FOUND"); + } + + #[test] + fn element_not_found_menu_probe_uses_right_click_specific_guidance() { + let _guard = HomeGuard::new(); + let snapshot_id = save_refmap(Some("TargetApp".into())); + + let value = execute( + RefArgs { + ref_id: "@e1".into(), + snapshot_id: Some(snapshot_id), + }, + &ProbeFailingAdapter { + tree_error: Some(ErrorCode::ElementNotFound), + }, + ) + .unwrap(); + + assert_eq!(value["action"], "right_click"); + assert_eq!(value["menu_probe"]["ok"], false); + assert_eq!(value["menu_probe"]["error"]["code"], "ELEMENT_NOT_FOUND"); + assert!(value["menu_probe"]["error"]["suggestion"] + .as_str() + .unwrap() + .contains("snapshot --surface menu")); + } } diff --git a/crates/core/src/commands/scroll.rs b/crates/core/src/commands/scroll.rs index f7147b1..5c65ef9 100644 --- a/crates/core/src/commands/scroll.rs +++ b/crates/core/src/commands/scroll.rs @@ -1,5 +1,5 @@ use crate::{ - action::{Action, Direction}, + action::{Action, ActionRequest, Direction}, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, @@ -8,12 +8,16 @@ use serde_json::Value; pub struct ScrollArgs { pub ref_id: String, + pub snapshot_id: Option, pub direction: Direction, pub amount: u32, } pub fn execute(args: ScrollArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Scroll(args.direction, args.amount))?; + let (_entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let result = adapter.execute_action( + handle.handle(), + ActionRequest::headless(Action::Scroll(args.direction, args.amount)), + )?; Ok(serde_json::to_value(result)?) } diff --git a/crates/core/src/commands/scroll_to.rs b/crates/core/src/commands/scroll_to.rs index a401048..91133d3 100644 --- a/crates/core/src/commands/scroll_to.rs +++ b/crates/core/src/commands/scroll_to.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct ScrollToArgs { - pub ref_id: String, -} - -pub fn execute(args: ScrollToArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::ScrollTo)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::ScrollTo)) } diff --git a/crates/core/src/commands/search_text.rs b/crates/core/src/commands/search_text.rs new file mode 100644 index 0000000..ee5f957 --- /dev/null +++ b/crates/core/src/commands/search_text.rs @@ -0,0 +1,31 @@ +pub(crate) fn normalize(value: &str) -> String { + if value.is_ascii() { + value.to_ascii_lowercase() + } else { + value.to_lowercase() + } +} + +pub(crate) fn contains(haystack: &str, normalized_needle: &str) -> bool { + if haystack.is_ascii() && normalized_needle.is_ascii() { + return haystack + .as_bytes() + .windows(normalized_needle.len()) + .any(|chunk| chunk.eq_ignore_ascii_case(normalized_needle.as_bytes())); + } + normalize(haystack).contains(normalized_needle) +} + +pub(crate) fn node_contains( + node: &crate::node::AccessibilityNode, + normalized_needle: &str, +) -> bool { + [ + node.name.as_deref(), + node.value.as_deref(), + node.description.as_deref(), + ] + .into_iter() + .flatten() + .any(|text| contains(text, normalized_needle)) +} diff --git a/crates/core/src/commands/select.rs b/crates/core/src/commands/select.rs index c52bf6f..6765c66 100644 --- a/crates/core/src/commands/select.rs +++ b/crates/core/src/commands/select.rs @@ -1,15 +1,22 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::resolve_ref, + error::AppError, }; use serde_json::Value; pub struct SelectArgs { pub ref_id: String, + pub snapshot_id: Option, pub value: String, } pub fn execute(args: SelectArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Select(args.value))?; + let (_entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let result = adapter.execute_action( + handle.handle(), + ActionRequest::headless(Action::Select(args.value)), + )?; Ok(serde_json::to_value(result)?) } diff --git a/crates/core/src/commands/set_value.rs b/crates/core/src/commands/set_value.rs index a45efcb..4937512 100644 --- a/crates/core/src/commands/set_value.rs +++ b/crates/core/src/commands/set_value.rs @@ -1,15 +1,22 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::resolve_ref, + error::AppError, }; use serde_json::Value; pub struct SetValueArgs { pub ref_id: String, + pub snapshot_id: Option, pub value: String, } pub fn execute(args: SetValueArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::SetValue(args.value))?; + let (_entry, handle) = resolve_ref(&args.ref_id, args.snapshot_id.as_deref(), adapter)?; + let result = adapter.execute_action( + handle.handle(), + ActionRequest::headless(Action::SetValue(args.value)), + )?; Ok(serde_json::to_value(result)?) } diff --git a/crates/core/src/commands/skills.rs b/crates/core/src/commands/skills.rs index e0f30fb..41ff01f 100644 --- a/crates/core/src/commands/skills.rs +++ b/crates/core/src/commands/skills.rs @@ -40,7 +40,7 @@ const SKILLS: &[Skill] = &[ Skill { canonical: "agent-desktop", aliases: &["desktop", "agent-desktop"], - summary: "Primary guide. Snapshot/ref loop, JSON envelope, 53 commands across observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.", + summary: "Primary guide. Snapshot/ref loop, JSON envelope, 54 commands across observation, interaction, keyboard/mouse, app lifecycle, notifications, clipboard, wait.", main: SKILL_DESKTOP_MAIN, refs: &[ SkillRef { rel_path: "references/commands-observation.md", body: SKILL_DESKTOP_REF_OBSERVATION }, diff --git a/crates/core/src/commands/snapshot.rs b/crates/core/src/commands/snapshot.rs index 3970df8..63be22b 100644 --- a/crates/core/src/commands/snapshot.rs +++ b/crates/core/src/commands/snapshot.rs @@ -16,6 +16,7 @@ pub struct SnapshotArgs { pub surface: SnapshotSurface, pub skeleton: bool, pub root_ref: Option, + pub snapshot_id: Option, } fn tree_options(args: &SnapshotArgs) -> crate::adapter::TreeOptions { @@ -55,7 +56,12 @@ pub fn execute(args: SnapshotArgs, adapter: &dyn PlatformAdapter) -> Result Result { "title": win.title }, "ref_count": ref_count, + "snapshot_id": result.snapshot_id, "tree": tree })) } @@ -110,6 +117,7 @@ mod tests { surface: SnapshotSurface::Window, skeleton: false, root_ref: None, + snapshot_id: None, } } diff --git a/crates/core/src/commands/status.rs b/crates/core/src/commands/status.rs index dc434c6..f812fd4 100644 --- a/crates/core/src/commands/status.rs +++ b/crates/core/src/commands/status.rs @@ -1,25 +1,74 @@ use crate::{ - adapter::{PermissionStatus, PlatformAdapter}, + adapter::PlatformAdapter, + commands::permissions::{self, PermissionsArgs}, error::AppError, - refs::RefMap, + refs_store::RefStore, + PermissionReport, }; use serde_json::{json, Value}; pub fn execute(adapter: &dyn PlatformAdapter) -> Result { - let permissions = match adapter.check_permissions() { - PermissionStatus::Granted => json!({ "granted": true }), - PermissionStatus::Denied { suggestion } => json!({ - "granted": false, - "suggestion": suggestion - }), - }; + let report = adapter.permission_report(); + execute_with_report(adapter, &report) +} - let ref_count = RefMap::load().ok().map(|m| m.len()); +pub fn execute_with_report( + adapter: &dyn PlatformAdapter, + report: &PermissionReport, +) -> Result { + let permissions = + permissions::execute_with_report(PermissionsArgs { request: false }, adapter, report)?; + + let store = RefStore::new().ok(); + let ref_count = store + .as_ref() + .and_then(|s| s.load_latest().ok()) + .map(|m| m.len()); + let snapshot_id = store.and_then(|s| s.latest_snapshot_id()); Ok(json!({ "platform": std::env::consts::OS, "version": env!("CARGO_PKG_VERSION"), "permissions": permissions, + "snapshot_id": snapshot_id, "ref_count": ref_count })) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::PermissionState; + + struct DeniedAdapter; + + impl PlatformAdapter for DeniedAdapter { + fn permission_report(&self) -> PermissionReport { + PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "should not be used".into(), + }, + screen_recording: PermissionState::Denied { + suggestion: "should not be used".into(), + }, + automation: PermissionState::Unknown, + } + } + } + + #[test] + fn status_uses_precomputed_permission_report() { + let report = PermissionReport { + accessibility: PermissionState::Granted, + screen_recording: PermissionState::Granted, + automation: PermissionState::NotRequired, + }; + + let value = execute_with_report(&DeniedAdapter, &report).unwrap(); + let permissions = value.get("permissions").unwrap(); + + assert_eq!(permissions["accessibility"]["state"], "granted"); + assert_eq!(permissions["screen_recording"]["state"], "granted"); + assert_eq!(permissions["automation"]["state"], "not_required"); + } +} diff --git a/crates/core/src/commands/toggle.rs b/crates/core/src/commands/toggle.rs index 0d317af..dd2d5ba 100644 --- a/crates/core/src/commands/toggle.rs +++ b/crates/core/src/commands/toggle.rs @@ -1,13 +1,11 @@ use crate::{ - action::Action, + action::{Action, ActionRequest}, adapter::PlatformAdapter, - commands::helpers::{resolve_ref, RefArgs}, + commands::helpers::{execute_ref_action, RefArgs}, error::AppError, }; use serde_json::Value; pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Toggle)?; - Ok(serde_json::to_value(result)?) + execute_ref_action(args, adapter, ActionRequest::headless(Action::Toggle)) } diff --git a/crates/core/src/commands/triple_click.rs b/crates/core/src/commands/triple_click.rs index 55e2f7a..b40a727 100644 --- a/crates/core/src/commands/triple_click.rs +++ b/crates/core/src/commands/triple_click.rs @@ -1,14 +1,11 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::{execute_ref_action, RefArgs}, + error::AppError, }; use serde_json::Value; -pub struct TripleClickArgs { - pub ref_id: String, -} - -pub fn execute(args: TripleClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::TripleClick)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::TripleClick)) } diff --git a/crates/core/src/commands/type_text.rs b/crates/core/src/commands/type_text.rs index f06974c..a605fe3 100644 --- a/crates/core/src/commands/type_text.rs +++ b/crates/core/src/commands/type_text.rs @@ -1,5 +1,8 @@ use crate::{ - action::Action, adapter::PlatformAdapter, commands::helpers::resolve_ref, error::AppError, + action::{Action, ActionRequest}, + adapter::PlatformAdapter, + commands::helpers::resolve_ref, + error::AppError, }; use serde_json::Value; @@ -7,6 +10,7 @@ const MAX_TEXT_LEN: usize = 10_000; pub struct TypeArgs { pub ref_id: String, + pub snapshot_id: Option, pub text: String, } @@ -17,8 +21,10 @@ pub fn execute(args: TypeArgs, adapter: &dyn PlatformAdapter) -> Result Result { - let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; - let result = adapter.execute_action(&handle, Action::Uncheck)?; - Ok(serde_json::to_value(result)?) +pub fn execute(args: RefArgs, adapter: &dyn PlatformAdapter) -> Result { + execute_ref_action(args, adapter, ActionRequest::headless(Action::Uncheck)) } diff --git a/crates/core/src/commands/wait.rs b/crates/core/src/commands/wait.rs index 816b9d2..3025f73 100644 --- a/crates/core/src/commands/wait.rs +++ b/crates/core/src/commands/wait.rs @@ -1,10 +1,11 @@ use crate::{ adapter::{PlatformAdapter, WindowFilter}, - commands::helpers::{resolve_app_pid, validate_ref_id}, - error::AppError, + commands::{helpers::resolve_app_pid, helpers::validate_ref_id, search_text}, + error::{AppError, ErrorCode}, node::AccessibilityNode, notification::NotificationFilter, refs::RefMap, + refs_store::RefStore, snapshot, }; use serde_json::{json, Value}; @@ -13,6 +14,7 @@ use std::time::{Duration, Instant}; pub struct WaitArgs { pub ms: Option, pub element: Option, + pub snapshot_id: Option, pub window: Option, pub text: Option, pub timeout_ms: u64, @@ -23,6 +25,8 @@ pub struct WaitArgs { } pub fn execute(args: WaitArgs, adapter: &dyn PlatformAdapter) -> Result { + validate_wait_mode(&args)?; + if let Some(ms) = args.ms { std::thread::sleep(Duration::from_millis(ms)); return Ok(json!({ "waited_ms": ms })); @@ -44,7 +48,7 @@ pub fn execute(args: WaitArgs, adapter: &dyn PlatformAdapter) -> Result Result Result<(), AppError> { + let selected = [ + args.ms.is_some(), + args.element.is_some(), + args.window.is_some(), + args.text.is_some() && !args.notification, + args.menu, + args.menu_closed, + args.notification, + ] + .into_iter() + .filter(|selected| *selected) + .count(); + if selected <= 1 { + return Ok(()); + } + Err(AppError::invalid_input_with_suggestion( + "wait accepts exactly one mode", + "Use one of: ms, --element, --window, --text, --menu, --menu-closed, or --notification.", + )) +} + fn wait_for_element( ref_id: String, + snapshot_id: Option, timeout_ms: u64, adapter: &dyn PlatformAdapter, ) -> Result { let start = Instant::now(); let timeout = Duration::from_millis(timeout_ms); + let store = RefStore::new()?; + let fixed_refmap = match snapshot_id.as_deref() { + Some(id) => Some(store.load_snapshot(id)?), + None => None, + }; + let mut latest_cache = if fixed_refmap.is_none() { + Some(LatestRefCache::new(&store)?) + } else { + None + }; + + if fixed_refmap + .as_ref() + .is_some_and(|refmap| refmap.get(&ref_id).is_none()) + { + return Err(AppError::invalid_input_with_suggestion( + format!("Ref {ref_id} is not present in the requested snapshot"), + "Use a ref returned by that snapshot_id, or omit --snapshot to wait against the latest refmap.", + )); + } loop { - if let Ok(refmap) = RefMap::load() { - if let Some(entry) = refmap.get(&ref_id) { - if adapter.resolve_element(entry).is_ok() { + let entry = fixed_refmap + .as_ref() + .and_then(|r| r.get(&ref_id).cloned()) + .or_else(|| latest_cache.as_ref().and_then(|c| c.entry(&ref_id))); + if let Some(entry) = entry { + match adapter.resolve_element(&entry) { + Ok(handle) => { + let _ = adapter.release_handle(&handle); let elapsed = start.elapsed().as_millis(); return Ok(json!({ "found": true, "ref": ref_id, "elapsed_ms": elapsed })); } + Err(err) if fixed_refmap.is_none() && err.code == ErrorCode::StaleRef => { + if let Some(cache) = latest_cache.as_mut() { + cache.refresh_if_due(); + } + } + Err(_) => {} } + } else if let Some(cache) = latest_cache.as_mut() { + cache.refresh_if_due(); } - if start.elapsed() >= timeout { + let remaining = timeout.saturating_sub(start.elapsed()); + if remaining.is_zero() { return Err(AppError::Adapter(crate::error::AdapterError::timeout( format!("Element {ref_id} not found within {timeout_ms}ms"), ))); } + std::thread::sleep(remaining.min(Duration::from_millis(100))); + } +} - std::thread::sleep(Duration::from_millis(100)); +struct LatestRefCache<'a> { + store: &'a RefStore, + snapshot_id: Option, + refmap: RefMap, + last_refresh: Instant, +} + +impl<'a> LatestRefCache<'a> { + fn new(store: &'a RefStore) -> Result { + let snapshot_id = store.latest_snapshot_id(); + let refmap = if let Some(id) = snapshot_id.as_deref() { + store.load_snapshot(id)? + } else { + store.load_latest()? + }; + Ok(Self { + store, + snapshot_id, + refmap, + last_refresh: Instant::now() - Duration::from_millis(500), + }) + } + + fn entry(&self, ref_id: &str) -> Option { + self.refmap.get(ref_id).cloned() + } + + fn refresh_if_due(&mut self) { + if self.last_refresh.elapsed() < Duration::from_millis(500) { + return; + } + self.last_refresh = Instant::now(); + if let Some(snapshot_id) = self.store.latest_snapshot_id() { + if self.snapshot_id.as_deref() == Some(snapshot_id.as_str()) { + return; + } + if let Ok(refmap) = self.store.load_snapshot(&snapshot_id) { + self.snapshot_id = Some(snapshot_id); + self.refmap = refmap; + } + } else if let Ok(refmap) = self.store.load_latest() { + self.refmap = refmap; + self.snapshot_id = self.store.latest_snapshot_id(); + } } } @@ -127,17 +234,19 @@ fn wait_for_text( let start = Instant::now(); let timeout = Duration::from_millis(timeout_ms); let opts = crate::adapter::TreeOptions::default(); - let text_lower = text.to_lowercase(); + let normalized_text = search_text::normalize(&text); loop { - if let Ok(result) = snapshot::run(adapter, &opts, app.as_deref(), None) { - if let Some(found) = find_text_in_tree(&result.tree, &text_lower) { + if let Ok(result) = snapshot::build(adapter, &opts, app.as_deref(), None) { + if let Some(found) = find_text_in_tree(&result.tree, &normalized_text) { + let snapshot_id = RefStore::new()?.save_new_snapshot(&result.refmap)?; let elapsed = start.elapsed().as_millis(); return Ok(json!({ "found": true, "text": text, "ref": found.ref_id, "role": found.role, + "snapshot_id": snapshot_id, "elapsed_ms": elapsed })); } @@ -167,7 +276,9 @@ fn wait_for_notification( text: args.text.clone(), ..Default::default() }; - let baseline = adapter.list_notifications(&filter)?; + let baseline = adapter + .list_notifications(&filter) + .map_err(AppError::Adapter)?; let baseline_indices: std::collections::HashSet = baseline.iter().map(|n| n.index).collect(); let interval = Duration::from_millis(500); @@ -180,40 +291,28 @@ fn wait_for_notification( format!("No new notification within {}ms", args.timeout_ms), ))); } - let current = adapter.list_notifications(&filter)?; - let new_notif = current + let current = adapter + .list_notifications(&filter) + .map_err(AppError::Adapter)?; + let Some(notif) = current .iter() - .find(|n| !baseline_indices.contains(&n.index)); - if let Some(notif) = new_notif.or(current.last()) { - if current.len() > baseline_indices.len() { - let elapsed = start.elapsed().as_millis(); - return Ok(json!({ - "condition": "notification", - "matched": true, - "notification": notif, - "elapsed_ms": elapsed, - })); - } - } - std::thread::sleep(interval); + .find(|n| !baseline_indices.contains(&n.index)) + else { + std::thread::sleep(interval); + continue; + }; + let elapsed = start.elapsed().as_millis(); + return Ok(json!({ + "condition": "notification", + "matched": true, + "notification": notif, + "elapsed_ms": elapsed, + })); } } fn find_text_in_tree(node: &AccessibilityNode, text_lower: &str) -> Option { - let in_name = node - .name - .as_deref() - .is_some_and(|n| n.to_lowercase().contains(text_lower)); - let in_value = node - .value - .as_deref() - .is_some_and(|v| v.to_lowercase().contains(text_lower)); - let in_desc = node - .description - .as_deref() - .is_some_and(|d| d.to_lowercase().contains(text_lower)); - - if in_name || in_value || in_desc { + if search_text::node_contains(node, text_lower) { return Some(TextMatch { ref_id: node.ref_id.clone(), role: node.role.clone(), @@ -227,3 +326,7 @@ fn find_text_in_tree(node: &AccessibilityNode, text_lower: &str) -> Option Result, AdapterError> { + Err(AdapterError::new( + ErrorCode::PlatformNotSupported, + "notifications unavailable", + )) + } +} + +fn snapshot_with_one_ref() -> String { + let mut refmap = RefMap::new(); + refmap.allocate(RefEntry { + pid: 1, + role: "button".into(), + name: Some("Run".into()), + value: None, + states: Vec::new(), + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap() +} + +#[test] +fn snapshot_pinned_missing_ref_is_invalid_args() { + let _guard = HomeGuard::new(); + let snapshot_id = snapshot_with_one_ref(); + + let err = wait_for_element("@e2".into(), Some(snapshot_id), 1, &NoopAdapter).unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(err.suggestion().is_some()); +} + +#[test] +fn notification_wait_propagates_adapter_error() { + let err = execute( + WaitArgs { + ms: None, + element: None, + snapshot_id: None, + window: None, + text: None, + timeout_ms: 1, + menu: false, + menu_closed: false, + notification: true, + app: None, + }, + &NotificationErrorAdapter, + ) + .unwrap_err(); + + assert_eq!(err.code(), "PLATFORM_NOT_SUPPORTED"); +} + +#[test] +fn rejects_multiple_wait_modes() { + let err = execute( + WaitArgs { + ms: Some(1), + element: Some("@e1".into()), + snapshot_id: None, + window: None, + text: None, + timeout_ms: 1, + menu: false, + menu_closed: false, + notification: false, + app: None, + }, + &NoopAdapter, + ) + .unwrap_err(); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(err.suggestion().is_some()); +} + +#[test] +fn notification_wait_allows_text_filter() { + let result = validate_wait_mode(&WaitArgs { + ms: None, + element: None, + snapshot_id: None, + window: None, + text: Some("done".into()), + timeout_ms: 1, + menu: false, + menu_closed: false, + notification: true, + app: None, + }); + + assert!(result.is_ok()); +} + +#[test] +fn latest_ref_cache_picks_up_newer_snapshot_after_refresh() { + let _guard = HomeGuard::new(); + let _ = snapshot_with_one_ref(); + let store = RefStore::new().unwrap(); + let first_id = store.latest_snapshot_id().unwrap(); + + let mut cache = LatestRefCache::new(&store).unwrap(); + assert_eq!(cache.snapshot_id.as_deref(), Some(first_id.as_str())); + + let mut second = RefMap::new(); + second.allocate(RefEntry { + pid: 99, + role: "button".into(), + name: Some("Second".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + let second_id = store.save_new_snapshot(&second).unwrap(); + assert_ne!(first_id, second_id); + + cache.last_refresh = std::time::Instant::now() - std::time::Duration::from_secs(2); + cache.refresh_if_due(); + + assert_eq!(cache.snapshot_id.as_deref(), Some(second_id.as_str())); + assert!(cache.entry("@e1").is_some()); +} + +#[test] +fn latest_ref_cache_debounces_consecutive_refreshes() { + let _guard = HomeGuard::new(); + let _ = snapshot_with_one_ref(); + let store = RefStore::new().unwrap(); + let first_id = store.latest_snapshot_id().unwrap(); + + let mut cache = LatestRefCache::new(&store).unwrap(); + let pinned_snapshot_id = cache.snapshot_id.clone(); + let pinned_refresh = cache.last_refresh; + + let mut other = RefMap::new(); + other.allocate(RefEntry { + pid: 1, + role: "button".into(), + name: None, + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + }); + let _ = store.save_new_snapshot(&other).unwrap(); + + cache.last_refresh = std::time::Instant::now(); + cache.refresh_if_due(); + + assert_eq!(cache.snapshot_id, pinned_snapshot_id); + assert_eq!(cache.last_refresh, pinned_refresh.max(cache.last_refresh)); + let _ = first_id; +} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index e0655c9..056be0c 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -15,6 +15,8 @@ pub enum ErrorCode { Timeout, InvalidArgs, NotificationNotFound, + SnapshotNotFound, + PolicyDenied, Internal, } @@ -32,6 +34,8 @@ impl ErrorCode { ErrorCode::Timeout => "TIMEOUT", ErrorCode::InvalidArgs => "INVALID_ARGS", ErrorCode::NotificationNotFound => "NOTIFICATION_NOT_FOUND", + ErrorCode::SnapshotNotFound => "SNAPSHOT_NOT_FOUND", + ErrorCode::PolicyDenied => "POLICY_DENIED", ErrorCode::Internal => "INTERNAL", } } @@ -122,7 +126,21 @@ impl AdapterError { "Accessibility permission not granted", ) .with_suggestion( - "Open System Settings > Privacy & Security > Accessibility and add your terminal", + "Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop", + ) + } + + pub fn snapshot_not_found(snapshot_id: &str) -> Self { + Self::new( + ErrorCode::SnapshotNotFound, + format!("Snapshot '{snapshot_id}' not found"), + ) + .with_suggestion("Run 'snapshot' again and retry with the returned snapshot_id") + } + + pub fn policy_denied(message: impl Into) -> Self { + Self::new(ErrorCode::PolicyDenied, message).with_suggestion( + "Use an explicit mouse/focus command if physical interaction is intended", ) } } @@ -164,6 +182,15 @@ impl AppError { pub fn invalid_input(msg: impl Into) -> Self { AppError::Adapter(AdapterError::new(ErrorCode::InvalidArgs, msg)) } + + pub fn invalid_input_with_suggestion( + msg: impl Into, + suggestion: impl Into, + ) -> Self { + AppError::Adapter( + AdapterError::new(ErrorCode::InvalidArgs, msg).with_suggestion(suggestion), + ) + } } #[cfg(test)] diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index eaa9803..9970027 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -6,21 +6,32 @@ pub mod hints; pub mod node; pub mod notification; pub mod output; +pub mod permission_report; +pub mod permission_state; pub mod ref_alloc; pub mod refs; +mod refs_lock; +pub mod refs_store; +#[cfg(test)] +mod refs_test_support; +pub mod roles; pub mod snapshot; pub mod snapshot_ref; +mod window_lookup; pub use action::{ - Action, ActionResult, Direction, DragParams, ElementState, KeyCombo, Modifier, MouseButton, - MouseEvent, MouseEventKind, Point, WindowOp, + Action, ActionRequest, ActionResult, Direction, DragParams, ElementState, InteractionPolicy, + KeyCombo, Modifier, MouseButton, MouseEvent, MouseEventKind, Point, WindowOp, }; pub use adapter::{ - ImageBuffer, ImageFormat, NativeHandle, PermissionStatus, PlatformAdapter, ScreenshotTarget, - TreeOptions, WindowFilter, + ImageBuffer, ImageFormat, NativeHandle, PlatformAdapter, ScreenshotTarget, TreeOptions, + WindowFilter, }; pub use error::{AdapterError, AppError, ErrorCode}; pub use node::{AccessibilityNode, AppInfo, Rect, WindowInfo}; pub use notification::{NotificationFilter, NotificationInfo}; pub use output::{AppContext, ErrorPayload, Response, WindowContext}; +pub use permission_report::PermissionReport; +pub use permission_state::PermissionState; pub use refs::{RefEntry, RefMap}; +pub use refs_store::RefStore; diff --git a/crates/core/src/node.rs b/crates/core/src/node.rs index c64031b..2140c15 100644 --- a/crates/core/src/node.rs +++ b/crates/core/src/node.rs @@ -22,6 +22,9 @@ pub struct AccessibilityNode { #[serde(skip_serializing_if = "Vec::is_empty", default)] pub states: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub available_actions: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub bounds: Option, @@ -125,6 +128,7 @@ mod tests { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children_count: None, children: vec![], @@ -143,6 +147,7 @@ mod tests { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children_count: Some(47), children: vec![], diff --git a/crates/core/src/output.rs b/crates/core/src/output.rs index 3afae57..cd3ff1b 100644 --- a/crates/core/src/output.rs +++ b/crates/core/src/output.rs @@ -1,6 +1,8 @@ use serde::Serialize; use serde_json::Value; +pub const ENVELOPE_VERSION: &str = "2.0"; + /// Structured output envelope used by the Phase 3 MCP server transport layer. /// CLI commands currently build responses via inline `serde_json::json!` calls in `main.rs`; /// this type provides the typed equivalent for programmatic consumers. @@ -45,7 +47,7 @@ pub struct ErrorPayload { impl Response { pub fn ok(command: impl Into, data: Value) -> Self { Self { - version: "1.0", + version: ENVELOPE_VERSION, ok: true, command: command.into(), app: None, @@ -56,7 +58,7 @@ impl Response { pub fn err(command: impl Into, payload: ErrorPayload) -> Self { Self { - version: "1.0", + version: ENVELOPE_VERSION, ok: false, command: command.into(), app: None, diff --git a/crates/core/src/permission_report.rs b/crates/core/src/permission_report.rs new file mode 100644 index 0000000..8bfd75c --- /dev/null +++ b/crates/core/src/permission_report.rs @@ -0,0 +1,80 @@ +use crate::permission_state::PermissionState; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct PermissionReport { + pub accessibility: PermissionState, + pub screen_recording: PermissionState, + pub automation: PermissionState, +} + +impl PermissionReport { + pub fn accessibility_granted(&self) -> bool { + matches!(self.accessibility, PermissionState::Granted) + } + + pub fn screen_recording_granted(&self) -> bool { + matches!(self.screen_recording, PermissionState::Granted) + } + + pub fn accessibility_denied(&self) -> bool { + matches!(self.accessibility, PermissionState::Denied { .. }) + } + + pub fn screen_recording_denied(&self) -> bool { + matches!(self.screen_recording, PermissionState::Denied { .. }) + } + + pub fn accessibility_suggestion(&self) -> Option<&str> { + match &self.accessibility { + PermissionState::Denied { suggestion } => Some(suggestion.as_str()), + PermissionState::Granted | PermissionState::NotRequired | PermissionState::Unknown => { + None + } + } + } + + pub fn screen_recording_suggestion(&self) -> Option<&str> { + match &self.screen_recording { + PermissionState::Denied { suggestion } => Some(suggestion.as_str()), + PermissionState::Granted | PermissionState::NotRequired | PermissionState::Unknown => { + None + } + } + } +} + +impl Default for PermissionReport { + fn default() -> Self { + Self { + accessibility: PermissionState::Unknown, + screen_recording: PermissionState::Unknown, + automation: PermissionState::NotRequired, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn permission_state_serializes_with_stable_state_field() { + let report = PermissionReport { + accessibility: PermissionState::Granted, + screen_recording: PermissionState::Denied { + suggestion: "grant screen recording".into(), + }, + automation: PermissionState::NotRequired, + }; + + let value = serde_json::to_value(report).unwrap(); + + assert_eq!(value["accessibility"], json!({"state": "granted"})); + assert_eq!( + value["screen_recording"], + json!({"state": "denied", "suggestion": "grant screen recording"}) + ); + assert_eq!(value["automation"], json!({"state": "not_required"})); + } +} diff --git a/crates/core/src/permission_state.rs b/crates/core/src/permission_state.rs new file mode 100644 index 0000000..0b6bd6d --- /dev/null +++ b/crates/core/src/permission_state.rs @@ -0,0 +1,8 @@ +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case")] +pub enum PermissionState { + Granted, + Denied { suggestion: String }, + NotRequired, + Unknown, +} diff --git a/crates/core/src/ref_alloc.rs b/crates/core/src/ref_alloc.rs index 95e60e5..3844cd3 100644 --- a/crates/core/src/ref_alloc.rs +++ b/crates/core/src/ref_alloc.rs @@ -1,24 +1,7 @@ use crate::node::AccessibilityNode; use crate::refs::{RefEntry, RefMap}; -pub(crate) const INTERACTIVE_ROLES: &[&str] = &[ - "button", - "textfield", - "checkbox", - "link", - "menuitem", - "tab", - "slider", - "combobox", - "treeitem", - "cell", - "radiobutton", - "incrementor", - "menubutton", - "switch", - "colorwell", - "dockitem", -]; +pub(crate) use crate::roles::INTERACTIVE_ROLES; pub(crate) fn actions_for_role(role: &str) -> Vec { match role { @@ -37,7 +20,9 @@ pub(crate) fn ref_entry_from_node( node: &AccessibilityNode, pid: i32, source_app: Option<&str>, + source_window_title: Option<&str>, root_ref: Option, + path: &[usize], ) -> RefEntry { RefEntry { pid, @@ -47,9 +32,15 @@ pub(crate) fn ref_entry_from_node( states: node.states.clone(), bounds: node.bounds, bounds_hash: node.bounds.as_ref().map(|b| b.bounds_hash()), - available_actions: actions_for_role(&node.role), + available_actions: if node.available_actions.is_empty() { + actions_for_role(&node.role) + } else { + node.available_actions.clone() + }, source_app: source_app.map(str::to_string), + source_window_title: source_window_title.map(str::to_string), root_ref, + path: path.to_vec(), } } @@ -113,20 +104,36 @@ pub(crate) struct RefAllocConfig<'a> { pub compact: bool, pub pid: i32, pub source_app: Option<&'a str>, + pub source_window_title: Option<&'a str>, pub root_ref_id: Option<&'a str>, } pub(crate) fn allocate_refs( + node: AccessibilityNode, + refmap: &mut RefMap, + config: &RefAllocConfig, +) -> AccessibilityNode { + allocate_refs_at_path(node, refmap, config, &mut Vec::new()) +} + +fn allocate_refs_at_path( mut node: AccessibilityNode, refmap: &mut RefMap, config: &RefAllocConfig, + path: &mut Vec, ) -> AccessibilityNode { let root_ref_owned = config.root_ref_id.map(str::to_string); let is_interactive = INTERACTIVE_ROLES.contains(&node.role.as_str()); if is_interactive { - let entry = - ref_entry_from_node(&node, config.pid, config.source_app, root_ref_owned.clone()); + let entry = ref_entry_from_node( + &node, + config.pid, + config.source_app, + config.source_window_title, + root_ref_owned.clone(), + path, + ); node.ref_id = Some(refmap.allocate(entry)); } @@ -138,7 +145,14 @@ pub(crate) fn allocate_refs( && config.root_ref_id.is_none(); if is_skeleton_anchor { - let mut entry = ref_entry_from_node(&node, config.pid, config.source_app, None); + let mut entry = ref_entry_from_node( + &node, + config.pid, + config.source_app, + config.source_window_title, + None, + path, + ); entry.available_actions = vec![]; node.ref_id = Some(refmap.allocate(entry)); } @@ -150,8 +164,12 @@ pub(crate) fn allocate_refs( node.children = node .children .into_iter() + .enumerate() .filter_map(|child| { - let child = allocate_refs(child, refmap, config); + let (idx, child) = child; + path.push(idx); + let child = allocate_refs_at_path(child, refmap, config, path); + path.pop(); if config.compact && is_collapsible(&child) { return child.children.into_iter().next(); } @@ -184,6 +202,7 @@ mod tests { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: Some(Rect { x: 0.0, y: 0.0, @@ -253,4 +272,39 @@ mod tests { assert_eq!(out.children[0].role, "group"); assert_eq!(out.children[0].name.as_deref(), Some("Toolbar")); } + + #[test] + fn ref_entry_prefers_platform_actions() { + let mut button = node("button", Some("Save")); + button.available_actions = vec!["SetFocus".into()]; + + let entry = ref_entry_from_node(&button, 7, None, None, None, &[0]); + + assert_eq!(entry.available_actions, vec!["SetFocus"]); + } + + #[test] + fn allocate_refs_records_structural_paths() { + let mut root = node("window", Some("w")); + let mut group = node("group", Some("List")); + group.children = vec![node("button", Some("Open"))]; + root.children = vec![node("button", Some("Save")), group]; + + let mut refmap = RefMap::new(); + let config = RefAllocConfig { + include_bounds: true, + interactive_only: false, + compact: false, + pid: 7, + source_app: Some("Finder"), + source_window_title: Some("Documents"), + root_ref_id: None, + }; + let out = allocate_refs(root, &mut refmap, &config); + + let save_ref = out.children[0].ref_id.as_deref().unwrap(); + let open_ref = out.children[1].children[0].ref_id.as_deref().unwrap(); + assert_eq!(refmap.get(save_ref).unwrap().path, vec![0]); + assert_eq!(refmap.get(open_ref).unwrap().path, vec![1, 0]); + } } diff --git a/crates/core/src/refs.rs b/crates/core/src/refs.rs index 1320a13..0baf2fb 100644 --- a/crates/core/src/refs.rs +++ b/crates/core/src/refs.rs @@ -1,10 +1,15 @@ use crate::error::AppError; use serde::{Deserialize, Serialize}; use std::cell::RefCell; +use std::collections::hash_map::RandomState; use std::collections::HashMap; -use std::path::PathBuf; +use std::hash::BuildHasher; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; -const MAX_REFMAP_BYTES: u64 = 1_048_576; // 1 MB +pub(crate) const MAX_REFMAP_BYTES: u64 = 1_048_576; +static SNAPSHOT_COUNTER: AtomicU64 = AtomicU64::new(0); thread_local! { static HOME_OVERRIDE: RefCell> = const { RefCell::new(None) }; @@ -25,7 +30,11 @@ pub struct RefEntry { pub available_actions: Vec, pub source_app: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_window_title: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub root_ref: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub path: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -66,7 +75,7 @@ impl RefMap { .retain(|_, entry| entry.root_ref.as_deref() != Some(root)); } - fn serialize_with_size_check(&self) -> Result { + pub(crate) fn serialize_with_size_check(&self) -> Result { let json = serde_json::to_string(self)?; if json.len() as u64 > MAX_REFMAP_BYTES { return Err(AppError::Internal( @@ -76,45 +85,11 @@ impl RefMap { Ok(json) } - pub fn save(&self) -> Result<(), AppError> { + #[cfg(test)] + pub(crate) fn save(&self) -> Result<(), AppError> { let json = self.serialize_with_size_check()?; - let path = refmap_path()?; - let dir = path - .parent() - .ok_or_else(|| AppError::Internal("invalid refmap path".into()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(dir)?; - } - #[cfg(not(unix))] - std::fs::create_dir_all(dir)?; - - let tmp = path.with_extension("tmp"); - - #[cfg(unix)] - { - use std::io::Write; - use std::os::unix::fs::OpenOptionsExt; - let mut file = std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .mode(0o600) - .open(&tmp)?; - file.write_all(json.as_bytes())?; - file.flush()?; - } - #[cfg(not(unix))] - std::fs::write(&tmp, json.as_bytes())?; - - std::fs::rename(&tmp, &path)?; - Ok(()) + write_private_file(&path, json.as_bytes()) } pub fn load() -> Result { @@ -144,358 +119,130 @@ fn refmap_path() -> Result { Ok(home.join(".agent-desktop").join("last_refmap.json")) } -fn home_dir() -> Option { - if let Some(p) = HOME_OVERRIDE.with(|cell| cell.borrow().clone()) { - return Some(p); - } - std::env::var_os("HOME") - .map(PathBuf::from) - .or_else(|| std::env::var_os("USERPROFILE").map(PathBuf::from)) +pub(crate) fn new_snapshot_id() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + let counter = SNAPSHOT_COUNTER.fetch_add(1, Ordering::Relaxed); + let seed = RandomState::new(); + let mixed = seed.hash_one((nanos, std::process::id(), counter)); + format!("s{}", base36(mixed)) } -#[cfg(test)] -pub(crate) struct HomeGuard { - _dir: tempdir::TempDir, - prev: Option, -} +fn base36(mut value: u64) -> String { + const DIGITS: &[u8; 36] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + const MIN_LEN: usize = 4; -#[cfg(test)] -mod tempdir { - use std::fs; - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::time::{SystemTime, UNIX_EPOCH}; - - static COUNTER: AtomicU64 = AtomicU64::new(0); - - pub struct TempDir(PathBuf); - - impl TempDir { - pub fn new() -> Self { - let n = COUNTER.fetch_add(1, Ordering::SeqCst); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let path = std::env::temp_dir().join(format!("agent-desktop-test-{nanos}-{n}")); - fs::create_dir_all(&path).expect("create tempdir"); - Self(path) - } - - pub fn path(&self) -> &std::path::Path { - &self.0 + let mut buf = [b'0'; 13]; + let mut i = buf.len(); + if value == 0 { + i -= 1; + } else { + while value > 0 { + i -= 1; + buf[i] = DIGITS[(value % 36) as usize]; + value /= 36; } } - impl Drop for TempDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); - } + let digits = buf.len() - i; + if digits < MIN_LEN { + let pad = MIN_LEN - digits; + i -= pad; + } + + String::from_utf8_lossy(&buf[i..]).into_owned() +} + +pub fn validate_snapshot_id(snapshot_id: &str) -> Result<(), AppError> { + let valid = snapshot_id.len() <= 64 + && snapshot_id.len() >= 3 + && snapshot_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if !valid { + return Err(AppError::invalid_input(format!( + "Invalid snapshot_id '{snapshot_id}': use the value returned by snapshot" + ))); + } + Ok(()) +} + +pub(crate) fn write_private_file(path: &Path, bytes: &[u8]) -> Result<(), AppError> { + let dir = path + .parent() + .ok_or_else(|| AppError::Internal("invalid ref store path".into()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(dir)?; + } + #[cfg(not(unix))] + std::fs::create_dir_all(dir)?; + + let tmp = path.with_extension("tmp"); + #[cfg(unix)] + { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&tmp)?; + file.write_all(bytes)?; + file.flush()?; + } + #[cfg(not(unix))] + std::fs::write(&tmp, bytes)?; + + std::fs::rename(tmp, path)?; + Ok(()) +} + +pub(crate) fn home_dir() -> Option { + let home = HOME_OVERRIDE + .with(|cell| cell.borrow().clone()) + .or_else(|| std::env::var_os("HOME").map(PathBuf::from)) + .or_else(|| std::env::var_os("USERPROFILE").map(PathBuf::from))?; + validate_home_dir(&home).then_some(home) +} + +fn validate_home_dir(home: &Path) -> bool { + let Ok(link_meta) = std::fs::symlink_metadata(home) else { + return false; + }; + if link_meta.file_type().is_symlink() { + return false; + } + let Ok(meta) = std::fs::metadata(home) else { + return false; + }; + if !meta.is_dir() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + meta.uid() == unsafe { libc::getuid() } + } + #[cfg(not(unix))] + { + true } } #[cfg(test)] -impl HomeGuard { - pub fn new() -> Self { - let dir = tempdir::TempDir::new(); - let prev = HOME_OVERRIDE.with(|cell| cell.borrow().clone()); - HOME_OVERRIDE.with(|cell| *cell.borrow_mut() = Some(dir.path().to_path_buf())); - Self { _dir: dir, prev } - } +pub(crate) fn set_home_override(home: Option) -> Option { + HOME_OVERRIDE.with(|cell| std::mem::replace(&mut *cell.borrow_mut(), home)) } #[cfg(test)] -impl Drop for HomeGuard { - fn drop(&mut self) { - let prev = self.prev.take(); - HOME_OVERRIDE.with(|cell| *cell.borrow_mut() = prev); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_allocate_sequential() { - let mut map = RefMap::new(); - let entry = RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }; - let r1 = map.allocate(entry.clone()); - let r2 = map.allocate(entry); - assert_eq!(r1, "@e1"); - assert_eq!(r2, "@e2"); - assert_eq!(map.len(), 2); - } - - #[test] - fn test_get_existing() { - let mut map = RefMap::new(); - let entry = RefEntry { - pid: 42, - role: "textfield".into(), - name: None, - value: None, - states: vec![], - bounds: None, - bounds_hash: Some(12345), - available_actions: vec![], - source_app: Some("Finder".into()), - root_ref: None, - }; - let ref_id = map.allocate(entry); - let retrieved = map.get(&ref_id).unwrap(); - assert_eq!(retrieved.pid, 42); - assert_eq!(retrieved.role, "textfield"); - } - - #[test] - fn test_get_missing() { - let map = RefMap::new(); - assert!(map.get("@e99").is_none()); - } - - #[test] - fn test_remove_by_root_ref() { - let mut map = RefMap::new(); - let base = RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }; - - map.allocate(base.clone()); - - let drilled = RefEntry { - root_ref: Some("@e1".into()), - ..base.clone() - }; - map.allocate(drilled.clone()); - map.allocate(drilled); - assert_eq!(map.len(), 3); - - map.remove_by_root_ref("@e1"); - assert_eq!(map.len(), 1); - assert!(map.get("@e1").is_some()); - } - - #[test] - fn test_counter_continues_after_skeleton_into_drill_down() { - let mut map = RefMap::new(); - let skeleton_entry = RefEntry { - pid: 1, - role: "button".into(), - name: Some("Skeleton".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![], - source_app: None, - root_ref: None, - }; - - let last_skeleton = (0..10) - .map(|_| map.allocate(skeleton_entry.clone())) - .last() - .unwrap(); - assert_eq!(last_skeleton, "@e10"); - - let drilled = RefEntry { - root_ref: Some("@e3".into()), - ..skeleton_entry - }; - - let first_drilled = map.allocate(drilled.clone()); - let second_drilled = map.allocate(drilled); - assert_eq!( - first_drilled, "@e11", - "counter should continue past skeleton ids, not reset" - ); - assert_eq!(second_drilled, "@e12"); - assert_eq!(map.len(), 12); - - map.remove_by_root_ref("@e3"); - assert_eq!( - map.len(), - 10, - "scoped invalidation should drop only the drill-down refs" - ); - assert!(map.get("@e3").is_some(), "skeleton @e3 must survive"); - } - - #[test] - fn test_root_ref_serde_roundtrip() { - let entry = RefEntry { - pid: 1, - role: "button".into(), - name: None, - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![], - source_app: None, - root_ref: Some("@e5".into()), - }; - let json = serde_json::to_string(&entry).unwrap(); - assert!(json.contains("root_ref")); - let back: RefEntry = serde_json::from_str(&json).unwrap(); - assert_eq!(back.root_ref.as_deref(), Some("@e5")); - } - - #[test] - fn test_serialize_with_size_check_rejects_oversized() { - let mut map = RefMap::new(); - let big_name = "x".repeat(2048); - for _ in 0..600 { - map.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some(big_name.clone()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }); - } - - let result = map.serialize_with_size_check(); - assert!(result.is_err(), "oversized refmap should be rejected"); - let err = result.unwrap_err(); - let msg = err.to_string(); - assert!( - msg.contains("1MB"), - "error should mention the 1MB limit, got: {msg}" - ); - } - - #[test] - fn test_serialize_with_size_check_accepts_normal() { - let mut map = RefMap::new(); - for _ in 0..50 { - map.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some("OK".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }); - } - - let result = map.serialize_with_size_check(); - assert!(result.is_ok(), "normal-sized refmap should serialize"); - } - - #[test] - fn test_save_load_roundtrip_with_home_override() { - let _guard = HomeGuard::new(); - let mut map = RefMap::new(); - map.allocate(RefEntry { - pid: 7, - role: "button".into(), - name: Some("Send".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: Some(42), - available_actions: vec!["Click".into()], - source_app: Some("TestApp".into()), - root_ref: None, - }); - map.save().expect("save should succeed under HomeGuard"); - - let loaded = RefMap::load().expect("load should succeed"); - assert_eq!(loaded.len(), 1); - let entry = loaded.get("@e1").unwrap(); - assert_eq!(entry.pid, 7); - assert_eq!(entry.name.as_deref(), Some("Send")); - } - - #[test] - fn test_save_oversize_preserves_previous_file() { - let _guard = HomeGuard::new(); - - let mut original = RefMap::new(); - original.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some("Original".into()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }); - original.save().expect("baseline save"); - - let mut oversize = RefMap::new(); - let big = "x".repeat(2048); - for _ in 0..600 { - oversize.allocate(RefEntry { - pid: 1, - role: "button".into(), - name: Some(big.clone()), - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec!["Click".into()], - source_app: None, - root_ref: None, - }); - } - let result = oversize.save(); - assert!(result.is_err(), "oversize save must reject"); - - let reloaded = RefMap::load().expect("previous file must still load"); - assert_eq!(reloaded.len(), 1); - let entry = reloaded.get("@e1").unwrap(); - assert_eq!(entry.name.as_deref(), Some("Original")); - } - - #[test] - fn test_root_ref_none_omitted() { - let entry = RefEntry { - pid: 1, - role: "button".into(), - name: None, - value: None, - states: vec![], - bounds: None, - bounds_hash: None, - available_actions: vec![], - source_app: None, - root_ref: None, - }; - let json = serde_json::to_string(&entry).unwrap(); - assert!(!json.contains("root_ref")); - } -} +#[path = "refs_tests.rs"] +mod tests; diff --git a/crates/core/src/refs_lock.rs b/crates/core/src/refs_lock.rs new file mode 100644 index 0000000..9297080 --- /dev/null +++ b/crates/core/src/refs_lock.rs @@ -0,0 +1,296 @@ +use crate::error::AppError; +use std::io::{ErrorKind, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const LOCK_TIMEOUT: Duration = Duration::from_secs(2); +const MALFORMED_LOCK_WINDOW: Duration = Duration::from_secs(30); + +pub(crate) struct RefStoreLock { + path: PathBuf, + token: String, +} + +impl RefStoreLock { + pub(crate) fn acquire(path: &Path) -> Result { + if let Some(dir) = path.parent() { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(dir)?; + } + #[cfg(not(unix))] + std::fs::create_dir_all(dir)?; + } + let start = Instant::now(); + loop { + let token = lock_token(); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + { + Ok(mut file) => { + if let Err(err) = + writeln!(file, "{} {} {}", std::process::id(), now_secs(), token) + { + let _ = std::fs::remove_file(path); + return Err(err.into()); + } + return Ok(Self { + path: path.to_path_buf(), + token, + }); + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + if try_remove_stale_lock(path) { + continue; + } + if start.elapsed() > LOCK_TIMEOUT { + return Err(AppError::Internal(format!( + "Timed out waiting for ref store lock at {}", + path.display() + ))); + } + std::thread::sleep(Duration::from_millis(10)); + } + Err(err) => return Err(err.into()), + } + } + } +} + +fn try_remove_stale_lock(path: &Path) -> bool { + let snapshot = match LockSnapshot::read(path) { + Ok(Some(snapshot)) => snapshot, + Ok(None) => return true, + Err(()) => return false, + }; + if !snapshot.is_stale() { + return false; + } + let current = match LockSnapshot::read(path) { + Ok(Some(current)) => current, + Ok(None) => return true, + Err(()) => return false, + }; + if current.contents != snapshot.contents || current.modified != snapshot.modified { + return false; + } + match std::fs::remove_file(path) { + Ok(()) => true, + Err(err) if err.kind() == ErrorKind::NotFound => true, + Err(_) => false, + } +} + +struct LockSnapshot { + contents: String, + modified: Option, +} + +impl LockSnapshot { + fn read(path: &Path) -> Result, ()> { + let contents = match std::fs::read_to_string(path) { + Ok(contents) => contents, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(()), + }; + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); + Ok(Some(Self { contents, modified })) + } + + fn is_stale(&self) -> bool { + if self.contents.trim().is_empty() { + return self.modified_age_exceeds(MALFORMED_LOCK_WINDOW); + } + match self.pid() { + Some(pid) => match process_is_alive(pid) { + Some(true) => self.live_pid_lock_is_stale(pid), + Some(false) => true, + None => self.modified_age_exceeds(MALFORMED_LOCK_WINDOW), + }, + None => self.modified_age_exceeds(MALFORMED_LOCK_WINDOW), + } + } + + fn live_pid_lock_is_stale(&self, pid: u32) -> bool { + if !self.token_shape_matches_pid(pid) { + return self.modified_age_exceeds(LOCK_TIMEOUT); + } + self.modified_age_exceeds(MALFORMED_LOCK_WINDOW) + } + + fn pid(&self) -> Option { + self.contents + .split_whitespace() + .next() + .and_then(|s| s.parse::().ok()) + } + + fn token(&self) -> Option<&str> { + self.contents.split_whitespace().nth(2) + } + + fn owned_by(&self, token: &str) -> bool { + self.pid() == Some(std::process::id()) && self.token() == Some(token) + } + + fn token_shape_matches_pid(&self, pid: u32) -> bool { + self.token() + .and_then(|token| token.split_once('-')) + .and_then(|(prefix, _)| prefix.parse::().ok()) + == Some(pid) + } + + fn modified_age_exceeds(&self, duration: Duration) -> bool { + self.modified + .and_then(|modified| SystemTime::now().duration_since(modified).ok()) + .is_some_and(|age| age > duration) + } +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn lock_token() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + format!("{}-{nanos}", std::process::id()) +} + +#[cfg(unix)] +fn process_is_alive(pid: u32) -> Option { + unsafe extern "C" { + fn kill(pid: i32, sig: i32) -> i32; + } + let result = unsafe { kill(pid as i32, 0) }; + if result == 0 { + return Some(true); + } + match std::io::Error::last_os_error().raw_os_error() { + Some(errno) if errno == libc::ESRCH => Some(false), + Some(errno) if errno == libc::EPERM => None, + _ => None, + } +} + +#[cfg(not(unix))] +fn process_is_alive(_pid: u32) -> Option { + None +} + +impl Drop for RefStoreLock { + fn drop(&mut self) { + let should_remove = LockSnapshot::read(&self.path) + .ok() + .flatten() + .is_some_and(|snapshot| snapshot.owned_by(&self.token)); + if should_remove { + let _ = std::fs::remove_file(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn lock_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "agent-desktop-{name}-{}-{}.lock", + std::process::id(), + lock_token() + )) + } + + #[test] + fn acquire_removes_lock_on_drop() { + let path = lock_path("drop"); + { + let _lock = RefStoreLock::acquire(&path).unwrap(); + assert!(path.exists()); + } + assert!(!path.exists()); + } + + #[test] + fn stale_dead_pid_lock_is_replaced() { + let path = lock_path("stale-pid"); + fs::write(&path, "999999 1 stale").unwrap(); + let _lock = RefStoreLock::acquire(&path).unwrap(); + let contents = fs::read_to_string(&path).unwrap(); + assert!(contents.starts_with(&std::process::id().to_string())); + } + + #[test] + fn old_same_process_lock_with_foreign_token_shape_is_stale() { + let snapshot = LockSnapshot { + contents: format!("{} 1 old-token", std::process::id()), + modified: Some(SystemTime::now() - Duration::from_secs(60)), + }; + + assert!(snapshot.is_stale()); + } + + #[test] + fn recent_live_pid_lock_is_not_stale() { + let snapshot = LockSnapshot { + contents: format!( + "{} {} {}-token", + std::process::id(), + now_secs(), + std::process::id() + ), + modified: Some(SystemTime::now()), + }; + + assert!(!snapshot.is_stale()); + } + + #[test] + fn old_live_pid_lock_with_foreign_token_shape_is_stale() { + let snapshot = LockSnapshot { + contents: "2000 1 1000-old".into(), + modified: Some(SystemTime::now() - Duration::from_secs(3)), + }; + + assert!(snapshot.live_pid_lock_is_stale(2000)); + } + + #[test] + fn recent_live_pid_lock_with_matching_token_shape_is_not_stale() { + let snapshot = LockSnapshot { + contents: "2000 1 2000-token".into(), + modified: Some(SystemTime::now() - Duration::from_secs(3)), + }; + + assert!(!snapshot.live_pid_lock_is_stale(2000)); + } + + #[test] + fn drop_does_not_remove_replaced_lock() { + let path = lock_path("replaced"); + let lock = RefStoreLock::acquire(&path).unwrap(); + fs::write( + &path, + format!("{} {} replacement-token", std::process::id(), now_secs()), + ) + .unwrap(); + + drop(lock); + + assert!(path.exists()); + fs::remove_file(&path).unwrap(); + } +} diff --git a/crates/core/src/refs_store.rs b/crates/core/src/refs_store.rs new file mode 100644 index 0000000..4346768 --- /dev/null +++ b/crates/core/src/refs_store.rs @@ -0,0 +1,196 @@ +use crate::{ + error::{AdapterError, AppError}, + refs::{home_dir, new_snapshot_id, validate_snapshot_id, write_private_file, RefMap}, + refs_lock::RefStoreLock, +}; +use std::io::Read; +use std::path::PathBuf; + +const LATEST_SNAPSHOT_FILE: &str = "latest_snapshot_id"; +const MAX_SAVED_SNAPSHOTS: usize = 512; + +#[derive(Debug, Clone)] +pub struct RefStore { + base_dir: PathBuf, +} + +impl RefStore { + pub fn new() -> Result { + let home = + home_dir().ok_or_else(|| AppError::Internal("HOME directory not found".into()))?; + Ok(Self { + base_dir: home.join(".agent-desktop"), + }) + } + + #[cfg(test)] + pub fn for_tests() -> Result { + Self::new() + } + + pub fn save_new_snapshot(&self, refmap: &RefMap) -> Result { + self.with_write_lock(|| { + let snapshot_id = new_snapshot_id(); + self.save_snapshot_unlocked(&snapshot_id, refmap)?; + self.set_latest_unlocked(&snapshot_id)?; + self.prune_old_snapshots_unlocked(&snapshot_id)?; + Ok(snapshot_id) + }) + } + + pub fn save_snapshot(&self, snapshot_id: &str, refmap: &RefMap) -> Result<(), AppError> { + self.with_write_lock(|| self.save_snapshot_unlocked(snapshot_id, refmap)) + } + + pub fn save_existing_snapshot( + &self, + snapshot_id: &str, + refmap: &RefMap, + ) -> Result<(), AppError> { + self.with_write_lock(|| self.save_snapshot_unlocked(snapshot_id, refmap)) + } + + pub fn load(&self, snapshot_id: Option<&str>) -> Result { + match snapshot_id { + Some(id) => self.load_snapshot(id), + None => self.load_latest(), + } + } + + pub fn load_latest(&self) -> Result { + if let Ok(id) = std::fs::read_to_string(self.latest_path()) { + let id = id.trim(); + if !id.is_empty() { + return self.load_snapshot(id); + } + } + if let Some(refmap) = self.migrate_legacy_latest()? { + return Ok(refmap); + } + Err(AppError::Adapter(AdapterError::snapshot_not_found( + "latest", + ))) + } + + pub fn load_snapshot(&self, snapshot_id: &str) -> Result { + validate_snapshot_id(snapshot_id)?; + let path = self.snapshot_path(snapshot_id); + let mut file = std::fs::File::open(&path) + .map_err(|_| AppError::Adapter(AdapterError::snapshot_not_found(snapshot_id)))?; + let metadata = file.metadata()?; + if metadata.len() > crate::refs::MAX_REFMAP_BYTES { + return Err(AppError::Internal( + "RefMap file exceeds 1MB size limit".into(), + )); + } + let mut json = String::with_capacity(metadata.len() as usize); + file.read_to_string(&mut json)?; + if json.len() as u64 > crate::refs::MAX_REFMAP_BYTES { + return Err(AppError::Internal( + "RefMap file exceeds 1MB size limit".into(), + )); + } + Ok(serde_json::from_str(&json)?) + } + + pub fn set_latest(&self, snapshot_id: &str) -> Result<(), AppError> { + self.with_write_lock(|| self.set_latest_unlocked(snapshot_id)) + } + + pub fn latest_snapshot_id(&self) -> Option { + std::fs::read_to_string(self.latest_path()) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + } + + fn save_snapshot_unlocked(&self, snapshot_id: &str, refmap: &RefMap) -> Result<(), AppError> { + validate_snapshot_id(snapshot_id)?; + let json = refmap.serialize_with_size_check()?; + let path = self.snapshot_path(snapshot_id); + write_private_file(&path, json.as_bytes()) + } + + fn set_latest_unlocked(&self, snapshot_id: &str) -> Result<(), AppError> { + validate_snapshot_id(snapshot_id)?; + write_private_file(&self.latest_path(), snapshot_id.as_bytes()) + } + + fn latest_path(&self) -> PathBuf { + self.base_dir.join(LATEST_SNAPSHOT_FILE) + } + + fn snapshot_path(&self, snapshot_id: &str) -> PathBuf { + self.base_dir + .join("snapshots") + .join(snapshot_id) + .join("refmap.json") + } + + fn snapshots_dir(&self) -> PathBuf { + self.base_dir.join("snapshots") + } + + fn prune_old_snapshots_unlocked(&self, latest_id: &str) -> Result<(), AppError> { + let dir = self.snapshots_dir(); + let Ok(entries) = std::fs::read_dir(&dir) else { + return Ok(()); + }; + let mut snapshots = Vec::new(); + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let id = entry.file_name().to_string_lossy().to_string(); + if validate_snapshot_id(&id).is_err() { + continue; + } + let modified = entry + .metadata() + .and_then(|metadata| metadata.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + snapshots.push((modified, id, entry.path())); + } + if snapshots.len() <= MAX_SAVED_SNAPSHOTS { + return Ok(()); + } + snapshots.sort_by_key(|(modified, id, _)| (*modified, id.clone())); + let remove_count = snapshots.len() - MAX_SAVED_SNAPSHOTS; + for (_, id, path) in snapshots.into_iter().take(remove_count) { + if id != latest_id { + let _ = std::fs::remove_dir_all(path); + } + } + Ok(()) + } + + fn lock_path(&self) -> PathBuf { + self.base_dir.join("refstore.lock") + } + + fn with_write_lock(&self, f: impl FnOnce() -> Result) -> Result { + let _lock = RefStoreLock::acquire(&self.lock_path())?; + f() + } + + fn migrate_legacy_latest(&self) -> Result, AppError> { + self.with_write_lock(|| { + if let Ok(id) = std::fs::read_to_string(self.latest_path()) { + let id = id.trim(); + if !id.is_empty() { + return self.load_snapshot(id).map(Some); + } + } + let Ok(refmap) = RefMap::load() else { + return Ok(None); + }; + let snapshot_id = new_snapshot_id(); + self.save_snapshot_unlocked(&snapshot_id, &refmap)?; + self.set_latest_unlocked(&snapshot_id)?; + Ok(Some(refmap)) + }) + } +} diff --git a/crates/core/src/refs_test_support.rs b/crates/core/src/refs_test_support.rs new file mode 100644 index 0000000..9fab538 --- /dev/null +++ b/crates/core/src/refs_test_support.rs @@ -0,0 +1,51 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +static COUNTER: AtomicU64 = AtomicU64::new(0); + +pub(crate) struct HomeGuard { + _dir: TempDir, + prev: Option, +} + +impl HomeGuard { + pub(crate) fn new() -> Self { + let dir = TempDir::new(); + let prev = crate::refs::set_home_override(Some(dir.path().to_path_buf())); + Self { _dir: dir, prev } + } +} + +impl Drop for HomeGuard { + fn drop(&mut self) { + let prev = self.prev.take(); + crate::refs::set_home_override(prev); + } +} + +struct TempDir(PathBuf); + +impl TempDir { + fn new() -> Self { + let n = COUNTER.fetch_add(1, Ordering::SeqCst); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let path = std::env::temp_dir().join(format!("agent-desktop-test-{nanos}-{n}")); + fs::create_dir_all(&path).expect("create tempdir"); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} diff --git a/crates/core/src/refs_tests.rs b/crates/core/src/refs_tests.rs new file mode 100644 index 0000000..c9cd57d --- /dev/null +++ b/crates/core/src/refs_tests.rs @@ -0,0 +1,309 @@ +use super::*; +use crate::{refs_store::RefStore, refs_test_support::HomeGuard}; + +fn entry(role: &str, name: Option<&str>) -> RefEntry { + RefEntry { + pid: 1, + role: role.into(), + name: name.map(String::from), + value: None, + states: vec![], + bounds: None, + bounds_hash: None, + available_actions: vec!["Click".into()], + source_app: None, + source_window_title: None, + root_ref: None, + path: Vec::new(), + } +} + +#[test] +fn test_allocate_sequential() { + let mut map = RefMap::new(); + let r1 = map.allocate(entry("button", Some("OK"))); + let r2 = map.allocate(entry("button", Some("OK"))); + assert_eq!(r1, "@e1"); + assert_eq!(r2, "@e2"); + assert_eq!(map.len(), 2); +} + +#[test] +fn test_get_existing() { + let mut map = RefMap::new(); + let ref_id = map.allocate(RefEntry { + pid: 42, + role: "textfield".into(), + name: None, + value: None, + states: vec![], + bounds: None, + bounds_hash: Some(12345), + available_actions: vec![], + source_app: Some("Finder".into()), + source_window_title: Some("Documents".into()), + root_ref: None, + path: Vec::new(), + }); + let retrieved = map.get(&ref_id).unwrap(); + assert_eq!(retrieved.pid, 42); + assert_eq!(retrieved.role, "textfield"); +} + +#[test] +fn test_get_missing() { + let map = RefMap::new(); + assert!(map.get("@e99").is_none()); +} + +#[test] +fn test_remove_by_root_ref() { + let mut map = RefMap::new(); + let base = entry("button", Some("OK")); + + map.allocate(base.clone()); + + let drilled = RefEntry { + root_ref: Some("@e1".into()), + ..base + }; + map.allocate(drilled.clone()); + map.allocate(drilled); + assert_eq!(map.len(), 3); + + map.remove_by_root_ref("@e1"); + assert_eq!(map.len(), 1); + assert!(map.get("@e1").is_some()); +} + +#[test] +fn test_counter_continues_after_skeleton_into_drill_down() { + let mut map = RefMap::new(); + let skeleton_entry = entry("button", Some("Skeleton")); + + let last_skeleton = (0..10) + .map(|_| map.allocate(skeleton_entry.clone())) + .last() + .unwrap(); + assert_eq!(last_skeleton, "@e10"); + + let drilled = RefEntry { + root_ref: Some("@e3".into()), + ..skeleton_entry + }; + + let first_drilled = map.allocate(drilled.clone()); + let second_drilled = map.allocate(drilled); + assert_eq!( + first_drilled, "@e11", + "counter should continue past skeleton ids, not reset" + ); + assert_eq!(second_drilled, "@e12"); + assert_eq!(map.len(), 12); + + map.remove_by_root_ref("@e3"); + assert_eq!( + map.len(), + 10, + "scoped invalidation should drop only the drill-down refs" + ); + assert!(map.get("@e3").is_some(), "skeleton @e3 must survive"); +} + +#[test] +fn test_root_ref_serde_roundtrip() { + let entry = RefEntry { + root_ref: Some("@e5".into()), + ..entry("button", None) + }; + let json = serde_json::to_string(&entry).unwrap(); + assert!(json.contains("root_ref")); + let back: RefEntry = serde_json::from_str(&json).unwrap(); + assert_eq!(back.root_ref.as_deref(), Some("@e5")); +} + +#[test] +fn test_serialize_with_size_check_rejects_oversized() { + let mut map = RefMap::new(); + let big_name = "x".repeat(2048); + for _ in 0..600 { + map.allocate(entry("button", Some(&big_name))); + } + + let result = map.serialize_with_size_check(); + assert!(result.is_err(), "oversized refmap should be rejected"); + let err = result.unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("1MB"), + "error should mention the 1MB limit, got: {msg}" + ); +} + +#[test] +fn test_serialize_with_size_check_accepts_normal() { + let mut map = RefMap::new(); + for _ in 0..50 { + map.allocate(entry("button", Some("OK"))); + } + + let result = map.serialize_with_size_check(); + assert!(result.is_ok(), "normal-sized refmap should serialize"); +} + +#[test] +fn test_save_load_roundtrip_with_home_override() { + let _guard = HomeGuard::new(); + let mut map = RefMap::new(); + map.allocate(RefEntry { + pid: 7, + role: "button".into(), + name: Some("Send".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: Some(42), + available_actions: vec!["Click".into()], + source_app: Some("TestApp".into()), + source_window_title: Some("Test Window".into()), + root_ref: None, + path: Vec::new(), + }); + map.save().expect("save should succeed under HomeGuard"); + + let loaded = RefMap::load().expect("load should succeed"); + assert_eq!(loaded.len(), 1); + let entry = loaded.get("@e1").unwrap(); + assert_eq!(entry.pid, 7); + assert_eq!(entry.name.as_deref(), Some("Send")); +} + +#[test] +fn test_refstore_snapshot_roundtrip_and_latest_pointer() { + let _guard = HomeGuard::new(); + let store = RefStore::new().unwrap(); + let mut map = RefMap::new(); + map.allocate(RefEntry { + pid: 7, + role: "button".into(), + name: Some("Send".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash: Some(42), + available_actions: vec!["Click".into()], + source_app: Some("TestApp".into()), + source_window_title: Some("Test Window".into()), + root_ref: None, + path: Vec::new(), + }); + + let snapshot_id = store.save_new_snapshot(&map).unwrap(); + + assert_eq!( + store.latest_snapshot_id().as_deref(), + Some(snapshot_id.as_str()) + ); + assert_eq!(store.load(Some(&snapshot_id)).unwrap().len(), 1); + assert_eq!(store.load(None).unwrap().len(), 1); +} + +#[test] +fn test_save_existing_snapshot_does_not_promote_latest_pointer() { + let _guard = HomeGuard::new(); + let store = RefStore::new().unwrap(); + + let mut first = RefMap::new(); + first.allocate(entry("button", Some("First"))); + let first_id = store.save_new_snapshot(&first).unwrap(); + + let mut second = RefMap::new(); + second.allocate(entry("button", Some("Second"))); + let second_id = store.save_new_snapshot(&second).unwrap(); + + first.allocate(entry("button", Some("First Child"))); + store.save_existing_snapshot(&first_id, &first).unwrap(); + + assert_eq!( + store.latest_snapshot_id().as_deref(), + Some(second_id.as_str()) + ); + assert_eq!(store.load(Some(&first_id)).unwrap().len(), 2); +} + +#[test] +fn test_snapshot_ids_are_compact_and_valid() { + let id = new_snapshot_id(); + + assert!(id.starts_with('s')); + assert!(id.len() <= 14, "snapshot id should stay token-light: {id}"); + validate_snapshot_id(&id).unwrap(); +} + +#[test] +fn test_base36_encoding() { + assert_eq!(base36(0), "0000"); + assert_eq!(base36(35), "000z"); + assert_eq!(base36(36), "0010"); + assert_eq!(base36(36 * 36 * 36 + 35), "100z"); + assert!(base36(u64::MAX).len() >= 4); +} + +#[test] +fn test_new_snapshot_id_passes_validation() { + for _ in 0..256 { + let id = new_snapshot_id(); + validate_snapshot_id(&id).expect("generated snapshot id must validate"); + } +} + +#[test] +fn test_refstore_migrates_legacy_latest_refmap() { + let _guard = HomeGuard::new(); + let mut map = RefMap::new(); + map.allocate(entry("button", Some("Legacy"))); + map.save().unwrap(); + + let store = RefStore::new().unwrap(); + let loaded = store.load_latest().unwrap(); + + assert_eq!(loaded.len(), 1); + assert!(store.latest_snapshot_id().is_some()); +} + +#[test] +fn test_validate_snapshot_id_rejects_bad_values() { + assert!(validate_snapshot_id("").is_err()); + assert!(validate_snapshot_id("s").is_err()); + assert!(validate_snapshot_id(&format!("s{}", "x".repeat(64))).is_err()); + assert!(validate_snapshot_id("bad/id").is_err()); +} + +#[test] +fn test_save_oversize_preserves_previous_file() { + let _guard = HomeGuard::new(); + + let mut original = RefMap::new(); + original.allocate(entry("button", Some("Original"))); + original.save().expect("baseline save"); + + let mut oversize = RefMap::new(); + let big = "x".repeat(2048); + for _ in 0..600 { + oversize.allocate(entry("button", Some(&big))); + } + let result = oversize.save(); + assert!(result.is_err(), "oversize save must reject"); + + let reloaded = RefMap::load().expect("previous file must still load"); + assert_eq!(reloaded.len(), 1); + let entry = reloaded.get("@e1").unwrap(); + assert_eq!(entry.name.as_deref(), Some("Original")); +} + +#[test] +fn test_root_ref_none_omitted() { + let entry = entry("button", None); + let json = serde_json::to_string(&entry).unwrap(); + assert!(!json.contains("root_ref")); +} diff --git a/crates/core/src/roles.rs b/crates/core/src/roles.rs new file mode 100644 index 0000000..a2c6f55 --- /dev/null +++ b/crates/core/src/roles.rs @@ -0,0 +1,104 @@ +/// Interactive roles that receive refs during snapshot allocation. +/// +/// Each entry must be produced by at least one platform adapter's native-to-canonical +/// role mapping. Read-only roles (statictext, image) and container roles (group, list, +/// table) stay out. Platform-private extensions live in the adapter, not here. +pub const INTERACTIVE_ROLES: &[&str] = &[ + "button", + "cell", + "checkbox", + "colorwell", + "combobox", + "dockitem", + "incrementor", + "link", + "menubutton", + "menuitem", + "radiobutton", + "slider", + "switch", + "tab", + "textfield", + "treeitem", +]; + +/// Returns true when `role` is in [`INTERACTIVE_ROLES`]. +pub fn is_interactive_role(role: &str) -> bool { + INTERACTIVE_ROLES.contains(&role) +} + +/// Returns true for roles whose checked/unchecked state can be queried and set. +pub fn is_toggleable_role(role: &str) -> bool { + matches!(role, "checkbox" | "switch" | "radiobutton") +} + +/// Returns true for roles that carry an expanded/collapsed surface state. +pub fn is_expandable_role(role: &str) -> bool { + matches!(role, "combobox" | "menubutton" | "treeitem") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interactive_roles_are_sorted_and_unique() { + let mut sorted = INTERACTIVE_ROLES.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.as_slice(), INTERACTIVE_ROLES); + } + + #[test] + fn toggleable_roles_are_a_subset_of_interactive() { + for role in ["checkbox", "switch", "radiobutton"] { + assert!(is_toggleable_role(role)); + assert!(is_interactive_role(role)); + } + assert!(!is_toggleable_role("button")); + assert!(!is_toggleable_role("textfield")); + } + + #[test] + fn expandable_roles_are_a_subset_of_interactive() { + for role in ["combobox", "menubutton", "treeitem"] { + assert!(is_expandable_role(role)); + assert!(is_interactive_role(role)); + } + assert!(!is_expandable_role("button")); + assert!(!is_expandable_role("checkbox")); + assert!(!is_expandable_role("disclosure")); + } + + #[test] + fn every_expandable_role_is_interactive() { + for role in ["combobox", "menubutton", "treeitem"] { + assert!( + is_expandable_role(role), + "{role} expected expandable for subset check" + ); + assert!( + INTERACTIVE_ROLES.contains(&role), + "expandable role {role} missing from INTERACTIVE_ROLES" + ); + } + } + + #[test] + fn every_toggleable_role_is_interactive() { + for role in ["checkbox", "switch", "radiobutton"] { + assert!(is_toggleable_role(role)); + assert!( + INTERACTIVE_ROLES.contains(&role), + "toggleable role {role} missing from INTERACTIVE_ROLES" + ); + } + } + + #[test] + fn read_only_roles_are_never_interactive() { + for role in ["statictext", "image", "group", "list", "table"] { + assert!(!is_interactive_role(role)); + } + } +} diff --git a/crates/core/src/snapshot.rs b/crates/core/src/snapshot.rs index bf8e292..05f9bb6 100644 --- a/crates/core/src/snapshot.rs +++ b/crates/core/src/snapshot.rs @@ -4,12 +4,14 @@ use crate::{ node::{AccessibilityNode, WindowInfo}, ref_alloc::{self, RefAllocConfig}, refs::RefMap, + refs_store::RefStore, }; pub struct SnapshotResult { pub tree: AccessibilityNode, pub refmap: RefMap, pub window: WindowInfo, + pub snapshot_id: Option, } pub fn build( @@ -82,6 +84,7 @@ pub fn build( compact: opts.compact, pid: window.pid, source_app: Some(window.app.as_str()), + source_window_title: Some(window.title.as_str()), root_ref_id: None, }; let mut tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); @@ -92,6 +95,7 @@ pub fn build( tree, refmap, window, + snapshot_id: None, }) } @@ -101,8 +105,10 @@ pub fn run( app_name: Option<&str>, window_id: Option<&str>, ) -> Result { - let result = build(adapter, opts, app_name, window_id)?; - result.refmap.save()?; + let mut result = build(adapter, opts, app_name, window_id)?; + let store = RefStore::new()?; + let snapshot_id = store.save_new_snapshot(&result.refmap)?; + result.snapshot_id = Some(snapshot_id); Ok(result) } @@ -111,31 +117,39 @@ pub fn append_surface_refs( pid: i32, source_app: Option<&str>, surface: SnapshotSurface, -) -> Option { +) -> Result, AppError> { let filter = WindowFilter { focused_only: false, app: None, }; - let windows = adapter.list_windows(&filter).ok()?; - let window = windows.into_iter().find(|w| w.pid == pid)?; + let windows = adapter.list_windows(&filter)?; + let Some(window) = windows.into_iter().find(|w| w.pid == pid) else { + return Ok(None); + }; let opts = TreeOptions { surface, interactive_only: true, ..Default::default() }; - let raw_tree = adapter.get_tree(&window, &opts).ok()?; - let mut refmap = RefMap::load().ok()?; + let raw_tree = adapter.get_tree(&window, &opts)?; + let store = RefStore::new()?; + let mut refmap = store.load_latest()?; let config = RefAllocConfig { include_bounds: false, interactive_only: true, compact: false, pid, source_app, + source_window_title: Some(window.title.as_str()), root_ref_id: None, }; let tree = ref_alloc::allocate_refs(raw_tree, &mut refmap, &config); - refmap.save().ok()?; - Some(tree) + if let Some(id) = store.latest_snapshot_id() { + store.save_existing_snapshot(&id, &refmap)?; + } else { + store.save_new_snapshot(&refmap)?; + } + Ok(Some(tree)) } #[cfg(test)] diff --git a/crates/core/src/snapshot_ref.rs b/crates/core/src/snapshot_ref.rs index 76c4e98..5cae448 100644 --- a/crates/core/src/snapshot_ref.rs +++ b/crates/core/src/snapshot_ref.rs @@ -1,9 +1,9 @@ use crate::{ - adapter::{PlatformAdapter, TreeOptions, WindowFilter}, + adapter::{PlatformAdapter, TreeOptions}, error::AppError, node::WindowInfo, ref_alloc::{self, RefAllocConfig}, - refs::RefMap, + refs_store::RefStore, snapshot::SnapshotResult, }; @@ -11,8 +11,13 @@ pub fn run_from_ref( adapter: &dyn PlatformAdapter, opts: &TreeOptions, root_ref_id: &str, + snapshot_id: Option<&str>, ) -> Result { - let mut refmap = RefMap::load()?; + let store = RefStore::new()?; + let mut refmap = store.load(snapshot_id)?; + let active_snapshot_id = snapshot_id + .map(str::to_string) + .or_else(|| store.latest_snapshot_id()); let entry = refmap .get(root_ref_id) @@ -26,12 +31,14 @@ pub fn run_from_ref( refmap.remove_by_root_ref(root_ref_id); let source_app = entry.source_app.as_deref(); + let source_window_title = entry.source_window_title.as_deref(); let config = RefAllocConfig { include_bounds: opts.include_bounds, interactive_only: opts.interactive_only, compact: opts.compact, pid: entry.pid, source_app, + source_window_title, root_ref_id: Some(root_ref_id), }; @@ -39,18 +46,19 @@ pub fn run_from_ref( crate::hints::add_structural_hints(&mut tree); - refmap.save()?; + let saved_snapshot_id = if let Some(id) = active_snapshot_id { + store.save_existing_snapshot(&id, &refmap)?; + Some(id) + } else { + Some(store.save_new_snapshot(&refmap)?) + }; - let window = adapter - .list_windows(&WindowFilter { - focused_only: false, - app: None, - }) - .ok() - .and_then(|ws| ws.into_iter().find(|w| w.pid == entry.pid)) - .unwrap_or(WindowInfo { + let window = + crate::window_lookup::find_window_for_pid(entry.pid, adapter).unwrap_or(WindowInfo { id: String::new(), - title: format!("subtree from {root_ref_id}"), + title: entry + .source_window_title + .unwrap_or_else(|| format!("subtree from {root_ref_id}")), app: entry.source_app.unwrap_or_default(), pid: entry.pid, bounds: None, @@ -61,9 +69,14 @@ pub fn run_from_ref( tree, refmap, window, + snapshot_id: saved_snapshot_id, }) } #[cfg(test)] #[path = "snapshot_ref_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "snapshot_ref_alloc_tests.rs"] +mod alloc_tests; diff --git a/crates/core/src/snapshot_ref_alloc_tests.rs b/crates/core/src/snapshot_ref_alloc_tests.rs new file mode 100644 index 0000000..ac06e26 --- /dev/null +++ b/crates/core/src/snapshot_ref_alloc_tests.rs @@ -0,0 +1,107 @@ +use crate::node::AccessibilityNode; +use crate::ref_alloc::{self, RefAllocConfig}; +use crate::refs::RefMap; + +fn node(role: &str) -> AccessibilityNode { + AccessibilityNode { + ref_id: None, + role: role.into(), + name: None, + value: None, + description: None, + hint: None, + states: vec![], + available_actions: vec![], + bounds: None, + children_count: None, + children: vec![], + } +} + +fn drill_config<'a>( + source_app: Option<&'a str>, + pid: i32, + root_ref_id: &'a str, + interactive_only: bool, + compact: bool, +) -> RefAllocConfig<'a> { + RefAllocConfig { + include_bounds: false, + interactive_only, + compact, + pid, + source_app, + source_window_title: Some("Drill Window"), + root_ref_id: Some(root_ref_id), + } +} + +#[test] +fn test_drill_alloc_tags_entries() { + let mut btn = node("button"); + btn.name = Some("Submit".into()); + let mut root = node("group"); + root.children = vec![btn]; + + let mut refmap = RefMap::new(); + let config = drill_config(Some("TestApp"), 42, "@e5", false, false); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + + assert_eq!(refmap.len(), 1); + let btn_ref = tree.children[0] + .ref_id + .as_deref() + .expect("button should have ref"); + let entry = refmap.get(btn_ref).expect("entry should exist"); + assert_eq!(entry.root_ref.as_deref(), Some("@e5")); + assert_eq!(entry.pid, 42); + assert_eq!(entry.source_app.as_deref(), Some("TestApp")); +} + +#[test] +fn test_drill_alloc_respects_interactive_only() { + let btn = node("button"); + let text = node("statictext"); + let mut root = node("group"); + root.children = vec![btn, text]; + + let mut refmap = RefMap::new(); + let config = drill_config(None, 1, "@e1", true, false); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + + assert_eq!(tree.children.len(), 1); + assert_eq!(tree.children[0].role, "button"); +} + +#[test] +fn test_drill_alloc_preserves_truncated_child() { + let mut container = node("group"); + container.name = Some("Sidebar".into()); + container.children_count = Some(4); + let mut root = node("window"); + root.children = vec![container]; + + let mut refmap = RefMap::new(); + let config = drill_config(None, 1, "@e1", true, false); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + + assert_eq!(tree.children.len(), 1); + assert_eq!(tree.children[0].children_count, Some(4)); +} + +#[test] +fn test_drill_alloc_compact() { + let mut btn = node("button"); + btn.name = Some("OK".into()); + let mut wrapper = node("group"); + wrapper.children = vec![btn]; + let mut root = node("window"); + root.children = vec![wrapper]; + + let mut refmap = RefMap::new(); + let config = drill_config(None, 1, "@e1", false, true); + let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); + + assert_eq!(tree.children.len(), 1); + assert_eq!(tree.children[0].role, "button"); +} diff --git a/crates/core/src/snapshot_ref_tests.rs b/crates/core/src/snapshot_ref_tests.rs index c66502c..2c81850 100644 --- a/crates/core/src/snapshot_ref_tests.rs +++ b/crates/core/src/snapshot_ref_tests.rs @@ -1,11 +1,12 @@ use super::*; -use crate::action::Action; -use crate::adapter::{NativeHandle, PermissionStatus, PlatformAdapter}; +use crate::action::ActionRequest; +use crate::adapter::{NativeHandle, PlatformAdapter}; use crate::error::AdapterError; use crate::node::AccessibilityNode; use crate::ref_alloc::ref_entry_from_node; -use crate::refs::HomeGuard; -use std::cell::Cell; +use crate::refs_test_support::HomeGuard; +use crate::{refs::RefMap, refs_store::RefStore}; +use std::sync::atomic::{AtomicU32, Ordering}; fn node(role: &str) -> AccessibilityNode { AccessibilityNode { @@ -16,6 +17,7 @@ fn node(role: &str) -> AccessibilityNode { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children_count: None, children: vec![], @@ -30,31 +32,24 @@ fn named(role: &str, name: &str) -> AccessibilityNode { struct StubAdapter { subtree: AccessibilityNode, - resolve_calls: Cell, + resolve_calls: AtomicU32, } impl StubAdapter { fn new(subtree: AccessibilityNode) -> Self { Self { subtree, - resolve_calls: Cell::new(0), + resolve_calls: AtomicU32::new(0), } } } -unsafe impl Send for StubAdapter {} -unsafe impl Sync for StubAdapter {} - impl PlatformAdapter for StubAdapter { - fn check_permissions(&self) -> PermissionStatus { - PermissionStatus::Granted - } - fn resolve_element( &self, _entry: &crate::refs::RefEntry, ) -> Result { - self.resolve_calls.set(self.resolve_calls.get() + 1); + self.resolve_calls.fetch_add(1, Ordering::SeqCst); Ok(NativeHandle::null()) } @@ -69,17 +64,45 @@ impl PlatformAdapter for StubAdapter { fn execute_action( &self, _handle: &NativeHandle, - _action: Action, + _request: ActionRequest, ) -> Result { Err(AdapterError::not_supported("execute_action")) } } +fn save_latest(refmap: RefMap) -> String { + RefStore::new() + .unwrap() + .save_new_snapshot(&refmap) + .expect("snapshot refmap should save") +} + +fn load_latest() -> RefMap { + RefStore::new() + .unwrap() + .load_latest() + .expect("latest snapshot should load") +} + fn seed_skeleton_refmap() -> RefMap { let mut map = RefMap::new(); - let anchor = ref_entry_from_node(&named("group", "Sidebar"), 42, Some("TestApp"), None); + let anchor = ref_entry_from_node( + &named("group", "Sidebar"), + 42, + Some("TestApp"), + None, + None, + &[0], + ); let _ = map.allocate(anchor); - let other = ref_entry_from_node(&named("button", "Toolbar"), 42, Some("TestApp"), None); + let other = ref_entry_from_node( + &named("button", "Toolbar"), + 42, + Some("TestApp"), + None, + None, + &[1], + ); let _ = map.allocate(other); map } @@ -94,7 +117,7 @@ fn drill_opts() -> TreeOptions { #[test] fn test_run_from_ref_returns_subtree_and_persists_refs() { let _guard = HomeGuard::new(); - seed_skeleton_refmap().save().unwrap(); + save_latest(seed_skeleton_refmap()); let mut child_btn = named("button", "Save"); child_btn.children = vec![]; @@ -102,9 +125,9 @@ fn test_run_from_ref_returns_subtree_and_persists_refs() { subtree_root.children = vec![child_btn]; let adapter = StubAdapter::new(subtree_root); - let result = run_from_ref(&adapter, &drill_opts(), "@e1").expect("drill should succeed"); + let result = run_from_ref(&adapter, &drill_opts(), "@e1", None).expect("drill should succeed"); - let on_disk = RefMap::load().unwrap(); + let on_disk = load_latest(); assert_eq!(on_disk.len(), result.refmap.len()); assert!( result.refmap.len() >= 3, @@ -121,16 +144,16 @@ fn test_run_from_ref_returns_subtree_and_persists_refs() { .expect("button child should carry a ref"); let drill_entry = on_disk.get(drill_ref).expect("entry persisted"); assert_eq!(drill_entry.root_ref.as_deref(), Some("@e1")); - assert_eq!(adapter.resolve_calls.get(), 1); + assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1); } #[test] fn test_run_from_ref_stale_root_returns_stale_ref() { let _guard = HomeGuard::new(); - RefMap::new().save().unwrap(); + save_latest(RefMap::new()); let adapter = StubAdapter::new(named("group", "Sidebar")); - let result = run_from_ref(&adapter, &drill_opts(), "@e99"); + let result = run_from_ref(&adapter, &drill_opts(), "@e99", None); let err = match result { Ok(_) => panic!("stale root must error"), Err(e) => e, @@ -151,16 +174,16 @@ fn test_run_from_ref_stale_root_returns_stale_ref() { #[test] fn test_run_from_ref_re_drill_replaces_drill_refs_only() { let _guard = HomeGuard::new(); - seed_skeleton_refmap().save().unwrap(); + save_latest(seed_skeleton_refmap()); let subtree = named("button", "Save"); let adapter = StubAdapter::new(subtree); - let first = run_from_ref(&adapter, &drill_opts(), "@e1").unwrap(); + let first = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); let first_count = first.refmap.len(); let first_button_ref = first.tree.ref_id.clone().expect("button should get a ref"); - let second = run_from_ref(&adapter, &drill_opts(), "@e1").unwrap(); + let second = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); let second_count = second.refmap.len(); let second_button_ref = second.tree.ref_id.clone().expect("button should get a ref"); @@ -172,7 +195,7 @@ fn test_run_from_ref_re_drill_replaces_drill_refs_only() { first_button_ref, second_button_ref, "re-drill should issue a fresh ref id (counter continues)" ); - let on_disk = RefMap::load().unwrap(); + let on_disk = load_latest(); assert!(on_disk.get("@e1").is_some(), "skeleton anchor preserved"); assert!(on_disk.get(&second_button_ref).is_some()); assert!( @@ -184,17 +207,17 @@ fn test_run_from_ref_re_drill_replaces_drill_refs_only() { #[test] fn test_run_from_ref_multiple_drill_downs_accumulate() { let _guard = HomeGuard::new(); - seed_skeleton_refmap().save().unwrap(); + save_latest(seed_skeleton_refmap()); let adapter_one = StubAdapter::new(named("button", "FromE1")); - let first = run_from_ref(&adapter_one, &drill_opts(), "@e1").unwrap(); + let first = run_from_ref(&adapter_one, &drill_opts(), "@e1", None).unwrap(); let from_e1_ref = first.tree.ref_id.clone().expect("first drill ref"); let adapter_two = StubAdapter::new(named("button", "FromE2")); - let second = run_from_ref(&adapter_two, &drill_opts(), "@e2").unwrap(); + let second = run_from_ref(&adapter_two, &drill_opts(), "@e2", None).unwrap(); let from_e2_ref = second.tree.ref_id.clone().expect("second drill ref"); - let on_disk = RefMap::load().unwrap(); + let on_disk = load_latest(); assert!(on_disk.get("@e1").is_some(), "skeleton @e1 preserved"); assert!(on_disk.get("@e2").is_some(), "skeleton @e2 preserved"); let entry_one = on_disk.get(&from_e1_ref).expect("@e1 drill survives"); @@ -216,25 +239,29 @@ fn test_drilldown_refmap_matches_golden_fixture() { 42, Some("Fixture"), None, + None, + &[0], )); seed.allocate(ref_entry_from_node( &named("group", "Toolbar"), 42, Some("Fixture"), None, + None, + &[1], )); - seed.save().unwrap(); + save_latest(seed); let mut sidebar_subtree = named("outline", "Sidebar"); sidebar_subtree.children = vec![named("treeitem", "Recents"), named("treeitem", "Documents")]; let adapter = StubAdapter::new(sidebar_subtree); - let _ = run_from_ref(&adapter, &drill_opts(), "@e1").unwrap(); + let _ = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); let toolbar_subtree = named("button", "Back"); let adapter = StubAdapter::new(toolbar_subtree); - let _ = run_from_ref(&adapter, &drill_opts(), "@e2").unwrap(); + let _ = run_from_ref(&adapter, &drill_opts(), "@e2", None).unwrap(); - let on_disk = RefMap::load().unwrap(); + let on_disk = load_latest(); assert_eq!( on_disk.len(), expected_total, @@ -263,10 +290,10 @@ fn test_drilldown_refmap_matches_golden_fixture() { #[test] fn test_run_from_ref_empty_subtree() { let _guard = HomeGuard::new(); - seed_skeleton_refmap().save().unwrap(); + save_latest(seed_skeleton_refmap()); let adapter = StubAdapter::new(node("group")); - let result = run_from_ref(&adapter, &drill_opts(), "@e1").unwrap(); + let result = run_from_ref(&adapter, &drill_opts(), "@e1", None).unwrap(); assert!(result.tree.children.is_empty()); assert_eq!( @@ -275,90 +302,3 @@ fn test_run_from_ref_empty_subtree() { "no new refs added for empty subtree" ); } - -fn drill_config<'a>( - source_app: Option<&'a str>, - pid: i32, - root_ref_id: &'a str, - interactive_only: bool, - compact: bool, -) -> RefAllocConfig<'a> { - RefAllocConfig { - include_bounds: false, - interactive_only, - compact, - pid, - source_app, - root_ref_id: Some(root_ref_id), - } -} - -#[test] -fn test_drill_alloc_tags_entries() { - let mut btn = node("button"); - btn.name = Some("Submit".into()); - let mut root = node("group"); - root.children = vec![btn]; - - let mut refmap = RefMap::new(); - let config = drill_config(Some("TestApp"), 42, "@e5", false, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); - - assert_eq!(refmap.len(), 1); - let btn_ref = tree.children[0] - .ref_id - .as_deref() - .expect("button should have ref"); - let entry = refmap.get(btn_ref).expect("entry should exist"); - assert_eq!(entry.root_ref.as_deref(), Some("@e5")); - assert_eq!(entry.pid, 42); - assert_eq!(entry.source_app.as_deref(), Some("TestApp")); -} - -#[test] -fn test_drill_alloc_respects_interactive_only() { - let btn = node("button"); - let text = node("statictext"); - let mut root = node("group"); - root.children = vec![btn, text]; - - let mut refmap = RefMap::new(); - let config = drill_config(None, 1, "@e1", true, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); - - assert_eq!(tree.children.len(), 1); - assert_eq!(tree.children[0].role, "button"); -} - -#[test] -fn test_drill_alloc_preserves_truncated_child() { - let mut container = node("group"); - container.name = Some("Sidebar".into()); - container.children_count = Some(4); - let mut root = node("window"); - root.children = vec![container]; - - let mut refmap = RefMap::new(); - let config = drill_config(None, 1, "@e1", true, false); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); - - assert_eq!(tree.children.len(), 1); - assert_eq!(tree.children[0].children_count, Some(4)); -} - -#[test] -fn test_drill_alloc_compact() { - let mut btn = node("button"); - btn.name = Some("OK".into()); - let mut wrapper = node("group"); - wrapper.children = vec![btn]; - let mut root = node("window"); - root.children = vec![wrapper]; - - let mut refmap = RefMap::new(); - let config = drill_config(None, 1, "@e1", false, true); - let tree = ref_alloc::allocate_refs(root, &mut refmap, &config); - - assert_eq!(tree.children.len(), 1); - assert_eq!(tree.children[0].role, "button"); -} diff --git a/crates/core/src/snapshot_tests.rs b/crates/core/src/snapshot_tests.rs index de72f81..e48a58a 100644 --- a/crates/core/src/snapshot_tests.rs +++ b/crates/core/src/snapshot_tests.rs @@ -10,6 +10,7 @@ fn node(role: &str) -> AccessibilityNode { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children_count: None, children: vec![], @@ -23,6 +24,7 @@ fn run_config(compact: bool, interactive_only: bool) -> RefAllocConfig<'static> compact, pid: 1, source_app: Some("Test"), + source_window_title: Some("Test Window"), root_ref_id: None, } } @@ -181,6 +183,7 @@ fn test_skeleton_anchor_suppressed_in_drilldown() { compact: false, pid: 1, source_app: Some("Test"), + source_window_title: Some("Test Window"), root_ref_id: Some("@e3"), }; let result = ref_alloc::allocate_refs(root, &mut refmap, &config); @@ -254,6 +257,7 @@ fn test_skeleton_fixture_matches_golden() { compact: false, pid: 42, source_app: Some("Fixture"), + source_window_title: Some("Fixture Window"), root_ref_id: None, }; let result = ref_alloc::allocate_refs(root, &mut refmap, &config); diff --git a/crates/core/src/window_lookup.rs b/crates/core/src/window_lookup.rs new file mode 100644 index 0000000..e24a0d7 --- /dev/null +++ b/crates/core/src/window_lookup.rs @@ -0,0 +1,27 @@ +use crate::{ + adapter::{PlatformAdapter, WindowFilter}, + error::AppError, + node::WindowInfo, +}; + +pub(crate) fn find_window_for_pid( + pid: i32, + adapter: &dyn PlatformAdapter, +) -> Result { + let filter = WindowFilter { + focused_only: false, + app: None, + }; + let windows = adapter.list_windows(&filter)?; + let mut candidates: Vec<_> = windows.into_iter().filter(|w| w.pid == pid).collect(); + if candidates.is_empty() { + return Err(AppError::invalid_input( + "No window found for this application", + )); + } + let selected = candidates + .iter() + .position(|w| w.is_focused) + .unwrap_or_default(); + Ok(candidates.swap_remove(selected)) +} diff --git a/crates/ffi/cbindgen.toml b/crates/ffi/cbindgen.toml index a35a9bf..2944005 100644 --- a/crates/ffi/cbindgen.toml +++ b/crates/ffi/cbindgen.toml @@ -19,6 +19,7 @@ include = [ "AdModifier", "AdMouseButton", "AdMouseEventKind", + "AdPolicyKind", "AdScreenshotKind", "AdSnapshotSurface", "AdWindowOpKind", diff --git a/crates/ffi/include/agent_desktop.h b/crates/ffi/include/agent_desktop.h index daf8577..4696b97 100644 --- a/crates/ffi/include/agent_desktop.h +++ b/crates/ffi/include/agent_desktop.h @@ -67,6 +67,13 @@ enum AdMouseEventKind { }; typedef int32_t AdMouseEventKind; +enum AdPolicyKind { + AD_POLICY_KIND_HEADLESS = 0, + AD_POLICY_KIND_FOCUS_FALLBACK = 1, + AD_POLICY_KIND_PHYSICAL = 2, +}; +typedef int32_t AdPolicyKind; + enum AdResult { AD_RESULT_OK = 0, AD_RESULT_ERR_PERM_DENIED = -1, @@ -81,6 +88,8 @@ enum AdResult { AD_RESULT_ERR_INVALID_ARGS = -10, AD_RESULT_ERR_NOTIFICATION_NOT_FOUND = -11, AD_RESULT_ERR_INTERNAL = -12, + AD_RESULT_ERR_SNAPSHOT_NOT_FOUND = -13, + AD_RESULT_ERR_POLICY_DENIED = -14, }; typedef int32_t AdResult; @@ -375,6 +384,20 @@ AdResult ad_execute_action(const struct AdAdapter *adapter, const struct AdAction *action, struct AdActionResult *out); +/** + * # Safety + * + * `adapter` must be a non-null pointer returned by `ad_adapter_create`. + * `handle` must be a non-null pointer to a valid `AdNativeHandle`. + * `action` must be a non-null pointer to a valid `AdAction`. + * `out` must be a non-null pointer to an `AdActionResult` to write the result into. + */ +AdResult ad_execute_action_with_policy(const struct AdAdapter *adapter, + const struct AdNativeHandle *handle, + const struct AdAction *action, + int32_t policy, + struct AdActionResult *out); + /** * Releases a handle previously returned by `ad_resolve_element` and * zeroes the caller's struct so accidentally calling this twice is diff --git a/crates/ffi/src/actions/execute.rs b/crates/ffi/src/actions/execute.rs index 8d24642..007bfa9 100644 --- a/crates/ffi/src/actions/execute.rs +++ b/crates/ffi/src/actions/execute.rs @@ -2,9 +2,9 @@ use crate::actions::conversion::action_from_c; use crate::actions::result::action_result_to_c; use crate::error::{self, AdResult}; use crate::ffi_try::trap_panic; -use crate::types::{AdAction, AdActionResult, AdNativeHandle}; +use crate::types::{AdAction, AdActionResult, AdNativeHandle, AdPolicyKind}; use crate::AdAdapter; -use agent_desktop_core::adapter::NativeHandle; +use agent_desktop_core::{action::ActionRequest, adapter::NativeHandle}; /// # Safety /// @@ -18,6 +18,25 @@ pub unsafe extern "C" fn ad_execute_action( handle: *const AdNativeHandle, action: *const AdAction, out: *mut AdActionResult, +) -> AdResult { + unsafe { + ad_execute_action_with_policy(adapter, handle, action, AdPolicyKind::Headless as i32, out) + } +} + +/// # Safety +/// +/// `adapter` must be a non-null pointer returned by `ad_adapter_create`. +/// `handle` must be a non-null pointer to a valid `AdNativeHandle`. +/// `action` must be a non-null pointer to a valid `AdAction`. +/// `out` must be a non-null pointer to an `AdActionResult` to write the result into. +#[no_mangle] +pub unsafe extern "C" fn ad_execute_action_with_policy( + adapter: *const AdAdapter, + handle: *const AdNativeHandle, + action: *const AdAction, + policy: i32, + out: *mut AdActionResult, ) -> AdResult { trap_panic(|| unsafe { crate::pointer_guard::guard_non_null!(out, c"out is null"); @@ -49,7 +68,15 @@ pub unsafe extern "C" fn ad_execute_action( } }; let native_handle = NativeHandle::from_ptr(handle_ref.ptr); - match adapter.inner.execute_action(&native_handle, core_action) { + let Some(policy) = AdPolicyKind::from_c(policy) else { + error::set_last_error(&agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::InvalidArgs, + "invalid policy kind discriminant", + )); + return AdResult::ErrInvalidArgs; + }; + let request = action_request(policy, core_action); + match adapter.inner.execute_action(&native_handle, request) { Ok(result) => { *out = action_result_to_c(&result); AdResult::Ok @@ -61,3 +88,14 @@ pub unsafe extern "C" fn ad_execute_action( } }) } + +fn action_request( + policy: AdPolicyKind, + action: agent_desktop_core::action::Action, +) -> ActionRequest { + match policy { + AdPolicyKind::Headless => ActionRequest::headless(action), + AdPolicyKind::FocusFallback => ActionRequest::focus_fallback(action), + AdPolicyKind::Physical => ActionRequest::physical(action), + } +} diff --git a/crates/ffi/src/actions/resolve.rs b/crates/ffi/src/actions/resolve.rs index 5c50dfc..5d1999f 100644 --- a/crates/ffi/src/actions/resolve.rs +++ b/crates/ffi/src/actions/resolve.rs @@ -66,7 +66,9 @@ pub unsafe extern "C" fn ad_resolve_element( bounds_hash, available_actions: vec![], source_app: None, + source_window_title: None, root_ref: None, + path: Vec::new(), }; match adapter.inner.resolve_element(&core_entry) { Ok(handle) => { diff --git a/crates/ffi/src/adapter.rs b/crates/ffi/src/adapter.rs index 46af26d..457b935 100644 --- a/crates/ffi/src/adapter.rs +++ b/crates/ffi/src/adapter.rs @@ -1,6 +1,6 @@ use crate::error::{self, AdResult}; use crate::ffi_try::{trap_panic, trap_panic_ptr, trap_panic_void}; -use agent_desktop_core::adapter::PlatformAdapter; +use agent_desktop_core::{adapter::PlatformAdapter, PermissionState}; pub struct AdAdapter { pub(crate) inner: Box, @@ -65,9 +65,9 @@ pub unsafe extern "C" fn ad_check_permissions(adapter: *const AdAdapter) -> AdRe trap_panic(|| { crate::pointer_guard::guard_non_null!(adapter, c"adapter is null"); let adapter = unsafe { &*adapter }; - match adapter.inner.check_permissions() { - agent_desktop_core::adapter::PermissionStatus::Granted => AdResult::Ok, - agent_desktop_core::adapter::PermissionStatus::Denied { suggestion } => { + match adapter.inner.permission_report().accessibility { + PermissionState::Granted => AdResult::Ok, + PermissionState::Denied { suggestion } => { error::set_last_error( &agent_desktop_core::error::AdapterError::new( agent_desktop_core::error::ErrorCode::PermDenied, @@ -77,10 +77,32 @@ pub unsafe extern "C" fn ad_check_permissions(adapter: *const AdAdapter) -> AdRe ); AdResult::ErrPermDenied } + PermissionState::NotRequired => AdResult::Ok, + PermissionState::Unknown => unknown_permission_result(adapter.inner.as_ref()), } }) } +fn unknown_permission_result(adapter: &dyn PlatformAdapter) -> AdResult { + let (code, message, suggestion) = if adapter.unknown_accessibility_means_unsupported() { + ( + agent_desktop_core::error::ErrorCode::PlatformNotSupported, + "Accessibility permission state is unknown because this platform adapter does not support permission probing", + "Use a platform adapter with implemented permission probing before executing desktop actions.", + ) + } else { + ( + agent_desktop_core::error::ErrorCode::Internal, + "Accessibility permission state is unknown", + "Run the platform-specific permission report before executing desktop actions.", + ) + }; + let err = + agent_desktop_core::error::AdapterError::new(code, message).with_suggestion(suggestion); + error::set_last_error(&err); + crate::error::last_error_code() +} + #[cfg(test)] mod tests { use super::*; @@ -96,4 +118,54 @@ mod tests { fn test_destroy_null_is_noop() { unsafe { ad_adapter_destroy(std::ptr::null_mut()) }; } + + struct UnknownPermissionAdapter; + + impl PlatformAdapter for UnknownPermissionAdapter { + fn permission_report(&self) -> agent_desktop_core::PermissionReport { + agent_desktop_core::PermissionReport { + accessibility: PermissionState::Unknown, + screen_recording: PermissionState::Unknown, + automation: PermissionState::NotRequired, + } + } + } + + #[test] + fn check_permissions_maps_default_unknown_accessibility_to_platform_unsupported() { + let adapter = AdAdapter { + inner: Box::new(UnknownPermissionAdapter), + }; + + let result = unsafe { ad_check_permissions(&adapter) }; + + assert_eq!(result, AdResult::ErrPlatformNotSupported); + } + + struct AmbiguousPermissionAdapter; + + impl PlatformAdapter for AmbiguousPermissionAdapter { + fn permission_report(&self) -> agent_desktop_core::PermissionReport { + agent_desktop_core::PermissionReport { + accessibility: PermissionState::Unknown, + screen_recording: PermissionState::Unknown, + automation: PermissionState::NotRequired, + } + } + + fn unknown_accessibility_means_unsupported(&self) -> bool { + false + } + } + + #[test] + fn check_permissions_preserves_ambiguous_unknown_accessibility_as_internal() { + let adapter = AdAdapter { + inner: Box::new(AmbiguousPermissionAdapter), + }; + + let result = unsafe { ad_check_permissions(&adapter) }; + + assert_eq!(result, AdResult::ErrInternal); + } } diff --git a/crates/ffi/src/enum_validation.rs b/crates/ffi/src/enum_validation.rs index 6344d03..6491f0a 100644 --- a/crates/ffi/src/enum_validation.rs +++ b/crates/ffi/src/enum_validation.rs @@ -10,7 +10,7 @@ use crate::types::{ AdActionKind, AdDirection, AdImageFormat, AdModifier, AdMouseButton, AdMouseEventKind, - AdScreenshotKind, AdSnapshotSurface, AdWindowOpKind, + AdPolicyKind, AdScreenshotKind, AdSnapshotSurface, AdWindowOpKind, }; macro_rules! try_from_c_enum { @@ -37,6 +37,12 @@ try_from_c_enum! { } } +try_from_c_enum! { + AdPolicyKind { + Headless = 0, FocusFallback = 1, Physical = 2, + } +} + try_from_c_enum! { AdDirection { Up = 0, Down = 1, Left = 2, Right = 3, @@ -114,6 +120,14 @@ mod tests { assert!(AdDirection::from_c(4).is_none()); } + #[test] + fn test_policy_kind_valid_range() { + for raw in 0..=2 { + assert!(AdPolicyKind::from_c(raw).is_some()); + } + assert!(AdPolicyKind::from_c(3).is_none()); + } + #[test] fn test_modifier_valid_range() { for raw in 0..=3 { @@ -170,6 +184,7 @@ mod tests { for raw in [-1_000_000_i32, -1, 0, 1, 20, 21, 999, i32::MAX, i32::MIN] { let _ = AdActionKind::from_c(raw); let _ = AdDirection::from_c(raw); + let _ = AdPolicyKind::from_c(raw); let _ = AdModifier::from_c(raw); let _ = AdMouseButton::from_c(raw); let _ = AdMouseEventKind::from_c(raw); diff --git a/crates/ffi/src/error.rs b/crates/ffi/src/error.rs index d32998c..8fbffcf 100644 --- a/crates/ffi/src/error.rs +++ b/crates/ffi/src/error.rs @@ -16,6 +16,8 @@ const fn error_code_variant_count() -> usize { ErrorCode::Timeout, ErrorCode::InvalidArgs, ErrorCode::NotificationNotFound, + ErrorCode::SnapshotNotFound, + ErrorCode::PolicyDenied, ErrorCode::Internal, ]; let mut i = 0; @@ -40,6 +42,8 @@ const fn ad_result_error_variant_count() -> usize { AdResult::ErrInvalidArgs, AdResult::ErrNotificationNotFound, AdResult::ErrInternal, + AdResult::ErrSnapshotNotFound, + AdResult::ErrPolicyDenied, ]; let mut count = 0; let mut i = 0; @@ -50,10 +54,6 @@ const fn ad_result_error_variant_count() -> usize { count } -// Compile-time parity check: every core `ErrorCode` variant must have a -// matching `AdResult::Err*`. Adding a variant to either enum without -// updating the other fails the build with the message below — preferable -// to the silent-drop we'd otherwise see at the FFI boundary. const _: () = assert!( error_code_variant_count() == ad_result_error_variant_count(), "ErrorCode variants must match AdResult error-code variants one-to-one" @@ -75,8 +75,25 @@ pub enum AdResult { ErrInvalidArgs = -10, ErrNotificationNotFound = -11, ErrInternal = -12, + ErrSnapshotNotFound = -13, + ErrPolicyDenied = -14, } +const _: () = assert!(AdResult::ErrPermDenied as i32 == -1); +const _: () = assert!(AdResult::ErrElementNotFound as i32 == -2); +const _: () = assert!(AdResult::ErrAppNotFound as i32 == -3); +const _: () = assert!(AdResult::ErrActionFailed as i32 == -4); +const _: () = assert!(AdResult::ErrActionNotSupported as i32 == -5); +const _: () = assert!(AdResult::ErrStaleRef as i32 == -6); +const _: () = assert!(AdResult::ErrWindowNotFound as i32 == -7); +const _: () = assert!(AdResult::ErrPlatformNotSupported as i32 == -8); +const _: () = assert!(AdResult::ErrTimeout as i32 == -9); +const _: () = assert!(AdResult::ErrInvalidArgs as i32 == -10); +const _: () = assert!(AdResult::ErrNotificationNotFound as i32 == -11); +const _: () = assert!(AdResult::ErrInternal as i32 == -12); +const _: () = assert!(AdResult::ErrSnapshotNotFound as i32 == -13); +const _: () = assert!(AdResult::ErrPolicyDenied as i32 == -14); + enum MessageSource { Owned(CString), Static(&'static CStr), @@ -126,6 +143,8 @@ fn error_code_to_result(code: &ErrorCode) -> AdResult { ErrorCode::InvalidArgs => AdResult::ErrInvalidArgs, ErrorCode::NotificationNotFound => AdResult::ErrNotificationNotFound, ErrorCode::Internal => AdResult::ErrInternal, + ErrorCode::SnapshotNotFound => AdResult::ErrSnapshotNotFound, + ErrorCode::PolicyDenied => AdResult::ErrPolicyDenied, } } @@ -249,76 +268,5 @@ pub extern "C" fn ad_last_error_platform_detail() -> *const c_char { } #[cfg(test)] -mod tests { - use super::*; - - fn last_error_message_str() -> Option { - LAST_ERROR.with(|cell| cell.borrow().as_ref().map(|e| e.message.to_owned_string())) - } - - fn last_error_suggestion_str() -> Option { - LAST_ERROR.with(|cell| { - cell.borrow().as_ref().and_then(|e| { - e.suggestion - .as_ref() - .map(|s| s.to_string_lossy().into_owned()) - }) - }) - } - - fn last_error_platform_detail_str() -> Option { - LAST_ERROR.with(|cell| { - cell.borrow().as_ref().and_then(|e| { - e.platform_detail - .as_ref() - .map(|s| s.to_string_lossy().into_owned()) - }) - }) - } - - #[test] - fn test_no_error_initially() { - clear_last_error(); - assert!(last_error_message_str().is_none()); - } - - #[test] - fn test_set_and_get_error() { - let err = AdapterError::new(ErrorCode::ElementNotFound, "element @e5 gone") - .with_suggestion("run snapshot"); - set_last_error(&err); - assert_eq!(last_error_code(), AdResult::ErrElementNotFound); - assert_eq!(last_error_message_str().unwrap(), "element @e5 gone"); - assert_eq!(last_error_suggestion_str().unwrap(), "run snapshot"); - assert!(last_error_platform_detail_str().is_none()); - } - - #[test] - fn test_clear_error() { - let err = AdapterError::internal("oops"); - set_last_error(&err); - clear_last_error(); - assert!(last_error_message_str().is_none()); - } - - #[test] - fn test_error_isolation_across_threads() { - clear_last_error(); - let err = AdapterError::internal("thread1"); - set_last_error(&err); - - let handle = std::thread::spawn(|| last_error_message_str().is_none()); - assert!(handle.join().unwrap(), "other thread should see no error"); - assert_eq!(last_error_message_str().unwrap(), "thread1"); - } - - #[test] - fn test_interior_nul_falls_back_to_static() { - let err = AdapterError::new(ErrorCode::Internal, "before\0after"); - set_last_error(&err); - assert_eq!( - last_error_message_str().unwrap(), - "(message contained null byte)" - ); - } -} +#[path = "error_tests.rs"] +mod tests; diff --git a/crates/ffi/src/error_tests.rs b/crates/ffi/src/error_tests.rs new file mode 100644 index 0000000..806fe07 --- /dev/null +++ b/crates/ffi/src/error_tests.rs @@ -0,0 +1,78 @@ +use super::*; + +fn last_error_message_str() -> Option { + LAST_ERROR.with(|cell| cell.borrow().as_ref().map(|e| e.message.to_owned_string())) +} + +fn last_error_suggestion_str() -> Option { + LAST_ERROR.with(|cell| { + cell.borrow().as_ref().and_then(|e| { + e.suggestion + .as_ref() + .map(|s| s.to_string_lossy().into_owned()) + }) + }) +} + +fn last_error_platform_detail_str() -> Option { + LAST_ERROR.with(|cell| { + cell.borrow().as_ref().and_then(|e| { + e.platform_detail + .as_ref() + .map(|s| s.to_string_lossy().into_owned()) + }) + }) +} + +#[test] +fn test_no_error_initially() { + clear_last_error(); + assert!(last_error_message_str().is_none()); +} + +#[test] +fn result_discriminants_preserve_existing_abi_values() { + assert_eq!(AdResult::ErrInternal as i32, -12); + assert_eq!(AdResult::ErrSnapshotNotFound as i32, -13); + assert_eq!(AdResult::ErrPolicyDenied as i32, -14); +} + +#[test] +fn test_set_and_get_error() { + let err = AdapterError::new(ErrorCode::ElementNotFound, "element @e5 gone") + .with_suggestion("run snapshot"); + set_last_error(&err); + assert_eq!(last_error_code(), AdResult::ErrElementNotFound); + assert_eq!(last_error_message_str().unwrap(), "element @e5 gone"); + assert_eq!(last_error_suggestion_str().unwrap(), "run snapshot"); + assert!(last_error_platform_detail_str().is_none()); +} + +#[test] +fn test_clear_error() { + let err = AdapterError::internal("oops"); + set_last_error(&err); + clear_last_error(); + assert!(last_error_message_str().is_none()); +} + +#[test] +fn test_error_isolation_across_threads() { + clear_last_error(); + let err = AdapterError::internal("thread1"); + set_last_error(&err); + + let handle = std::thread::spawn(|| last_error_message_str().is_none()); + assert!(handle.join().unwrap(), "other thread should see no error"); + assert_eq!(last_error_message_str().unwrap(), "thread1"); +} + +#[test] +fn test_interior_nul_falls_back_to_static() { + let err = AdapterError::new(ErrorCode::Internal, "before\0after"); + set_last_error(&err); + assert_eq!( + last_error_message_str().unwrap(), + "(message contained null byte)" + ); +} diff --git a/crates/ffi/src/lib.rs b/crates/ffi/src/lib.rs index 9c374a2..a22ccce 100644 --- a/crates/ffi/src/lib.rs +++ b/crates/ffi/src/lib.rs @@ -84,6 +84,7 @@ pub use types::notification_filter::AdNotificationFilter; pub use types::notification_info::AdNotificationInfo; pub use types::notification_list::AdNotificationList; pub use types::point::AdPoint; +pub use types::policy_kind::AdPolicyKind; pub use types::rect::AdRect; pub use types::ref_entry::AdRefEntry; pub use types::screenshot_kind::AdScreenshotKind; diff --git a/crates/ffi/src/observation/find.rs b/crates/ffi/src/observation/find.rs index 2f24459..bb3b662 100644 --- a/crates/ffi/src/observation/find.rs +++ b/crates/ffi/src/observation/find.rs @@ -100,7 +100,9 @@ pub unsafe extern "C" fn ad_find( bounds_hash, available_actions: Vec::new(), source_app: None, + source_window_title: Some(core_win.title.clone()), root_ref: None, + path: Vec::new(), }; match adapter.inner.resolve_element(&ref_entry) { Ok(handle) => { diff --git a/crates/ffi/src/observation/is.rs b/crates/ffi/src/observation/is.rs index 5494815..6eabf22 100644 --- a/crates/ffi/src/observation/is.rs +++ b/crates/ffi/src/observation/is.rs @@ -171,6 +171,7 @@ mod tests { description: None, hint: None, states: states.iter().map(|s| s.to_string()).collect(), + available_actions: vec![], bounds: None, children: vec![], children_count: None, diff --git a/crates/ffi/src/observation/walk.rs b/crates/ffi/src/observation/walk.rs index 95a5760..10d92df 100644 --- a/crates/ffi/src/observation/walk.rs +++ b/crates/ffi/src/observation/walk.rs @@ -65,6 +65,7 @@ mod tests { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children: vec![], children_count: None, diff --git a/crates/ffi/src/tree/flatten.rs b/crates/ffi/src/tree/flatten.rs index 4611d50..ff1c701 100644 --- a/crates/ffi/src/tree/flatten.rs +++ b/crates/ffi/src/tree/flatten.rs @@ -126,6 +126,7 @@ mod tests { description: None, hint: None, states: vec![], + available_actions: vec![], bounds: None, children: vec![], children_count: None, diff --git a/crates/ffi/src/types/mod.rs b/crates/ffi/src/types/mod.rs index a1ffd95..fc095e6 100644 --- a/crates/ffi/src/types/mod.rs +++ b/crates/ffi/src/types/mod.rs @@ -21,6 +21,7 @@ pub mod notification_filter; pub mod notification_info; pub mod notification_list; pub mod point; +pub mod policy_kind; pub mod rect; pub mod ref_entry; pub mod screenshot_kind; @@ -58,6 +59,7 @@ pub use notification_filter::AdNotificationFilter; pub use notification_info::AdNotificationInfo; pub use notification_list::AdNotificationList; pub use point::AdPoint; +pub use policy_kind::AdPolicyKind; pub use rect::AdRect; pub use ref_entry::AdRefEntry; pub use screenshot_kind::AdScreenshotKind; diff --git a/crates/ffi/src/types/policy_kind.rs b/crates/ffi/src/types/policy_kind.rs new file mode 100644 index 0000000..94fecda --- /dev/null +++ b/crates/ffi/src/types/policy_kind.rs @@ -0,0 +1,19 @@ +#[repr(i32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdPolicyKind { + Headless = 0, + FocusFallback = 1, + Physical = 2, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discriminants_are_abi_stable() { + assert_eq!(AdPolicyKind::Headless as i32, 0); + assert_eq!(AdPolicyKind::FocusFallback as i32, 1); + assert_eq!(AdPolicyKind::Physical as i32, 2); + } +} diff --git a/crates/ffi/tests/c_abi_actions.rs b/crates/ffi/tests/c_abi_actions.rs new file mode 100644 index 0000000..12bd6e6 --- /dev/null +++ b/crates/ffi/tests/c_abi_actions.rs @@ -0,0 +1,62 @@ +mod common; + +use common::{ + ad_execute_action, ad_execute_action_with_policy, default_action, with_adapter, AdActionResult, + AdNativeHandle, AdPolicyKind, AdResult, +}; + +#[test] +fn enum_fuzz_invalid_discriminant_rejected() { + with_adapter(|adapter| unsafe { + let mut action = default_action(); + action.kind = i32::MAX; + let handle = AdNativeHandle { + ptr: std::ptr::null(), + }; + let mut out: AdActionResult = std::mem::zeroed(); + let rc = ad_execute_action(adapter, &handle, &action, &mut out); + assert!( + matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), + "arbitrary enum bit pattern must be rejected, got {:?}", + rc + ); + }); +} + +#[test] +fn invalid_policy_discriminant_rejected_without_ub() { + with_adapter(|adapter| unsafe { + let action = default_action(); + let handle = AdNativeHandle { + ptr: std::ptr::dangling::(), + }; + let mut out: AdActionResult = std::mem::zeroed(); + let rc = ad_execute_action_with_policy( + adapter, + &handle, + &action, + AdPolicyKind::Physical as i32 + 1, + &mut out, + ); + assert!(matches!( + rc, + AdResult::ErrInvalidArgs | AdResult::ErrInternal + )); + }); +} + +#[test] +fn execute_action_rejects_null_handle_ptr() { + with_adapter(|adapter| unsafe { + let action = default_action(); + let handle = AdNativeHandle { + ptr: std::ptr::null(), + }; + let mut out: AdActionResult = std::mem::zeroed(); + let rc = ad_execute_action(adapter, &handle, &action, &mut out); + assert!(matches!( + rc, + AdResult::ErrInvalidArgs | AdResult::ErrInternal + )); + }); +} diff --git a/crates/ffi/tests/c_abi_harness.rs b/crates/ffi/tests/c_abi_harness.rs deleted file mode 100644 index 26e6a8c..0000000 --- a/crates/ffi/tests/c_abi_harness.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! C-ABI contract harness. -//! -//! Drives the FFI exactly as a C program would — raw `extern "C"` -//! declarations with `#[allow(improper_ctypes)]` over the opaque -//! handle types, no high-level Rust access. Exercises bug classes the -//! inline `#[cfg(test)]` modules can't reach on their own: -//! -//! 1. Struct layouts that a consumer memcpy-copies (AdRect, AdAction). -//! 2. Enum fuzzing — `int32_t` bit patterns written into enum-typed -//! fields must not UB, must return `AD_RESULT_ERR_INVALID_ARGS`. -//! 3. Null tolerance in the free-* family and accessor family. -//! 4. Interior-NUL inputs funnel through `string_to_c_lossy` without -//! returning null. -//! 5. List handle lifecycle (count on null, _free on null, _get OOB). - -#![allow(improper_ctypes)] - -use agent_desktop_ffi::error::AdResult; -use agent_desktop_ffi::{ - AdAction, AdActionResult, AdAdapter, AdAppList, AdDirection, AdDragParams, AdFindQuery, - AdKeyCombo, AdNativeHandle, AdPoint, AdRect, AdRefEntry, AdScrollParams, AdWindowInfo, - AdWindowList, -}; -use std::ffi::CStr; -use std::os::raw::c_char; - -extern "C" { - fn ad_adapter_create() -> *mut AdAdapter; - fn ad_adapter_destroy(adapter: *mut AdAdapter); - fn ad_check_permissions(adapter: *const AdAdapter) -> AdResult; - - fn ad_last_error_code() -> AdResult; - fn ad_last_error_message() -> *const c_char; - - fn ad_list_apps(adapter: *const AdAdapter, out: *mut *mut AdAppList) -> AdResult; - fn ad_app_list_count(list: *const AdAppList) -> u32; - fn ad_app_list_get(list: *const AdAppList, index: u32) -> *const u8; - fn ad_app_list_free(list: *mut AdAppList); - - fn ad_list_windows( - adapter: *const AdAdapter, - app_filter: *const c_char, - focused_only: bool, - out: *mut *mut AdWindowList, - ) -> AdResult; - fn ad_window_list_count(list: *const AdWindowList) -> u32; - fn ad_window_list_free(list: *mut AdWindowList); - - fn ad_launch_app( - adapter: *const AdAdapter, - id: *const c_char, - timeout_ms: u64, - out: *mut AdWindowInfo, - ) -> AdResult; - - fn ad_execute_action( - adapter: *const AdAdapter, - handle: *const AdNativeHandle, - action: *const AdAction, - out: *mut AdActionResult, - ) -> AdResult; - - fn ad_find( - adapter: *const AdAdapter, - win: *const AdWindowInfo, - query: *const AdFindQuery, - out: *mut AdNativeHandle, - ) -> AdResult; - - fn ad_free_handle(adapter: *const AdAdapter, handle: *mut AdNativeHandle) -> AdResult; - - fn ad_resolve_element( - adapter: *const AdAdapter, - entry: *const AdRefEntry, - out: *mut AdNativeHandle, - ) -> AdResult; -} - -fn with_adapter(body: F) { - unsafe { - let adapter = ad_adapter_create(); - assert!(!adapter.is_null(), "ad_adapter_create must not return null"); - body(adapter); - ad_adapter_destroy(adapter); - } -} - -fn default_scroll() -> AdScrollParams { - AdScrollParams { - direction: AdDirection::Down as i32, - amount: 0, - } -} - -fn default_key() -> AdKeyCombo { - AdKeyCombo { - key: std::ptr::null(), - modifiers: std::ptr::null(), - modifier_count: 0, - } -} - -fn default_drag() -> AdDragParams { - AdDragParams { - from: AdPoint { x: 0.0, y: 0.0 }, - to: AdPoint { x: 0.0, y: 0.0 }, - duration_ms: 0, - } -} - -#[test] -fn rect_and_point_layouts_are_memcpyable() { - // A C consumer that does { AdRect r; memcpy(&r, src, sizeof(r)); } must - // see the same field values Rust wrote. Plain #[repr(C)] without - // padding games makes this a byte copy. - let rect = AdRect { - x: 1.25, - y: -2.5, - width: 640.0, - height: 480.0, - }; - let copied = unsafe { std::ptr::read(&rect as *const AdRect) }; - assert_eq!(copied.x, 1.25); - assert_eq!(copied.y, -2.5); - assert_eq!(copied.width, 640.0); - assert_eq!(copied.height, 480.0); - - let point = AdPoint { x: 3.0, y: 4.0 }; - let copied = unsafe { std::ptr::read(&point as *const AdPoint) }; - assert_eq!(copied.x, 3.0); - assert_eq!(copied.y, 4.0); -} - -#[test] -fn enum_fuzz_invalid_discriminant_rejected() { - // AdAction.kind is `i32` — a buggy C caller can legally stuff any - // value in here. Must not UB the Rust side; the validator should - // surface AD_RESULT_ERR_INVALID_ARGS before any adapter code runs. - with_adapter(|adapter| unsafe { - let mut action: AdAction = std::mem::zeroed(); - action.kind = i32::MAX; - action.scroll = default_scroll(); - action.key = default_key(); - action.drag = default_drag(); - action.text = std::ptr::null(); - - let handle = AdNativeHandle { - ptr: std::ptr::null(), - }; - let mut out: AdActionResult = std::mem::zeroed(); - let rc = ad_execute_action(adapter, &handle, &action, &mut out); - // Either the enum validator rejects (expected) or the cargo-test - // worker thread trips the macOS main-thread assert and returns - // ErrInternal. Both prove the absence of UB; what matters is that - // the arbitrary bit pattern never results in AD_RESULT_OK. - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "arbitrary enum bit pattern must be rejected, got {:?}", - rc - ); - }); -} - -#[test] -fn null_adapter_rejected_without_ub() { - unsafe { - let mut list: *mut AdAppList = std::ptr::null_mut(); - let rc = ad_list_apps(std::ptr::null(), &mut list); - // On cargo-test worker threads the macOS main-thread guard - // fires first (ErrInternal); on the main thread the null-adapter - // guard wins (ErrInvalidArgs). Either way: no dereference, no UB. - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); - assert!(list.is_null(), "out-param must stay null on failure"); - - let rc2 = ad_check_permissions(std::ptr::null()); - // ad_check_permissions has no main-thread guard — null adapter - // must hit the null-check and return InvalidArgs deterministically. - assert_eq!(rc2, AdResult::ErrInvalidArgs); - } -} - -#[test] -fn dirty_out_param_is_cleared_before_early_return_on_worker_thread() { - // Regression for todo 006: prior shape ran require_main_thread() - // *before* zeroing *out, so a worker-thread early-return left - // whatever the caller had in the struct. A follow-up free on that - // stale pointer would double-free. This test seeds the out slot - // with fake pointers that *must* be cleared before the fn returns. - with_adapter(|adapter| unsafe { - let fake_ptr = 0xDEAD_BEEF as *mut AdAppList; - let mut list: *mut AdAppList = fake_ptr; - let rc = ad_list_apps(adapter, &mut list); - // Either the main-thread guard (ErrInternal on worker) or a - // successful call zeroed the slot before anything else. - if rc != AdResult::Ok { - assert!( - list.is_null(), - "dirty out-param must be zeroed before early return, got {:?}", - list - ); - } - }); -} - -#[test] -fn invalid_utf8_filter_rejected_not_silently_widened() { - // Regression for todo 010: prior c_to_string conflated null with - // invalid UTF-8, so a non-null buffer with bogus bytes in the - // app_filter slot would be treated as "no filter" and widen - // ad_list_windows to every app on the system. Must now fail closed. - with_adapter(|adapter| unsafe { - let bad: [u8; 2] = [0xC3, 0x00]; - let mut list: *mut AdWindowList = std::ptr::null_mut(); - let rc = ad_list_windows(adapter, bad.as_ptr() as *const c_char, false, &mut list); - // Main-thread guard (ErrInternal on worker) or UTF-8 rejection - // (ErrInvalidArgs) — either way we do NOT produce a list by - // silently treating bad bytes as "no filter". - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); - assert!(list.is_null()); - }); -} - -#[test] -fn null_out_param_rejected_before_write() { - with_adapter(|adapter| unsafe { - let rc = ad_list_apps(adapter, std::ptr::null_mut()); - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); - }); -} - -#[test] -fn null_tolerance_on_list_accessors_and_free() { - unsafe { - assert_eq!(ad_app_list_count(std::ptr::null()), 0); - assert!(ad_app_list_get(std::ptr::null(), 0).is_null()); - ad_app_list_free(std::ptr::null_mut()); - - assert_eq!(ad_window_list_count(std::ptr::null()), 0); - ad_window_list_free(std::ptr::null_mut()); - } -} - -#[test] -fn list_handle_lifecycle_roundtrip() { - with_adapter(|adapter| unsafe { - let mut list: *mut AdAppList = std::ptr::null_mut(); - let rc = ad_list_apps(adapter, &mut list); - // On CI without accessibility permission this may return PermDenied; - // both outcomes preserve the contract we're testing (no UB, valid - // pointer-or-null). - if rc == AdResult::Ok { - assert!(!list.is_null()); - let count = ad_app_list_count(list); - // Request an index past the end — must return null, not segfault. - assert!(ad_app_list_get(list, count).is_null()); - ad_app_list_free(list); - } else { - assert!(list.is_null(), "failed list call must leave out null"); - let msg_ptr = ad_last_error_message(); - assert!(!msg_ptr.is_null()); - let _ = CStr::from_ptr(msg_ptr).to_string_lossy(); - assert_eq!(ad_last_error_code(), rc); - } - }); -} - -#[test] -fn list_windows_focused_only_runs() { - with_adapter(|adapter| unsafe { - let mut list: *mut AdWindowList = std::ptr::null_mut(); - let rc = ad_list_windows(adapter, std::ptr::null(), true, &mut list); - if rc == AdResult::Ok { - assert!(!list.is_null()); - let _ = ad_window_list_count(list); - ad_window_list_free(list); - } else { - assert!(list.is_null()); - } - }); -} - -#[test] -fn invalid_utf8_app_id_rejected() { - with_adapter(|adapter| unsafe { - // A mid-byte of a multi-byte UTF-8 sequence, not valid on its own. - let bad: [u8; 2] = [0xC3, 0]; - let mut out: AdWindowInfo = std::mem::zeroed(); - let rc = ad_launch_app(adapter, bad.as_ptr() as *const c_char, 0, &mut out); - // Either the UTF-8 check (ErrInvalidArgs) or the macOS main-thread - // guard (ErrInternal — cargo tests run on worker threads) rejects - // the call. Either way no UB occurred. - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "must reject without UB, got {:?}", - rc - ); - }); -} - -#[test] -fn find_returns_not_found_on_empty_query_against_no_window() { - with_adapter(|adapter| unsafe { - let bad_win: AdWindowInfo = std::mem::zeroed(); - let query = AdFindQuery { - role: std::ptr::null(), - name_substring: std::ptr::null(), - value_substring: std::ptr::null(), - }; - let mut handle = AdNativeHandle { - ptr: std::ptr::null(), - }; - let rc = ad_find(adapter, &bad_win, &query, &mut handle); - // A zeroed window has null id/title → InvalidArgs. On cargo-test - // worker threads the main-thread assert trips first and returns - // ErrInternal. Either outcome means the FFI refused a malformed - // input without UB. - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "zeroed window must not succeed, got {:?}", - rc - ); - }); -} - -#[test] -fn free_handle_null_is_noop() { - with_adapter(|adapter| unsafe { - let mut handle = AdNativeHandle { - ptr: std::ptr::null(), - }; - let rc = ad_free_handle(adapter, &mut handle); - assert_eq!(rc, AdResult::Ok); - assert!(handle.ptr.is_null()); - - let rc2 = ad_free_handle(adapter, std::ptr::null_mut()); - assert_eq!(rc2, AdResult::Ok); - }); -} - -#[cfg(not(target_os = "macos"))] -#[test] -fn free_handle_zeroes_ptr_so_double_free_is_noop() { - // macOS is excluded: ad_free_handle invokes CFRelease on the - // underlying pointer, and a fabricated "fake live" pointer will - // SIGBUS before we can observe the zeroing. On Windows/Linux - // release_handle is NotSupported (no platform call), so the zeroing - // contract is safely observable with a fake pointer. - with_adapter(|adapter| unsafe { - let fake_live_ptr = 0x1234 as *const std::ffi::c_void; - let mut handle = AdNativeHandle { ptr: fake_live_ptr }; - - let _ = ad_free_handle(adapter, &mut handle); - assert!(handle.ptr.is_null()); - - let rc = ad_free_handle(adapter, &mut handle); - assert_eq!(rc, AdResult::Ok); - }); -} - -#[test] -fn resolve_element_rejects_invalid_utf8_name() { - // Regression for todo 004: prior c_to_string(entry.name) conflated - // null with invalid UTF-8, so a non-null buffer with bogus bytes in - // the `name` slot was treated as "no name filter" and widened the - // re-resolution match. Must now fail closed with InvalidArgs. - with_adapter(|adapter| unsafe { - let role = std::ffi::CString::new("button").unwrap(); - let bad_name: [u8; 2] = [0xC3, 0x00]; // partial multi-byte + NUL - let entry = AdRefEntry { - pid: 0, - role: role.as_ptr(), - name: bad_name.as_ptr() as *const c_char, - bounds_hash: 0, - has_bounds_hash: false, - }; - let mut out = AdNativeHandle { - ptr: std::ptr::null(), - }; - let rc = ad_resolve_element(adapter, &entry, &mut out); - // Either the UTF-8 check wins (ErrInvalidArgs) or the main-thread - // guard wins on worker threads (ErrInternal). Either way, no - // silent widening, no UB, and the out-handle stays null. - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "must reject without UB, got {:?}", - rc - ); - assert!(out.ptr.is_null()); - }); -} - -#[test] -fn execute_action_rejects_null_handle_ptr() { - with_adapter(|adapter| unsafe { - let action = AdAction { - kind: 0, - text: std::ptr::null(), - scroll: AdScrollParams { - direction: 0, - amount: 0, - }, - key: AdKeyCombo { - key: std::ptr::null(), - modifiers: std::ptr::null(), - modifier_count: 0, - }, - drag: AdDragParams { - from: AdPoint { x: 0.0, y: 0.0 }, - to: AdPoint { x: 0.0, y: 0.0 }, - duration_ms: 0, - }, - }; - let handle = AdNativeHandle { - ptr: std::ptr::null(), - }; - let mut out: AdActionResult = std::mem::zeroed(); - let rc = ad_execute_action(adapter, &handle, &action, &mut out); - // Main-thread guard or null-ptr guard wins; both avoid UB and - // keep the struct-level handle accepted while rejecting the - // inner null pointer. - assert!(matches!( - rc, - AdResult::ErrInvalidArgs | AdResult::ErrInternal - )); - }); -} - -#[test] -fn last_error_survives_successful_calls() { - with_adapter(|adapter| unsafe { - let mut out: AdWindowInfo = std::mem::zeroed(); - let rc = ad_launch_app(adapter, std::ptr::null(), 0, &mut out); - // Worker-thread cargo tests hit the main-thread guard first, - // producing ErrInternal. Main-thread runs would see - // ErrInvalidArgs. What we need is that *some* failure populates - // last-error and its pointer stays stable across the success - // calls that follow. - assert!( - matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), - "must fail, got {:?}", - rc - ); - let msg_ptr = ad_last_error_message(); - assert!(!msg_ptr.is_null()); - - // Use accessors that are guaranteed to succeed and never set - // last-error. ad_check_permissions is NOT safe here — it can - // return ErrPermDenied on macOS without Accessibility permission, - // which overwrites the TLS error slot we're testing. - for _ in 0..5 { - let _ = ad_app_list_count(std::ptr::null()); - let _ = ad_window_list_count(std::ptr::null()); - } - - let after = ad_last_error_message(); - assert_eq!(msg_ptr, after); - assert_eq!(ad_last_error_code(), rc); - }); -} diff --git a/crates/ffi/tests/c_abi_inputs.rs b/crates/ffi/tests/c_abi_inputs.rs new file mode 100644 index 0000000..0af892f --- /dev/null +++ b/crates/ffi/tests/c_abi_inputs.rs @@ -0,0 +1,59 @@ +mod common; + +use common::{ + ad_launch_app, ad_list_windows, ad_resolve_element, c_char, with_adapter, AdNativeHandle, + AdRefEntry, AdResult, AdWindowInfo, AdWindowList, +}; + +#[test] +fn invalid_utf8_filter_rejected_not_silently_widened() { + with_adapter(|adapter| unsafe { + let bad: [u8; 2] = [0xC3, 0x00]; + let mut list: *mut AdWindowList = std::ptr::null_mut(); + let rc = ad_list_windows(adapter, bad.as_ptr() as *const c_char, false, &mut list); + assert!(matches!( + rc, + AdResult::ErrInvalidArgs | AdResult::ErrInternal + )); + assert!(list.is_null()); + }); +} + +#[test] +fn invalid_utf8_app_id_rejected() { + with_adapter(|adapter| unsafe { + let bad: [u8; 2] = [0xC3, 0]; + let mut out: AdWindowInfo = std::mem::zeroed(); + let rc = ad_launch_app(adapter, bad.as_ptr() as *const c_char, 0, &mut out); + assert!( + matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), + "must reject without UB, got {:?}", + rc + ); + }); +} + +#[test] +fn resolve_element_rejects_invalid_utf8_name() { + with_adapter(|adapter| unsafe { + let role = std::ffi::CString::new("button").unwrap(); + let bad_name: [u8; 2] = [0xC3, 0x00]; + let entry = AdRefEntry { + pid: 0, + role: role.as_ptr(), + name: bad_name.as_ptr() as *const c_char, + bounds_hash: 0, + has_bounds_hash: false, + }; + let mut out = AdNativeHandle { + ptr: std::ptr::null(), + }; + let rc = ad_resolve_element(adapter, &entry, &mut out); + assert!( + matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), + "must reject without UB, got {:?}", + rc + ); + assert!(out.ptr.is_null()); + }); +} diff --git a/crates/ffi/tests/c_abi_layout.rs b/crates/ffi/tests/c_abi_layout.rs new file mode 100644 index 0000000..d74d0b0 --- /dev/null +++ b/crates/ffi/tests/c_abi_layout.rs @@ -0,0 +1,23 @@ +mod common; + +use common::{AdPoint, AdRect}; + +#[test] +fn rect_and_point_layouts_are_memcpyable() { + let rect = AdRect { + x: 1.25, + y: -2.5, + width: 640.0, + height: 480.0, + }; + let copied = unsafe { std::ptr::read(&rect as *const AdRect) }; + assert_eq!(copied.x, 1.25); + assert_eq!(copied.y, -2.5); + assert_eq!(copied.width, 640.0); + assert_eq!(copied.height, 480.0); + + let point = AdPoint { x: 3.0, y: 4.0 }; + let copied = unsafe { std::ptr::read(&point as *const AdPoint) }; + assert_eq!(copied.x, 3.0); + assert_eq!(copied.y, 4.0); +} diff --git a/crates/ffi/tests/c_abi_lifecycle.rs b/crates/ffi/tests/c_abi_lifecycle.rs new file mode 100644 index 0000000..2af3865 --- /dev/null +++ b/crates/ffi/tests/c_abi_lifecycle.rs @@ -0,0 +1,176 @@ +mod common; + +use common::{ + ad_adapter_create, ad_adapter_destroy, ad_app_list_count, ad_app_list_free, ad_app_list_get, + ad_check_permissions, ad_find, ad_free_handle, ad_last_error_code, ad_last_error_message, + ad_list_apps, ad_list_windows, ad_window_list_count, ad_window_list_free, with_adapter, + AdAppList, AdFindQuery, AdNativeHandle, AdResult, AdWindowInfo, AdWindowList, CStr, +}; + +#[test] +fn null_adapter_rejected_without_ub() { + unsafe { + let mut list: *mut AdAppList = std::ptr::null_mut(); + let rc = ad_list_apps(std::ptr::null(), &mut list); + assert!(matches!( + rc, + AdResult::ErrInvalidArgs | AdResult::ErrInternal + )); + assert!(list.is_null(), "out-param must stay null on failure"); + + let rc2 = ad_check_permissions(std::ptr::null()); + assert_eq!(rc2, AdResult::ErrInvalidArgs); + } +} + +#[test] +fn null_out_param_rejected_before_write() { + with_adapter(|adapter| unsafe { + let rc = ad_list_apps(adapter, std::ptr::null_mut()); + assert!(matches!( + rc, + AdResult::ErrInvalidArgs | AdResult::ErrInternal + )); + }); +} + +#[test] +fn null_tolerance_on_list_accessors_and_free() { + unsafe { + assert_eq!(ad_app_list_count(std::ptr::null()), 0); + assert!(ad_app_list_get(std::ptr::null(), 0).is_null()); + ad_app_list_free(std::ptr::null_mut()); + + assert_eq!(ad_window_list_count(std::ptr::null()), 0); + ad_window_list_free(std::ptr::null_mut()); + } +} + +#[test] +fn dirty_out_param_is_cleared_before_early_return_on_worker_thread() { + with_adapter(|adapter| unsafe { + let fake_ptr = 0xDEAD_BEEF as *mut AdAppList; + let mut list: *mut AdAppList = fake_ptr; + let rc = ad_list_apps(adapter, &mut list); + if rc != AdResult::Ok { + assert!( + list.is_null(), + "dirty out-param must be zeroed before early return, got {:?}", + list + ); + } + }); +} + +#[test] +fn list_handle_lifecycle_roundtrip() { + with_adapter(|adapter| unsafe { + let mut list: *mut AdAppList = std::ptr::null_mut(); + let rc = ad_list_apps(adapter, &mut list); + if rc == AdResult::Ok { + assert!(!list.is_null()); + let count = ad_app_list_count(list); + assert!(ad_app_list_get(list, count).is_null()); + ad_app_list_free(list); + } else { + assert!(list.is_null(), "failed list call must leave out null"); + let msg_ptr = ad_last_error_message(); + assert!(!msg_ptr.is_null()); + let _ = CStr::from_ptr(msg_ptr).to_string_lossy(); + assert_eq!(ad_last_error_code(), rc); + } + }); +} + +#[test] +fn list_windows_focused_only_runs() { + with_adapter(|adapter| unsafe { + let mut list: *mut AdWindowList = std::ptr::null_mut(); + let rc = ad_list_windows(adapter, std::ptr::null(), true, &mut list); + if rc == AdResult::Ok { + assert!(!list.is_null()); + let _ = ad_window_list_count(list); + ad_window_list_free(list); + } else { + assert!(list.is_null()); + } + }); +} + +#[test] +fn find_returns_not_found_on_empty_query_against_no_window() { + with_adapter(|adapter| unsafe { + let bad_win: AdWindowInfo = std::mem::zeroed(); + let query = AdFindQuery { + role: std::ptr::null(), + name_substring: std::ptr::null(), + value_substring: std::ptr::null(), + }; + let mut handle = AdNativeHandle { + ptr: std::ptr::null(), + }; + let rc = ad_find(adapter, &bad_win, &query, &mut handle); + assert!( + matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), + "zeroed window must not succeed, got {:?}", + rc + ); + }); +} + +#[test] +fn free_handle_null_is_noop() { + with_adapter(|adapter| unsafe { + let mut handle = AdNativeHandle { + ptr: std::ptr::null(), + }; + let rc = ad_free_handle(adapter, &mut handle); + assert_eq!(rc, AdResult::Ok); + assert!(handle.ptr.is_null()); + + let rc2 = ad_free_handle(adapter, std::ptr::null_mut()); + assert_eq!(rc2, AdResult::Ok); + }); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn free_handle_zeroes_ptr_so_double_free_is_noop() { + with_adapter(|adapter| unsafe { + let fake_live_ptr = 0x1234 as *const std::ffi::c_void; + let mut handle = AdNativeHandle { ptr: fake_live_ptr }; + + let _ = ad_free_handle(adapter, &mut handle); + assert!(handle.ptr.is_null()); + + let rc = ad_free_handle(adapter, &mut handle); + assert_eq!(rc, AdResult::Ok); + }); +} + +#[test] +fn last_error_survives_successful_calls() { + unsafe { + let adapter = ad_adapter_create(); + assert!(!adapter.is_null()); + let mut out: AdWindowInfo = std::mem::zeroed(); + let rc = common::ad_launch_app(adapter, std::ptr::null(), 0, &mut out); + assert!( + matches!(rc, AdResult::ErrInvalidArgs | AdResult::ErrInternal), + "must fail, got {:?}", + rc + ); + let msg_ptr = ad_last_error_message(); + assert!(!msg_ptr.is_null()); + + for _ in 0..5 { + let _ = ad_app_list_count(std::ptr::null()); + let _ = ad_window_list_count(std::ptr::null()); + } + + let after = ad_last_error_message(); + assert_eq!(msg_ptr, after); + assert_eq!(ad_last_error_code(), rc); + ad_adapter_destroy(adapter); + } +} diff --git a/crates/ffi/tests/common/mod.rs b/crates/ffi/tests/common/mod.rs new file mode 100644 index 0000000..4cf8fc7 --- /dev/null +++ b/crates/ffi/tests/common/mod.rs @@ -0,0 +1,100 @@ +#![allow(dead_code, unused_imports)] +#![allow(improper_ctypes)] + +pub use agent_desktop_ffi::error::AdResult; +pub use agent_desktop_ffi::{ + AdAction, AdActionResult, AdAdapter, AdAppList, AdDirection, AdDragParams, AdFindQuery, + AdKeyCombo, AdNativeHandle, AdPoint, AdPolicyKind, AdRect, AdRefEntry, AdScrollParams, + AdWindowInfo, AdWindowList, +}; +pub use std::ffi::CStr; +pub use std::os::raw::c_char; + +extern "C" { + pub fn ad_adapter_create() -> *mut AdAdapter; + pub fn ad_adapter_destroy(adapter: *mut AdAdapter); + pub fn ad_check_permissions(adapter: *const AdAdapter) -> AdResult; + + pub fn ad_last_error_code() -> AdResult; + pub fn ad_last_error_message() -> *const c_char; + + pub fn ad_list_apps(adapter: *const AdAdapter, out: *mut *mut AdAppList) -> AdResult; + pub fn ad_app_list_count(list: *const AdAppList) -> u32; + pub fn ad_app_list_get(list: *const AdAppList, index: u32) -> *const u8; + pub fn ad_app_list_free(list: *mut AdAppList); + + pub fn ad_list_windows( + adapter: *const AdAdapter, + app_filter: *const c_char, + focused_only: bool, + out: *mut *mut AdWindowList, + ) -> AdResult; + pub fn ad_window_list_count(list: *const AdWindowList) -> u32; + pub fn ad_window_list_free(list: *mut AdWindowList); + + pub fn ad_launch_app( + adapter: *const AdAdapter, + id: *const c_char, + timeout_ms: u64, + out: *mut AdWindowInfo, + ) -> AdResult; + + pub fn ad_execute_action( + adapter: *const AdAdapter, + handle: *const AdNativeHandle, + action: *const AdAction, + out: *mut AdActionResult, + ) -> AdResult; + pub fn ad_execute_action_with_policy( + adapter: *const AdAdapter, + handle: *const AdNativeHandle, + action: *const AdAction, + policy: i32, + out: *mut AdActionResult, + ) -> AdResult; + + pub fn ad_find( + adapter: *const AdAdapter, + win: *const AdWindowInfo, + query: *const AdFindQuery, + out: *mut AdNativeHandle, + ) -> AdResult; + + pub fn ad_free_handle(adapter: *const AdAdapter, handle: *mut AdNativeHandle) -> AdResult; + + pub fn ad_resolve_element( + adapter: *const AdAdapter, + entry: *const AdRefEntry, + out: *mut AdNativeHandle, + ) -> AdResult; +} + +pub fn with_adapter(body: F) { + unsafe { + let adapter = ad_adapter_create(); + assert!(!adapter.is_null(), "ad_adapter_create must not return null"); + body(adapter); + ad_adapter_destroy(adapter); + } +} + +pub fn default_action() -> AdAction { + AdAction { + kind: 0, + text: std::ptr::null(), + scroll: AdScrollParams { + direction: AdDirection::Down as i32, + amount: 0, + }, + key: AdKeyCombo { + key: std::ptr::null(), + modifiers: std::ptr::null(), + modifier_count: 0, + }, + drag: AdDragParams { + from: AdPoint { x: 0.0, y: 0.0 }, + to: AdPoint { x: 0.0, y: 0.0 }, + duration_ms: 0, + }, + } +} diff --git a/crates/ffi/tests/error_lifetime.rs b/crates/ffi/tests/error_lifetime.rs index 52b806b..2cbca7d 100644 --- a/crates/ffi/tests/error_lifetime.rs +++ b/crates/ffi/tests/error_lifetime.rs @@ -13,7 +13,6 @@ extern "C" { ) -> AdResult; fn ad_last_error_message() -> *const std::os::raw::c_char; fn ad_last_error_code() -> AdResult; - fn ad_check_permissions(adapter: *const agent_desktop_ffi::AdAdapter) -> AdResult; } #[test] @@ -39,7 +38,9 @@ fn last_error_pointer_survives_across_successful_calls() { let first_msg = CStr::from_ptr(first_msg_ptr).to_string_lossy().into_owned(); for _ in 0..10 { - let _rc = ad_check_permissions(adapter); + let next = ad_adapter_create(); + assert!(!next.is_null()); + ad_adapter_destroy(next); } let later_msg_ptr = ad_last_error_message(); diff --git a/crates/macos/src/actions/ax_helpers.rs b/crates/macos/src/actions/ax_helpers.rs index 78f7a1a..6ef46b5 100644 --- a/crates/macos/src/actions/ax_helpers.rs +++ b/crates/macos/src/actions/ax_helpers.rs @@ -1,13 +1,13 @@ -use agent_desktop_core::error::AdapterError; +use agent_desktop_core::error::{AdapterError, ErrorCode}; #[cfg(target_os = "macos")] mod imp { use super::*; use crate::tree::AXElement; use accessibility_sys::{ - kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, kAXValueAttribute, - AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, AXUIElementPerformAction, - AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout, + kAXErrorAPIDisabled, kAXErrorCannotComplete, kAXErrorSuccess, kAXFocusedAttribute, + kAXValueAttribute, AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, + AXUIElementPerformAction, AXUIElementSetAttributeValue, AXUIElementSetMessagingTimeout, }; use core_foundation::{ array::CFArray, @@ -24,20 +24,37 @@ mod imp { } pub fn try_ax_action_retried(el: &AXElement, name: &str) -> bool { + try_ax_action_retried_or_err(el, name).unwrap_or(false) + } + + pub fn try_ax_action_retried_or_err(el: &AXElement, name: &str) -> Result { let action = CFString::new(name); let err = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; if err == kAXErrorSuccess { - return true; + return Ok(true); } if err == kAXErrorCannotComplete { std::thread::sleep(std::time::Duration::from_millis(100)); let retry = unsafe { AXUIElementPerformAction(el.0, action.as_concrete_TypeRef()) }; - return retry == kAXErrorSuccess; + if retry == kAXErrorSuccess { + return Ok(true); + } + ax_error_result(name, retry)?; + return Ok(false); } - false + ax_error_result(name, err)?; + Ok(false) } pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool { + set_ax_bool_or_err(el, attr, value).unwrap_or(false) + } + + pub fn set_ax_bool_or_err( + el: &AXElement, + attr: &str, + value: bool, + ) -> Result { let cf_attr = CFString::new(attr); let cf_val = if value { CFBoolean::true_value() @@ -47,7 +64,11 @@ mod imp { let err = unsafe { AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) }; - err == kAXErrorSuccess + if err == kAXErrorSuccess { + return Ok(true); + } + ax_error_result(attr, err)?; + Ok(false) } pub fn set_ax_string_or_err( @@ -61,8 +82,9 @@ mod imp { AXUIElementSetAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), cf_val.as_CFTypeRef()) }; if err != kAXErrorSuccess { + ax_error_result(attr, err)?; return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::ActionFailed, + ErrorCode::ActionFailed, format!("AXSetAttributeValue({attr}) failed (err={err})"), ) .with_suggestion("Attribute may be read-only. Try 'click' or 'type' instead.")); @@ -142,8 +164,8 @@ mod imp { unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; } - pub fn ax_focus(el: &AXElement) -> bool { - set_ax_bool(el, kAXFocusedAttribute, true) + pub fn ax_focus_or_err(el: &AXElement) -> Result { + set_ax_bool_or_err(el, kAXFocusedAttribute, true) } pub fn ax_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> { @@ -159,6 +181,14 @@ mod imp { crate::tree::copy_string_attr(el, kAXRoleAttribute) .map(|r| crate::tree::roles::ax_role_to_str(&r).to_string()) } + + fn ax_error_result(operation: &str, err: i32) -> Result<(), AdapterError> { + if err == kAXErrorAPIDisabled { + return Err(AdapterError::permission_denied() + .with_platform_detail(format!("{operation} failed with kAXErrorAPIDisabled"))); + } + Ok(()) + } } #[cfg(not(target_os = "macos"))] @@ -172,9 +202,22 @@ mod imp { pub fn try_ax_action_retried(_el: &AXElement, _name: &str) -> bool { false } + pub fn try_ax_action_retried_or_err( + _el: &AXElement, + _name: &str, + ) -> Result { + Ok(false) + } pub fn set_ax_bool(_el: &AXElement, _attr: &str, _value: bool) -> bool { false } + pub fn set_ax_bool_or_err( + _el: &AXElement, + _attr: &str, + _value: bool, + ) -> Result { + Ok(false) + } pub fn set_ax_string_or_err( _el: &AXElement, _attr: &str, @@ -206,8 +249,8 @@ mod imp { } pub fn ensure_visible(_el: &AXElement) {} pub fn set_messaging_timeout(_el: &AXElement, _seconds: f32) {} - pub fn ax_focus(_el: &AXElement) -> bool { - false + pub fn ax_focus_or_err(_el: &AXElement) -> Result { + Ok(false) } pub fn ax_set_value(_el: &AXElement, _val: &str) -> Result<(), AdapterError> { Err(AdapterError::not_supported("ax_set_value")) @@ -221,7 +264,8 @@ mod imp { } pub(crate) use imp::{ - ax_focus, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action, - is_attr_settable, list_ax_actions, set_ax_bool, set_ax_string_or_err, set_messaging_timeout, - try_action_from_list, try_ax_action, try_ax_action_retried, try_each_ancestor, try_each_child, + ax_focus_or_err, ax_press, ax_set_value, element_role, ensure_visible, has_ax_action, + is_attr_settable, list_ax_actions, set_ax_bool, set_ax_bool_or_err, set_ax_string_or_err, + set_messaging_timeout, try_action_from_list, try_ax_action, try_ax_action_retried, + try_ax_action_retried_or_err, try_each_ancestor, try_each_child, }; diff --git a/crates/macos/src/actions/chain.rs b/crates/macos/src/actions/chain.rs index 47ae1ba..52ed70c 100644 --- a/crates/macos/src/actions/chain.rs +++ b/crates/macos/src/actions/chain.rs @@ -1,51 +1,12 @@ -use agent_desktop_core::action::MouseButton; +use agent_desktop_core::action::InteractionPolicy; use agent_desktop_core::error::{AdapterError, ErrorCode}; use crate::actions::discovery::ElementCaps; use crate::tree::AXElement; -#[allow(dead_code)] -pub enum ChainStep { - Action(&'static str), - SetBool { - attr: &'static str, - value: bool, - }, - SetDynamic { - attr: &'static str, - }, - FocusThenAction(&'static str), - FocusThenSetDynamic { - attr: &'static str, - }, - FocusThenConfirmOrPress, - ChildActions { - actions: &'static [&'static str], - limit: usize, - }, - AncestorActions { - actions: &'static [&'static str], - limit: usize, - }, - Custom { - label: &'static str, - func: fn(&AXElement, &ElementCaps) -> bool, - }, - CGClick { - button: MouseButton, - count: u32, - }, -} - -pub struct ChainDef { - pub pre_scroll: bool, - pub steps: &'static [ChainStep], - pub suggestion: &'static str, -} - -pub struct ChainContext<'a> { - pub dynamic_value: Option<&'a str>, -} +pub(crate) use super::chain_context::ChainContext; +pub(crate) use super::chain_def::ChainDef; +pub(crate) use super::chain_step::ChainStep; #[cfg(target_os = "macos")] mod imp { @@ -53,15 +14,19 @@ mod imp { use crate::actions::ax_helpers; use std::time::{Duration, Instant}; - const CHAIN_TIMEOUT: Duration = Duration::from_secs(10); + const DEFAULT_CHAIN_TIMEOUT: Duration = Duration::from_secs(10); + const MAX_CHAIN_TIMEOUT_MS: u64 = 300_000; pub fn execute_chain( el: &AXElement, caps: &ElementCaps, def: &ChainDef, ctx: &ChainContext, + policy: InteractionPolicy, ) -> Result<(), AdapterError> { - let deadline = Instant::now() + CHAIN_TIMEOUT; + let deadline = ctx + .deadline + .unwrap_or_else(|| Instant::now() + chain_timeout()); let total = def.steps.len(); if let Some(pid) = crate::system::app_ops::pid_from_element(el) { @@ -82,15 +47,25 @@ mod imp { .iter() .find(|s| matches!(s, ChainStep::CGClick { .. })) { - if execute_step(el, caps, cg, ctx) { + if physical_click_permitted(policy) && execute_step(el, caps, cg, ctx, policy)? + { tracing::debug!("chain: CGClick fallback succeeded"); return Ok(()); } } - return Err(AdapterError::timeout("Chain execution exceeded 10s")); + return Err( + AdapterError::timeout("Chain execution deadline exceeded").with_suggestion( + "Retry the command, refresh the snapshot, or increase AGENT_DESKTOP_CHAIN_TIMEOUT_MS for slow apps.", + ), + ); + } + if matches!(step, ChainStep::CGClick { .. }) && !physical_click_permitted(policy) { + return Err(AdapterError::policy_denied( + "Physical click fallback is disabled by the current interaction policy", + )); } let label = step_label(step); - if execute_step(el, caps, step, ctx) { + if execute_step(el, caps, step, ctx, policy)? { tracing::debug!("chain: [{}/{}] {} -> success", i + 1, total, label); return Ok(()); } @@ -109,9 +84,8 @@ mod imp { ChainStep::Action(name) => name, ChainStep::SetBool { attr, .. } => attr, ChainStep::SetDynamic { attr } => attr, - ChainStep::FocusThenAction(name) => name, ChainStep::FocusThenSetDynamic { attr } => attr, - ChainStep::FocusThenConfirmOrPress => "FocusThenConfirmOrPress", + ChainStep::FocusThenClearByKeyboard => "FocusThenClearByKeyboard", ChainStep::ChildActions { .. } => "ChildActions", ChainStep::AncestorActions { .. } => "AncestorActions", ChainStep::Custom { label, .. } => label, @@ -124,9 +98,10 @@ mod imp { caps: &ElementCaps, step: &ChainStep, ctx: &ChainContext, - ) -> bool { + policy: InteractionPolicy, + ) -> Result { match step { - ChainStep::Action(name) => ax_helpers::try_ax_action_retried(el, name), + ChainStep::Action(name) => ax_helpers::try_ax_action_retried_or_err(el, name), ChainStep::SetBool { attr, value } => { let settable = match *attr { @@ -135,71 +110,177 @@ mod imp { "AXFocused" => caps.settable_focus, _ => ax_helpers::is_attr_settable(el, attr), }; - settable && ax_helpers::set_ax_bool(el, attr, *value) + Ok(settable && set_bool_verified(el, attr, *value)?) } ChainStep::SetDynamic { attr } => { let value = match ctx.dynamic_value { Some(v) => v, - None => return false, + None => return Ok(false), }; - ax_helpers::set_ax_string_or_err(el, attr, value).is_ok() - } - - ChainStep::FocusThenAction(name) => { - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(Duration::from_millis(50)); - ax_helpers::try_ax_action_retried(el, name) + set_dynamic_verified(el, attr, value) } ChainStep::FocusThenSetDynamic { attr } => { + if !policy.allow_focus_steal { + return Ok(false); + } let value = match ctx.dynamic_value { Some(v) => v, - None => return false, + None => return Ok(false), }; - if !ax_helpers::ax_focus(el) { - return false; + if !ax_helpers::ax_focus_or_err(el)? { + return Ok(false); } std::thread::sleep(Duration::from_millis(50)); - ax_helpers::set_ax_string_or_err(el, attr, value).is_ok() + set_dynamic_verified(el, attr, value) } - ChainStep::FocusThenConfirmOrPress => { - if !ax_helpers::ax_focus(el) { - return false; + ChainStep::FocusThenClearByKeyboard => { + if !policy.allow_focus_steal { + return Ok(false); } - std::thread::sleep(Duration::from_millis(50)); - ax_helpers::try_ax_action_retried(el, "AXConfirm") - || ax_helpers::try_ax_action_retried(el, "AXPress") + if !ax_helpers::ax_focus_or_err(el)? { + return Ok(false); + } + std::thread::sleep(Duration::from_millis(20)); + Ok(crate::input::keyboard::synthesize_key_for_element( + el, + &agent_desktop_core::action::KeyCombo { + key: "a".into(), + modifiers: vec![agent_desktop_core::action::Modifier::Cmd], + }, + ) + .and_then(|_| { + crate::input::keyboard::synthesize_key_for_element( + el, + &agent_desktop_core::action::KeyCombo { + key: "delete".into(), + modifiers: vec![], + }, + ) + }) + .is_ok()) } - ChainStep::ChildActions { actions, limit } => ax_helpers::try_each_child( + ChainStep::ChildActions { actions, limit } => Ok(ax_helpers::try_each_child( el, |child| { let child_actions = ax_helpers::list_ax_actions(child); ax_helpers::try_action_from_list(child, &child_actions, actions) }, *limit, - ), + )), - ChainStep::AncestorActions { actions, limit } => ax_helpers::try_each_ancestor( + ChainStep::AncestorActions { actions, limit } => Ok(ax_helpers::try_each_ancestor( el, |ancestor| { let al = ax_helpers::list_ax_actions(ancestor); ax_helpers::try_action_from_list(ancestor, &al, actions) }, *limit, - ), + )), ChainStep::Custom { label: _, func } => func(el, caps), ChainStep::CGClick { button, count } => { - crate::actions::dispatch::click_via_bounds(el, button.clone(), *count).is_ok() + Ok( + crate::actions::dispatch::click_via_bounds(el, button.clone(), *count, policy) + .is_ok(), + ) } } } + + fn chain_timeout() -> Duration { + std::env::var("AGENT_DESKTOP_CHAIN_TIMEOUT_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|ms| *ms > 0) + .map(|ms| ms.min(MAX_CHAIN_TIMEOUT_MS)) + .map(Duration::from_millis) + .unwrap_or(DEFAULT_CHAIN_TIMEOUT) + } + + fn physical_click_permitted(policy: InteractionPolicy) -> bool { + policy.allow_focus_steal && policy.allow_cursor_move + } + + fn set_dynamic_verified(el: &AXElement, attr: &str, value: &str) -> Result { + ax_helpers::set_ax_string_or_err(el, attr, value)?; + Ok(dynamic_write_had_effect( + attr, + ax_helpers::element_role(el).as_deref(), + value, + crate::tree::copy_value_typed(el).as_deref(), + )) + } + + fn set_bool_verified(el: &AXElement, attr: &str, value: bool) -> Result { + Ok(ax_helpers::set_ax_bool_or_err(el, attr, value)? + && bool_write_had_effect(attr, value, crate::tree::copy_bool_attr(el, attr))) + } + + fn bool_write_had_effect(attr: &str, expected: bool, observed: Option) -> bool { + !matches!( + attr, + "AXExpanded" | "AXDisclosing" | "AXSelected" | "AXFocused" + ) || observed == Some(expected) + } + + fn dynamic_write_had_effect( + attr: &str, + role: Option<&str>, + expected: &str, + observed: Option<&str>, + ) -> bool { + attr != "AXValue" || role == Some("AXSecureTextField") || observed == Some(expected) + } + + #[cfg(test)] + mod tests { + use super::{bool_write_had_effect, dynamic_write_had_effect}; + + #[test] + fn ax_value_write_requires_readback_match() { + assert!(!dynamic_write_had_effect( + "AXValue", + Some("AXTextField"), + "", + Some("unchanged") + )); + assert!(dynamic_write_had_effect( + "AXValue", + Some("AXTextField"), + "", + Some("") + )); + } + + #[test] + fn non_value_and_secure_writes_trust_ax_success() { + assert!(dynamic_write_had_effect( + "AXSelected", + Some("AXCheckBox"), + "true", + None + )); + assert!(dynamic_write_had_effect( + "AXValue", + Some("AXSecureTextField"), + "secret", + None + )); + } + + #[test] + fn bool_state_writes_require_readback_match_for_stateful_attrs() { + assert!(bool_write_had_effect("AXExpanded", true, Some(true))); + assert!(!bool_write_had_effect("AXExpanded", true, Some(false))); + assert!(!bool_write_had_effect("AXExpanded", false, None)); + assert!(bool_write_had_effect("AXFoo", true, None)); + } + } } #[cfg(not(target_os = "macos"))] @@ -211,6 +292,7 @@ mod imp { _caps: &ElementCaps, def: &ChainDef, _ctx: &ChainContext, + _policy: InteractionPolicy, ) -> Result<(), AdapterError> { Err(AdapterError::new( ErrorCode::ActionFailed, diff --git a/crates/macos/src/actions/chain_context.rs b/crates/macos/src/actions/chain_context.rs new file mode 100644 index 0000000..733a41a --- /dev/null +++ b/crates/macos/src/actions/chain_context.rs @@ -0,0 +1,4 @@ +pub(crate) struct ChainContext<'a> { + pub(crate) dynamic_value: Option<&'a str>, + pub(crate) deadline: Option, +} diff --git a/crates/macos/src/actions/chain_def.rs b/crates/macos/src/actions/chain_def.rs new file mode 100644 index 0000000..f1304fe --- /dev/null +++ b/crates/macos/src/actions/chain_def.rs @@ -0,0 +1,7 @@ +use super::chain_step::ChainStep; + +pub(crate) struct ChainDef { + pub(crate) pre_scroll: bool, + pub(crate) steps: &'static [ChainStep], + pub(crate) suggestion: &'static str, +} diff --git a/crates/macos/src/actions/chain_defs.rs b/crates/macos/src/actions/chain_defs.rs index 820abc1..f863bff 100644 --- a/crates/macos/src/actions/chain_defs.rs +++ b/crates/macos/src/actions/chain_defs.rs @@ -5,12 +5,12 @@ mod imp { use super::*; use crate::actions::{ ax_helpers, - chain::{execute_chain, ChainContext, ChainDef, ChainStep}, - chain_steps, + chain::{ChainDef, ChainStep}, + chain_menu_steps, chain_steps, discovery::ElementCaps, }; use crate::tree::AXElement; - use agent_desktop_core::action::MouseButton; + use agent_desktop_core::action::{InteractionPolicy, MouseButton}; pub static CLICK_CHAIN: ChainDef = ChainDef { pre_scroll: true, @@ -26,13 +26,9 @@ mod imp { label: "show_alternate_ui", func: chain_steps::try_show_alternate_ui, }, - ChainStep::ChildActions { - actions: &["AXPress", "AXConfirm", "AXOpen"], - limit: 3, - }, ChainStep::Custom { - label: "value_relay", - func: chain_steps::try_value_relay, + label: "containing_item_select", + func: chain_steps::try_select_containing_item, }, ChainStep::SetBool { attr: "AXSelected", @@ -42,22 +38,22 @@ mod imp { label: "parent_row_select", func: chain_steps::try_parent_row_select, }, + ChainStep::Custom { + label: "value_relay", + func: chain_steps::try_value_relay, + }, ChainStep::Custom { label: "select_via_parent", func: chain_steps::try_select_via_parent, }, + ChainStep::ChildActions { + actions: &["AXPress", "AXConfirm", "AXOpen"], + limit: 3, + }, ChainStep::Custom { label: "custom_actions", func: chain_steps::try_custom_actions, }, - ChainStep::Custom { - label: "focus_verified_confirm_or_press", - func: chain_steps::try_focus_then_verified_confirm_or_press, - }, - ChainStep::Custom { - label: "keyboard_activate", - func: chain_steps::try_keyboard_activate, - }, ChainStep::AncestorActions { actions: &["AXPress", "AXConfirm"], limit: 2, @@ -73,23 +69,25 @@ mod imp { pub static RIGHT_CLICK_CHAIN: ChainDef = ChainDef { pre_scroll: false, steps: &[ - ChainStep::Action("AXShowMenu"), ChainStep::Custom { - label: "focus_app_show_menu", - func: chain_steps::focus_app_then_show_menu, + label: "show_menu", + func: chain_menu_steps::show_menu, }, ChainStep::Custom { label: "select_then_show_menu", - func: chain_steps::select_then_show_menu, + func: chain_menu_steps::select_then_show_menu, }, - ChainStep::FocusThenAction("AXShowMenu"), - ChainStep::AncestorActions { - actions: &["AXShowMenu"], - limit: 3, + ChainStep::Custom { + label: "selected_items_menu", + func: chain_menu_steps::select_then_selected_items_menu, }, - ChainStep::ChildActions { - actions: &["AXShowMenu"], - limit: 5, + ChainStep::Custom { + label: "child_show_menu", + func: chain_menu_steps::show_menu_on_children, + }, + ChainStep::Custom { + label: "ancestor_show_menu", + func: chain_menu_steps::show_menu_on_ancestors, }, ChainStep::CGClick { button: MouseButton::Right, @@ -103,6 +101,10 @@ mod imp { pre_scroll: false, steps: &[ ChainStep::Action("AXExpand"), + ChainStep::SetBool { + attr: "AXExpanded", + value: true, + }, ChainStep::SetBool { attr: "AXDisclosing", value: true, @@ -115,6 +117,10 @@ mod imp { pre_scroll: false, steps: &[ ChainStep::Action("AXCollapse"), + ChainStep::SetBool { + attr: "AXExpanded", + value: false, + }, ChainStep::SetBool { attr: "AXDisclosing", value: false, @@ -123,12 +129,14 @@ mod imp { suggestion: "Try 'click' to close it instead.", }; + const VALUE_STEPS: &[ChainStep] = &[ + ChainStep::SetDynamic { attr: "AXValue" }, + ChainStep::FocusThenSetDynamic { attr: "AXValue" }, + ]; + pub static SET_VALUE_CHAIN: ChainDef = ChainDef { pre_scroll: false, - steps: &[ - ChainStep::SetDynamic { attr: "AXValue" }, - ChainStep::FocusThenSetDynamic { attr: "AXValue" }, - ], + steps: VALUE_STEPS, suggestion: "Try 'clear' then 'type', or check element is a text field.", }; @@ -137,10 +145,7 @@ mod imp { steps: &[ ChainStep::SetDynamic { attr: "AXValue" }, ChainStep::FocusThenSetDynamic { attr: "AXValue" }, - ChainStep::Custom { - label: "select_all_delete", - func: chain_steps::select_all_then_delete, - }, + ChainStep::FocusThenClearByKeyboard, ], suggestion: "Try 'press cmd+a' then 'press delete'.", }; @@ -171,35 +176,30 @@ mod imp { steps: &[ ChainStep::Action("AXScrollToVisible"), ChainStep::Custom { - label: "walk_parents_scroll", - func: chain_steps::walk_parents_and_scroll, + label: "visible_in_scroll_context", + func: chain_steps::element_is_visible_in_scroll_context, }, ], suggestion: "Element may not be in a scrollable container.", }; - pub fn double_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> { + pub fn double_click( + el: &AXElement, + _caps: &ElementCaps, + policy: InteractionPolicy, + ) -> Result<(), AdapterError> { if ax_helpers::try_ax_action(el, "AXOpen") { return Ok(()); } - let ctx = ChainContext { - dynamic_value: None, - }; - let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); - std::thread::sleep(std::time::Duration::from_millis(50)); - let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2) + crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 2, policy) } - pub fn triple_click(el: &AXElement, caps: &ElementCaps) -> Result<(), AdapterError> { - let ctx = ChainContext { - dynamic_value: None, - }; - for _ in 0..3 { - let _ = execute_chain(el, caps, &CLICK_CHAIN, &ctx); - std::thread::sleep(std::time::Duration::from_millis(30)); - } - crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3) + pub fn triple_click( + el: &AXElement, + _caps: &ElementCaps, + policy: InteractionPolicy, + ) -> Result<(), AdapterError> { + crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3, policy) } } diff --git a/crates/macos/src/actions/chain_menu_steps.rs b/crates/macos/src/actions/chain_menu_steps.rs new file mode 100644 index 0000000..28ec599 --- /dev/null +++ b/crates/macos/src/actions/chain_menu_steps.rs @@ -0,0 +1,249 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::actions::{ax_helpers, discovery::ElementCaps}; + use crate::tree::AXElement; + use agent_desktop_core::error::AdapterError; + + pub fn show_menu(el: &AXElement, _caps: &ElementCaps) -> Result { + show_menu_on_element(el) + } + + pub fn show_menu_on_ancestors( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..3 { + let Some(parent) = current else { + return Ok(false); + }; + if show_menu_on_element(&parent)? { + return Ok(true); + } + current = crate::tree::copy_element_attr(&parent, "AXParent"); + } + Ok(false) + } + + pub fn show_menu_on_children( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { + for child in crate::tree::copy_ax_array(el, "AXChildren") + .unwrap_or_default() + .iter() + .take(5) + { + if show_menu_on_element(child)? { + return Ok(true); + } + } + Ok(false) + } + + pub fn select_then_show_menu( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { + if !select_containing_item(el)? { + return Ok(false); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + show_menu_on_element(el) + } + + pub fn select_then_selected_items_menu( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { + if !select_containing_item(el)? { + tracing::debug!("selected-items menu: could not select containing item"); + return Ok(false); + } + let selected_name = crate::tree::resolve_element_name(el); + let Some(window) = window_ancestor(el) else { + tracing::debug!("selected-items menu: no window ancestor"); + return Ok(false); + }; + let Some(menu_button) = selected_items_menu_button(&window) else { + tracing::debug!("selected-items menu: no selected-items menu button"); + return Ok(false); + }; + show_menu_or_press_selected(&menu_button, selected_name.as_deref()) + } + + fn show_menu_on_element(el: &AXElement) -> Result { + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return Ok(false); + }; + let was_open = is_menu_open(pid); + if !ax_helpers::try_ax_action_retried_or_err(el, "AXShowMenu")? { + return Ok(false); + } + Ok(wait_for_new_menu(pid, was_open)) + } + + fn show_menu_or_press_selected( + el: &AXElement, + selected_name: Option<&str>, + ) -> Result { + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return Ok(false); + }; + if ax_helpers::try_ax_action_retried_or_err(el, "AXShowMenu")? + && wait_for_selected_menu(pid, selected_name) + { + return Ok(true); + } + Ok(ax_helpers::try_ax_action_retried_or_err(el, "AXPress")? + && wait_for_selected_menu(pid, selected_name)) + } + + fn wait_for_new_menu(pid: i32, was_open: bool) -> bool { + if was_open { + return false; + } + crate::system::wait::wait_for_menu(pid, true, crate::system::wait::menu_timeout_ms()) + .is_ok() + } + + fn is_menu_open(pid: i32) -> bool { + crate::system::wait::wait_for_menu(pid, true, 0).is_ok() + } + + fn wait_for_selected_menu(pid: i32, selected_name: Option<&str>) -> bool { + let deadline = std::time::Instant::now() + + std::time::Duration::from_millis(crate::system::wait::menu_timeout_ms()); + loop { + if menu_matches_selection(pid, selected_name) { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + + fn menu_matches_selection(pid: i32, selected_name: Option<&str>) -> bool { + let Some(menu) = crate::tree::surfaces::menu_element_for_pid(pid) else { + return false; + }; + selected_name + .filter(|name| !name.is_empty()) + .is_none_or(|name| element_text_contains(&menu, name, 0)) + } + + fn element_text_contains(el: &AXElement, needle: &str, depth: usize) -> bool { + if depth > 8 { + return false; + } + ["AXTitle", "AXDescription", "AXValue", "AXHelp"] + .into_iter() + .filter_map(|attr| crate::tree::copy_string_attr(el, attr)) + .any(|value| value.contains(needle)) + || crate::tree::copy_ax_array(el, "AXChildren") + .unwrap_or_default() + .iter() + .any(|child| element_text_contains(child, needle, depth + 1)) + } + + fn select_containing_item(el: &AXElement) -> Result { + Ok(ax_helpers::set_ax_bool_or_err(el, "AXSelected", true)? + || crate::actions::chain_steps::try_select_containing_item( + el, + &ElementCaps { + settable_focus: false, + settable_selected: false, + settable_disclosing: false, + }, + )?) + } + + fn window_ancestor(el: &AXElement) -> Option { + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..20 { + let ancestor = current?; + if crate::tree::copy_string_attr(&ancestor, "AXRole").as_deref() == Some("AXWindow") { + return Some(ancestor); + } + current = crate::tree::copy_element_attr(&ancestor, "AXParent"); + } + None + } + + fn selected_items_menu_button(root: &AXElement) -> Option { + find_descendant(root, 0, &|el| { + crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXMenuButton") + && is_selected_items_control(el) + }) + } + + fn is_selected_items_control(el: &AXElement) -> bool { + ["AXHelp", "AXDescription", "AXTitle"] + .into_iter() + .filter_map(|attr| crate::tree::copy_string_attr(el, attr)) + .any(|value| { + let matches = selected_items_text(&value); + if matches { + tracing::debug!("selected-items menu: matched control text {value:?}"); + } + matches + }) + } + + fn selected_items_text(value: &str) -> bool { + let value = value.to_ascii_lowercase(); + value.contains("selected item") + } + + fn find_descendant( + el: &AXElement, + depth: usize, + predicate: &impl Fn(&AXElement) -> bool, + ) -> Option { + if depth > 8 { + return None; + } + if predicate(el) { + return Some(el.clone()); + } + for child in crate::tree::copy_ax_array(el, "AXChildren").unwrap_or_default() { + if let Some(found) = find_descendant(&child, depth + 1, predicate) { + return Some(found); + } + } + None + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::actions::discovery::ElementCaps; + use crate::tree::AXElement; + + pub fn show_menu(_el: &AXElement, _caps: &ElementCaps) -> bool { + false + } + + pub fn show_menu_on_ancestors(_el: &AXElement, _caps: &ElementCaps) -> bool { + false + } + + pub fn show_menu_on_children(_el: &AXElement, _caps: &ElementCaps) -> bool { + false + } + + pub fn select_then_show_menu(_el: &AXElement, _caps: &ElementCaps) -> bool { + false + } + + pub fn select_then_selected_items_menu(_el: &AXElement, _caps: &ElementCaps) -> bool { + false + } +} + +pub(crate) use imp::{ + select_then_selected_items_menu, select_then_show_menu, show_menu, show_menu_on_ancestors, + show_menu_on_children, +}; diff --git a/crates/macos/src/actions/chain_step.rs b/crates/macos/src/actions/chain_step.rs new file mode 100644 index 0000000..7a99c5e --- /dev/null +++ b/crates/macos/src/actions/chain_step.rs @@ -0,0 +1,34 @@ +use agent_desktop_core::{action::MouseButton, error::AdapterError}; + +use crate::{actions::discovery::ElementCaps, tree::AXElement}; + +pub(crate) enum ChainStep { + Action(&'static str), + SetBool { + attr: &'static str, + value: bool, + }, + SetDynamic { + attr: &'static str, + }, + FocusThenSetDynamic { + attr: &'static str, + }, + FocusThenClearByKeyboard, + ChildActions { + actions: &'static [&'static str], + limit: usize, + }, + AncestorActions { + actions: &'static [&'static str], + limit: usize, + }, + Custom { + label: &'static str, + func: fn(&AXElement, &ElementCaps) -> Result, + }, + CGClick { + button: MouseButton, + count: u32, + }, +} diff --git a/crates/macos/src/actions/chain_steps.rs b/crates/macos/src/actions/chain_steps.rs index e8eb1b2..e996a1e 100644 --- a/crates/macos/src/actions/chain_steps.rs +++ b/crates/macos/src/actions/chain_steps.rs @@ -2,22 +2,25 @@ mod imp { use crate::actions::{ax_helpers, discovery::ElementCaps}; use crate::tree::AXElement; + use agent_desktop_core::error::AdapterError; - pub fn do_verified_press(el: &AXElement, caps: &ElementCaps) -> bool { - dispatch_verified_press(el, caps, is_in_webarea(el)) + pub fn do_verified_press(el: &AXElement, caps: &ElementCaps) -> Result { + dispatch_verified_press(el, caps, crate::actions::chain_web_steps::is_in_webarea(el)) } - fn dispatch_verified_press(el: &AXElement, _caps: &ElementCaps, in_web: bool) -> bool { + fn dispatch_verified_press( + el: &AXElement, + _caps: &ElementCaps, + in_web: bool, + ) -> Result { if !in_web { return verified_press_native(el); } tracing::debug!("verified_press: web element detected"); - activate_web_element(el) + Ok(crate::actions::chain_web_steps::activate_web_element(el)) } - /// Native (non-web) elements: AXPress with selection verification - /// for elements inside selection containers. - fn verified_press_native(el: &AXElement) -> bool { + fn verified_press_native(el: &AXElement) -> Result { use accessibility_sys::kAXRoleAttribute; let parent = crate::tree::copy_element_attr(el, "AXParent"); let in_container = parent.as_ref().is_some_and(|p| { @@ -27,174 +30,38 @@ mod imp { ) }); if !in_container { - return ax_helpers::try_ax_action_retried(el, "AXPress"); + return ax_helpers::try_ax_action_retried_or_err(el, "AXPress"); } tracing::debug!("verified_press: native element in container, using AXPress"); let selected_before = crate::tree::element::copy_bool_attr(el, "AXSelected"); - if !ax_helpers::try_ax_action_retried(el, "AXPress") { - return false; + if !ax_helpers::try_ax_action_retried_or_err(el, "AXPress")? { + return Ok(false); } if selected_before == Some(true) { - return true; + return Ok(true); } std::thread::sleep(std::time::Duration::from_millis(50)); let selected_after = crate::tree::element::copy_bool_attr(el, "AXSelected"); if selected_after == Some(true) { - return true; + return Ok(true); } if crate::tree::copy_string_attr(el, kAXRoleAttribute).is_none() { - return true; + return Ok(true); } tracing::debug!("verified_press: AXPress ok but no state change"); - false + Ok(false) } - fn is_in_webarea(el: &AXElement) -> bool { - use accessibility_sys::kAXRoleAttribute; - let mut current = crate::tree::copy_element_attr(el, "AXParent"); - for _ in 0..20 { - let Some(ref parent) = current else { - return false; - }; - if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() - == Some("AXWebArea") - { - return true; - } - current = crate::tree::copy_element_attr(parent, "AXParent"); - } - false - } - - /// Activate any Electron/web element with verified effect detection. - /// - /// Chromium's AX implementation lies: AXPress/AXConfirm return - /// kAXErrorSuccess but only toggle ARIA state — DOM click handlers - /// never fire. We verify by checking if the focused UI element - /// changed after AXPress. If nothing changed, CGClick is the - /// genuine last resort. - /// - /// This handles the FULL escalation for web elements so the chain - /// engine never reaches false-positive AX steps (AXConfirm, SetBool - /// AXSelected, etc. all lie on Chromium). - struct PreActionState { - focused: Option, - value: Option, - selected: Option, - } - - impl PreActionState { - fn capture(app: &AXElement, el: &AXElement) -> Self { - Self { - focused: crate::tree::copy_element_attr(app, "AXFocusedUIElement"), - value: crate::tree::copy_string_attr(el, "AXValue"), - selected: crate::tree::copy_bool_attr(el, "AXSelected"), - } - } - } - - fn activate_web_element(el: &AXElement) -> bool { - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return false; - }; - - let app = crate::tree::element_for_pid(pid); - let before = PreActionState::capture(&app, el); - - if ax_helpers::try_ax_action_retried(el, "AXPress") { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: AXPress had real effect"); - return true; - } - tracing::debug!("activate_web: AXPress returned success but no DOM effect"); - } - - let actions = ax_helpers::list_ax_actions(el); - for action in &["AXConfirm", "AXOpen"] { - if actions.iter().any(|a| a == action) && ax_helpers::try_ax_action(el, action) { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: {action} had real effect"); - return true; - } - } - } - - if ax_helpers::try_each_child( - el, - |child| { - let child_actions = ax_helpers::list_ax_actions(child); - ax_helpers::try_action_from_list( - child, - &child_actions, - &["AXPress", "AXConfirm", "AXOpen"], - ) - }, - 5, - ) { - std::thread::sleep(std::time::Duration::from_millis(100)); - if web_action_had_effect(&app, el, &before) { - tracing::debug!("activate_web: child action had real effect"); - return true; - } - } - - tracing::debug!("activate_web: all AX methods had no effect, CGClick"); - let _ = crate::system::app_ops::ensure_app_focused(pid); - crate::actions::dispatch::click_via_bounds( - el, - agent_desktop_core::action::MouseButton::Left, - 1, - ) - .is_ok() - } - - fn web_action_had_effect(app: &AXElement, el: &AXElement, before: &PreActionState) -> bool { - use core_foundation::base::{CFEqual, CFTypeRef}; - - let value_after = crate::tree::copy_string_attr(el, "AXValue"); - if before.value != value_after { - return true; - } - - let selected_after = crate::tree::copy_bool_attr(el, "AXSelected"); - if before.selected != selected_after { - return true; - } - - let focused_after = crate::tree::copy_element_attr(app, "AXFocusedUIElement"); - match (&before.focused, &focused_after) { - (Some(before_f), Some(after_f)) => unsafe { - CFEqual(before_f.0 as CFTypeRef, after_f.0 as CFTypeRef) == 0 - }, - (None, Some(_)) => true, - _ => false, - } - } - - pub fn try_focus_then_verified_confirm_or_press(el: &AXElement, caps: &ElementCaps) -> bool { - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let in_web = is_in_webarea(el); - if !in_web && ax_helpers::try_ax_action_retried(el, "AXConfirm") { - return true; - } - dispatch_verified_press(el, caps, in_web) - } - - pub fn try_value_relay(el: &AXElement, _caps: &ElementCaps) -> bool { + pub fn try_value_relay(el: &AXElement, _caps: &ElementCaps) -> Result { if !ax_helpers::list_ax_actions(el).is_empty() { - return false; + return Ok(false); } let win = crate::tree::copy_element_attr(el, "AXWindow"); let is_dialog = win.as_ref().is_some_and(|w| { crate::tree::copy_string_attr(w, "AXSubrole").as_deref() == Some("AXDialog") }); if !is_dialog { - return false; + return Ok(false); } let label = std::cell::RefCell::new(None::); ax_helpers::try_each_child( @@ -210,123 +77,208 @@ mod imp { 5, ); let Some(label) = label.into_inner() else { - return false; + return Ok(false); }; let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return false; + return Ok(false); }; let app = crate::tree::element_for_pid(pid); let Some(owner) = crate::tree::copy_element_attr(&app, "AXFocusedUIElement") else { - return false; + return Ok(false); }; + if !same_window(&owner, win.as_ref()) { + return Ok(false); + } if !ax_helpers::is_attr_settable(&owner, "AXValue") { - return false; + return Ok(false); } let orig = crate::tree::copy_string_attr(&owner, "AXValue"); - if ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label).is_err() { - return false; - } + ax_helpers::set_ax_string_or_err(&owner, "AXValue", &label)?; std::thread::sleep(std::time::Duration::from_millis(150)); - if !ax_helpers::try_ax_action(&owner, "AXConfirm") { + if !ax_helpers::try_ax_action_retried_or_err(&owner, "AXConfirm")? { if let Some(o) = &orig { let _ = ax_helpers::set_ax_string_or_err(&owner, "AXValue", o); } - return false; + return Ok(false); } std::thread::sleep(std::time::Duration::from_millis(150)); let final_val = crate::tree::copy_string_attr(&owner, "AXValue"); if final_val.as_deref() != Some(label.as_str()) { tracing::debug!("value_relay: reverted to {final_val:?}, expected {label:?}"); } - final_val.as_deref() == Some(label.as_str()) + Ok(final_val.as_deref() == Some(label.as_str())) } - pub fn select_all_then_delete(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey; - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + fn same_window(owner: &AXElement, expected_window: Option<&AXElement>) -> bool { + let Some(expected_window) = expected_window else { return false; }; - let a = crate::tree::element_for_pid(pid); - unsafe { - PostKey(a.0, 0, 55, true); - PostKey(a.0, 0, 0, true); - PostKey(a.0, 0, 0, false); - PostKey(a.0, 0, 55, false); - } - std::thread::sleep(std::time::Duration::from_millis(30)); - unsafe { - PostKey(a.0, 0, 51, true); - PostKey(a.0, 0, 51, false); - } - true + let Some(owner_window) = crate::tree::copy_element_attr(owner, "AXWindow") else { + return false; + }; + crate::tree::same_element(&owner_window, expected_window) } - pub fn walk_parents_and_scroll(el: &AXElement, _caps: &ElementCaps) -> bool { + pub fn element_is_visible_in_scroll_context( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { use accessibility_sys::kAXRoleAttribute; let Some(bounds) = crate::tree::read_bounds(el) else { - return false; + return Ok(false); }; + if !rect_has_area(&bounds) { + return Ok(false); + } let mut current = crate::tree::copy_element_attr(el, "AXParent"); for _ in 0..8 { - let Some(parent) = ¤t else { return false }; + let Some(parent) = ¤t else { + return Ok(true); + }; if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() == Some("AXScrollArea") { let Some(pb) = crate::tree::read_bounds(parent) else { - return false; + return Ok(false); }; - let ty = bounds.y + bounds.height / 2.0; - if ty < pb.y || ty > pb.y + pb.height { - let dy = if ty > pb.y + pb.height / 2.0 { -5 } else { 5 }; - let (cx, cy) = (pb.x + pb.width / 2.0, pb.y + pb.height / 2.0); - for _ in 0..20 { - let _ = crate::input::mouse::synthesize_scroll_at(cx, cy, dy, 0); - std::thread::sleep(std::time::Duration::from_millis(16)); - } - } - return true; + return Ok(rect_has_area(&pb) && center_is_inside(&bounds, &pb)); } current = crate::tree::copy_element_attr(parent, "AXParent"); } - false + Ok(true) } - pub fn try_show_alternate_ui(el: &AXElement, _caps: &ElementCaps) -> bool { + pub(crate) fn rect_has_area(rect: &agent_desktop_core::node::Rect) -> bool { + rect.width > 0.0 && rect.height > 0.0 + } + + pub(crate) fn center_is_inside( + inner: &agent_desktop_core::node::Rect, + outer: &agent_desktop_core::node::Rect, + ) -> bool { + let x = inner.x + inner.width / 2.0; + let y = inner.y + inner.height / 2.0; + x >= outer.x && x <= outer.x + outer.width && y >= outer.y && y <= outer.y + outer.height + } + + pub fn try_show_alternate_ui( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { if !ax_helpers::has_ax_action(el, "AXShowAlternateUI") { - return false; + return Ok(false); } - ax_helpers::try_ax_action(el, "AXShowAlternateUI"); + ax_helpers::try_ax_action_retried_or_err(el, "AXShowAlternateUI")?; std::thread::sleep(std::time::Duration::from_millis(100)); - ax_helpers::try_each_child( + Ok(ax_helpers::try_each_child( el, |child| { let ca = ax_helpers::list_ax_actions(child); ax_helpers::try_action_from_list(child, &ca, &["AXPress"]) }, 5, - ) + )) } - pub fn try_parent_row_select(el: &AXElement, _caps: &ElementCaps) -> bool { + pub fn try_parent_row_select( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { use accessibility_sys::kAXRoleAttribute; let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { - return false; + return Ok(false); }; let role = crate::tree::copy_string_attr(&parent, kAXRoleAttribute).unwrap_or_default(); if !matches!(role.as_str(), "AXRow" | "AXOutlineRow") { - return false; + return Ok(false); } if !ax_helpers::is_attr_settable(&parent, "AXSelected") { - return false; + return Ok(false); } - ax_helpers::set_ax_bool(&parent, "AXSelected", true) + ax_helpers::set_ax_bool_or_err(&parent, "AXSelected", true) } - pub fn try_select_via_parent(el: &AXElement, _caps: &ElementCaps) -> bool { + pub fn try_select_containing_item( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { + let mut current = Some(el.clone()); + for _ in 0..4 { + let Some(candidate) = current else { + return Ok(false); + }; + if select_candidate(&candidate)? || select_candidate_in_container(&candidate) { + return Ok(true); + } + current = crate::tree::copy_element_attr(&candidate, "AXParent"); + } + Ok(false) + } + + fn select_candidate(candidate: &AXElement) -> Result { + Ok(ax_helpers::is_attr_settable(candidate, "AXSelected") + && ax_helpers::set_ax_bool_or_err(candidate, "AXSelected", true)? + && selected_state_settled(candidate)) + } + + fn selected_state_settled(candidate: &AXElement) -> bool { + std::thread::sleep(std::time::Duration::from_millis(50)); + crate::tree::copy_bool_attr(candidate, "AXSelected") == Some(true) + } + + fn select_candidate_in_container(candidate: &AXElement) -> bool { + for attr in ["AXSelectedChildren", "AXSelectedRows"] { + if set_container_selection(candidate, attr) + && container_selection_contains(candidate, attr) + { + return true; + } + } + false + } + + fn set_container_selection(candidate: &AXElement, attr: &str) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + let Some(container) = crate::tree::copy_element_attr(candidate, "AXParent") else { + return false; + }; + if !ax_helpers::is_attr_settable(&container, attr) { + return false; + } + unsafe { CFRetain(candidate.0 as CFTypeRef) }; + let candidate_cf = unsafe { CFType::wrap_under_create_rule(candidate.0 as CFTypeRef) }; + let selected = CFArray::from_CFTypes(&[candidate_cf]); + let cf_attr = CFString::new(attr); + let err = unsafe { + AXUIElementSetAttributeValue( + container.0, + cf_attr.as_concrete_TypeRef(), + selected.as_CFTypeRef(), + ) + }; + err == kAXErrorSuccess + } + + fn container_selection_contains(candidate: &AXElement, attr: &str) -> bool { + std::thread::sleep(std::time::Duration::from_millis(50)); + let Some(container) = crate::tree::copy_element_attr(candidate, "AXParent") else { + return false; + }; + crate::tree::copy_ax_array(&container, attr) + .unwrap_or_default() + .iter() + .any(|selected| crate::tree::same_element(selected, candidate)) + } + + pub fn try_select_via_parent( + el: &AXElement, + _caps: &ElementCaps, + ) -> Result { use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; use core_foundation::{ array::CFArray, @@ -334,16 +286,16 @@ mod imp { string::CFString, }; let Some(parent) = crate::tree::copy_element_attr(el, "AXParent") else { - return false; + return Ok(false); }; let Some(role) = crate::tree::copy_string_attr(&parent, kAXRoleAttribute) else { - return false; + return Ok(false); }; if !matches!(role.as_str(), "AXTable" | "AXOutline" | "AXList") { - return false; + return Ok(false); } if !ax_helpers::is_attr_settable(&parent, "AXSelectedRows") { - return false; + return Ok(false); } unsafe { CFRetain(el.0 as CFTypeRef) }; let el_cf = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; @@ -356,50 +308,14 @@ mod imp { arr.as_CFTypeRef(), ) }; - err == kAXErrorSuccess + Ok(err == kAXErrorSuccess) } - pub fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> bool { + pub fn try_custom_actions(el: &AXElement, _caps: &ElementCaps) -> Result { let has = !crate::tree::copy_ax_array(el, "AXCustomActions") .unwrap_or_default() .is_empty(); - has && ax_helpers::try_ax_action(el, "AXPerformCustomAction") - } - - pub fn try_keyboard_activate(el: &AXElement, _caps: &ElementCaps) -> bool { - use accessibility_sys::AXUIElementPostKeyboardEvent as PostKey; - if !ax_helpers::ax_focus(el) { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return false; - }; - let a = crate::tree::element_for_pid(pid); - unsafe { - PostKey(a.0, 0, 49, true); - PostKey(a.0, 0, 49, false); - } - true - } - - pub fn focus_app_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { - let Some(pid) = crate::system::app_ops::pid_from_element(el) else { - return false; - }; - let _ = crate::system::app_ops::ensure_app_focused(pid); - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_helpers::try_ax_action(el, "AXShowMenu") - } - - pub fn select_then_show_menu(el: &AXElement, _caps: &ElementCaps) -> bool { - if !ax_helpers::is_attr_settable(el, "AXSelected") - || !ax_helpers::set_ax_bool(el, "AXSelected", true) - { - return false; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - ax_helpers::try_ax_action(el, "AXShowMenu") + Ok(has && ax_helpers::try_ax_action_retried_or_err(el, "AXPerformCustomAction")?) } } diff --git a/crates/macos/src/actions/chain_steps_tests.rs b/crates/macos/src/actions/chain_steps_tests.rs new file mode 100644 index 0000000..d66521a --- /dev/null +++ b/crates/macos/src/actions/chain_steps_tests.rs @@ -0,0 +1,47 @@ +use agent_desktop_core::node::Rect; + +use super::chain_steps::{center_is_inside, rect_has_area}; + +#[test] +fn rect_area_requires_positive_dimensions() { + assert!(rect_has_area(&Rect { + x: 0.0, + y: 0.0, + width: 1.0, + height: 1.0, + })); + assert!(!rect_has_area(&Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 1.0, + })); +} + +#[test] +fn center_visibility_uses_parent_bounds() { + let outer = Rect { + x: 10.0, + y: 10.0, + width: 100.0, + height: 100.0, + }; + assert!(center_is_inside( + &Rect { + x: 20.0, + y: 20.0, + width: 10.0, + height: 10.0, + }, + &outer + )); + assert!(!center_is_inside( + &Rect { + x: 200.0, + y: 20.0, + width: 10.0, + height: 10.0, + }, + &outer + )); +} diff --git a/crates/macos/src/actions/chain_web_steps.rs b/crates/macos/src/actions/chain_web_steps.rs new file mode 100644 index 0000000..f31d903 --- /dev/null +++ b/crates/macos/src/actions/chain_web_steps.rs @@ -0,0 +1,118 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::actions::ax_helpers; + use crate::tree::AXElement; + + struct PreActionState { + focused: Option, + value: Option, + selected: Option, + } + + impl PreActionState { + fn capture(app: &AXElement, el: &AXElement) -> Self { + Self { + focused: crate::tree::copy_element_attr(app, "AXFocusedUIElement"), + value: crate::tree::copy_string_attr(el, "AXValue"), + selected: crate::tree::copy_bool_attr(el, "AXSelected"), + } + } + } + + pub fn is_in_webarea(el: &AXElement) -> bool { + use accessibility_sys::kAXRoleAttribute; + let mut current = crate::tree::copy_element_attr(el, "AXParent"); + for _ in 0..20 { + let Some(ref parent) = current else { + return false; + }; + if crate::tree::copy_string_attr(parent, kAXRoleAttribute).as_deref() + == Some("AXWebArea") + { + return true; + } + current = crate::tree::copy_element_attr(parent, "AXParent"); + } + false + } + + pub fn activate_web_element(el: &AXElement) -> bool { + let Some(pid) = crate::system::app_ops::pid_from_element(el) else { + return false; + }; + + let app = crate::tree::element_for_pid(pid); + let before = PreActionState::capture(&app, el); + + if ax_helpers::try_ax_action_retried(el, "AXPress") { + std::thread::sleep(std::time::Duration::from_millis(100)); + if web_action_had_effect(&app, el, &before) { + tracing::debug!("activate_web: AXPress had real effect"); + return true; + } + tracing::debug!("activate_web: AXPress returned success but no DOM effect"); + } + + let actions = ax_helpers::list_ax_actions(el); + for action in &["AXConfirm", "AXOpen"] { + if actions.iter().any(|a| a == action) && ax_helpers::try_ax_action(el, action) { + std::thread::sleep(std::time::Duration::from_millis(100)); + if web_action_had_effect(&app, el, &before) { + tracing::debug!("activate_web: {action} had real effect"); + return true; + } + } + } + + if ax_helpers::try_each_child( + el, + |child| { + let child_actions = ax_helpers::list_ax_actions(child); + ax_helpers::try_action_from_list( + child, + &child_actions, + &["AXPress", "AXConfirm", "AXOpen"], + ) + }, + 5, + ) { + std::thread::sleep(std::time::Duration::from_millis(100)); + if web_action_had_effect(&app, el, &before) { + tracing::debug!("activate_web: child action had real effect"); + return true; + } + } + + tracing::debug!("activate_web: all AX methods had no effect"); + false + } + + fn web_action_had_effect(app: &AXElement, el: &AXElement, before: &PreActionState) -> bool { + use core_foundation::base::{CFEqual, CFTypeRef}; + + let value_after = crate::tree::copy_string_attr(el, "AXValue"); + if before.value != value_after { + return true; + } + + let selected_after = crate::tree::copy_bool_attr(el, "AXSelected"); + if before.selected != selected_after { + return true; + } + + let focused_after = crate::tree::copy_element_attr(app, "AXFocusedUIElement"); + match (&before.focused, &focused_after) { + (Some(before_f), Some(after_f)) => unsafe { + CFEqual(before_f.0 as CFTypeRef, after_f.0 as CFTypeRef) == 0 + }, + (None, Some(_)) => true, + _ => false, + } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp {} + +#[cfg(target_os = "macos")] +pub(crate) use imp::*; diff --git a/crates/macos/src/actions/dispatch.rs b/crates/macos/src/actions/dispatch.rs index ae39c6d..fc24662 100644 --- a/crates/macos/src/actions/dispatch.rs +++ b/crates/macos/src/actions/dispatch.rs @@ -1,5 +1,8 @@ use agent_desktop_core::{ - action::{Action, ActionResult, MouseButton, MouseEvent, MouseEventKind, Point}, + action::{ + Action, ActionRequest, ActionResult, InteractionPolicy, MouseButton, MouseEvent, + MouseEventKind, Point, + }, error::{AdapterError, ErrorCode}, }; @@ -9,7 +12,7 @@ mod imp { use crate::actions::{ ax_helpers, chain::{execute_chain, ChainContext}, - chain_defs, discovery, + chain_defs, discovery, toggle_state, }; use crate::tree::AXElement; @@ -17,7 +20,13 @@ mod imp { el: &AXElement, button: MouseButton, count: u32, + policy: InteractionPolicy, ) -> Result<(), AdapterError> { + if !policy.allow_cursor_move || !policy.allow_focus_steal { + return Err(AdapterError::policy_denied( + "Physical click fallback is disabled by the current interaction policy", + )); + } if let Some(pid) = crate::system::app_ops::pid_from_element(el) { let _ = crate::system::app_ops::ensure_app_focused(pid); } @@ -49,16 +58,11 @@ mod imp { }) } - const TOGGLEABLE_ROLES: &[&str] = &[ - "checkbox", - "switch", - "radiobutton", - "togglebutton", - "menuitemcheckbox", - "menuitemradio", - ]; - - pub fn perform_action(el: &AXElement, action: &Action) -> Result { + pub fn perform_action( + el: &AXElement, + request: &ActionRequest, + ) -> Result { + let action = &request.action; let label = action_label(action); tracing::debug!("action: perform {label}"); match action { @@ -66,62 +70,61 @@ mod imp { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx, request.policy)?; } Action::DoubleClick => { let caps = discovery::discover(el); - chain_defs::double_click(el, &caps)?; + chain_defs::double_click(el, &caps, request.policy)?; } Action::RightClick => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::RIGHT_CLICK_CHAIN, &ctx)?; + execute_chain( + el, + &caps, + &chain_defs::RIGHT_CLICK_CHAIN, + &ctx, + request.policy, + )?; } Action::Toggle => { - let role = ax_helpers::element_role(el); - if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - format!( - "Toggle not supported on role '{}'", - role.as_deref().unwrap_or("unknown") - ), - ) - .with_suggestion( - "Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.", - )); - } - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - }; - execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx)?; + toggle_state::toggle(el, request.policy)?; } Action::SetValue(val) => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: Some(val), + deadline: None, }; - execute_chain(el, &caps, &chain_defs::SET_VALUE_CHAIN, &ctx)?; + execute_chain( + el, + &caps, + &chain_defs::SET_VALUE_CHAIN, + &ctx, + request.policy, + )?; } Action::SetFocus => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx)?; + execute_chain(el, &caps, &chain_defs::FOCUS_CHAIN, &ctx, request.policy)?; } Action::TypeText(text) => { - execute_type(el, text)?; + crate::actions::type_text::execute_type(el, text, request.policy)?; } Action::PressKey(combo) => { @@ -132,16 +135,18 @@ mod imp { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::EXPAND_CHAIN, &ctx)?; + execute_chain(el, &caps, &chain_defs::EXPAND_CHAIN, &ctx, request.policy)?; } Action::Collapse => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::COLLAPSE_CHAIN, &ctx)?; + execute_chain(el, &caps, &chain_defs::COLLAPSE_CHAIN, &ctx, request.policy)?; } Action::Select(value) => { @@ -149,36 +154,44 @@ mod imp { } Action::Scroll(direction, amount) => { - crate::actions::extras::ax_scroll(el, direction, *amount)?; + crate::actions::scroll::ax_scroll(el, direction, *amount, request.policy)?; } Action::Check => { - check_uncheck(el, true)?; + toggle_state::check_uncheck(el, true, request.policy)?; } Action::Uncheck => { - check_uncheck(el, false)?; + toggle_state::check_uncheck(el, false, request.policy)?; } Action::TripleClick => { let caps = discovery::discover(el); - chain_defs::triple_click(el, &caps)?; + chain_defs::triple_click(el, &caps, request.policy)?; } Action::ScrollTo => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: None, + deadline: None, }; - execute_chain(el, &caps, &chain_defs::SCROLL_TO_CHAIN, &ctx)?; + execute_chain( + el, + &caps, + &chain_defs::SCROLL_TO_CHAIN, + &ctx, + request.policy, + )?; } Action::Clear => { let caps = discovery::discover(el); let ctx = ChainContext { dynamic_value: Some(""), + deadline: None, }; - execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx)?; + execute_chain(el, &caps, &chain_defs::CLEAR_CHAIN, &ctx, request.policy)?; } Action::KeyDown(_) | Action::KeyUp(_) | Action::Hover | Action::Drag(_) => { @@ -198,77 +211,12 @@ mod imp { } let mut result = ActionResult::new(label); - if let Some(state) = read_post_state(el, action) { + if let Some(state) = crate::actions::post_state::read_post_state(el, action) { result = result.with_state(state); } Ok(result) } - fn execute_type(el: &AXElement, text: &str) -> Result<(), AdapterError> { - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let _ = crate::system::app_ops::ensure_app_focused(pid); - } - ax_helpers::ax_focus(el); - std::thread::sleep(std::time::Duration::from_millis(50)); - let has_non_ascii = !text.is_ascii(); - if has_non_ascii { - type_via_clipboard_paste(el, text) - } else { - crate::input::keyboard::synthesize_text(text) - } - } - - fn type_via_clipboard_paste(_el: &AXElement, text: &str) -> Result<(), AdapterError> { - let saved = crate::input::clipboard::get().ok(); - crate::input::clipboard::set(text)?; - std::thread::sleep(std::time::Duration::from_millis(50)); - - crate::input::keyboard::synthesize_key(&agent_desktop_core::action::KeyCombo { - key: "v".into(), - modifiers: vec![agent_desktop_core::action::Modifier::Cmd], - })?; - std::thread::sleep(std::time::Duration::from_millis(100)); - - if let Some(prev) = saved { - let _ = crate::input::clipboard::set(&prev); - } - Ok(()) - } - - fn read_post_state( - el: &AXElement, - action: &Action, - ) -> Option { - let delay_ms = match action { - Action::Click - | Action::Toggle - | Action::Check - | Action::Uncheck - | Action::TypeText(_) => 50, - Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0, - _ => return None, - }; - if delay_ms > 0 { - std::thread::sleep(std::time::Duration::from_millis(delay_ms)); - } - let value = crate::tree::copy_value_typed(el); - let role = ax_helpers::element_role(el).unwrap_or_default(); - let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false); - let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true); - let mut states = Vec::new(); - if focused { - states.push("focused".into()); - } - if !enabled { - states.push("disabled".into()); - } - Some(agent_desktop_core::action::ElementState { - role, - states, - value, - }) - } - pub fn ax_press_or_fail(el: &AXElement, context: &str) -> Result<(), AdapterError> { if !ax_helpers::ax_press(el) { return Err(AdapterError::new( @@ -279,30 +227,6 @@ mod imp { } Ok(()) } - - fn check_uncheck(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> { - let role = ax_helpers::element_role(el); - if !TOGGLEABLE_ROLES.iter().any(|r| role.as_deref() == Some(*r)) { - return Err(AdapterError::new( - ErrorCode::ActionNotSupported, - format!( - "check/uncheck not supported on role '{}'", - role.as_deref().unwrap_or("unknown") - ), - ) - .with_suggestion("Only works on checkboxes, switches, and radio buttons.")); - } - let current = crate::tree::copy_string_attr(el, "AXValue"); - let is_checked = current.as_deref() == Some("1"); - if is_checked == want_checked { - return Ok(()); - } - let caps = discovery::discover(el); - let ctx = ChainContext { - dynamic_value: None, - }; - execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx) - } } #[cfg(not(target_os = "macos"))] @@ -310,7 +234,10 @@ mod imp { use super::*; use crate::tree::AXElement; - pub fn perform_action(_el: &AXElement, _action: &Action) -> Result { + pub fn perform_action( + _el: &AXElement, + _request: &ActionRequest, + ) -> Result { Err(AdapterError::not_supported("perform_action")) } } diff --git a/crates/macos/src/actions/extras.rs b/crates/macos/src/actions/extras.rs index e6d4274..019c84a 100644 --- a/crates/macos/src/actions/extras.rs +++ b/crates/macos/src/actions/extras.rs @@ -7,25 +7,30 @@ use crate::tree::AXElement; #[cfg(target_os = "macos")] pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterError> { use crate::actions::ax_helpers; - use crate::actions::dispatch::ax_press_or_fail; let role = ax_helpers::element_role(el); match role.as_deref() { Some("combobox") => { - ax_helpers::ax_set_value(el, value)?; + if set_value_and_verify(el, value) { + return Ok(()); + } + let pid = crate::system::app_ops::pid_from_element(el); + open_menu(el, pid, "select (open combo box)")?; + if !wait_for_menu_item(el, pid, value) { + press_escape(el); + return Err(option_not_found(value)); + } + wait_for_value(el, value)?; } Some("popupbutton") | Some("menubutton") => { - ax_press_or_fail(el, "select (open popup)")?; - std::thread::sleep(std::time::Duration::from_millis(200)); - if !find_and_press_menu_item(el, value) { + let pid = crate::system::app_ops::pid_from_element(el); + open_menu(el, pid, "select (open popup)")?; + if !wait_for_menu_item(el, pid, value) { press_escape(el); - return Err(AdapterError::new( - ErrorCode::ElementNotFound, - format!("No menu item matching '{value}' found"), - ) - .with_suggestion( - "Use 'click' to open the menu, then 'snapshot' to see available options.", - )); + return Err(option_not_found(value)); + } + if crate::tree::copy_value_typed(el).is_some() { + wait_for_value(el, value)?; } } Some("list") | Some("table") | Some("outline") => { @@ -53,6 +58,112 @@ pub(crate) fn select_value(el: &AXElement, value: &str) -> Result<(), AdapterErr Ok(()) } +#[cfg(target_os = "macos")] +fn set_value_and_verify(el: &AXElement, value: &str) -> bool { + crate::actions::ax_helpers::ax_set_value(el, value).is_ok() && wait_for_value(el, value).is_ok() +} + +#[cfg(target_os = "macos")] +fn wait_for_value(el: &AXElement, value: &str) -> Result<(), AdapterError> { + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(600); + loop { + if crate::tree::copy_value_typed(el) + .as_deref() + .is_some_and(|current| current.eq_ignore_ascii_case(value)) + { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + format!("Selection did not change to '{value}'"), + ) + .with_suggestion("Refresh the snapshot and inspect available values.")); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } +} + +#[cfg(target_os = "macos")] +fn option_not_found(value: &str) -> AdapterError { + AdapterError::new( + ErrorCode::ElementNotFound, + format!("No menu item matching '{value}' found"), + ) + .with_suggestion("Use 'click' to open the menu, then 'snapshot' to see available options.") +} + +#[cfg(target_os = "macos")] +fn find_and_press_open_menu_item(pid: i32, target_value: &str) -> bool { + crate::tree::surfaces::menu_element_for_pid(pid) + .as_ref() + .is_some_and(|menu| find_and_press_menu_item(menu, target_value)) +} + +#[cfg(target_os = "macos")] +fn open_menu(el: &AXElement, pid: Option, context: &str) -> Result<(), AdapterError> { + let was_open = pid.is_some_and(is_menu_open); + if was_open { + return Err(menu_already_open()); + } + if crate::actions::ax_helpers::try_ax_action_retried(el, "AXShowMenu") + && menu_opened_after_action(was_open, wait_for_open_menu(pid)) + { + return Ok(()); + } + crate::actions::dispatch::ax_press_or_fail(el, context)?; + if pid.is_none() || menu_opened_after_action(was_open, wait_for_open_menu(pid)) { + return Ok(()); + } + Err(AdapterError::timeout(format!( + "No context menu opened within {}ms", + crate::system::wait::menu_timeout_ms() + ))) +} + +#[cfg(target_os = "macos")] +fn wait_for_open_menu(pid: Option) -> bool { + pid.is_some_and(|p| { + crate::system::wait::wait_for_menu(p, true, crate::system::wait::menu_timeout_ms()).is_ok() + }) +} + +#[cfg(target_os = "macos")] +fn is_menu_open(pid: i32) -> bool { + crate::system::wait::wait_for_menu(pid, true, 60).is_ok() +} + +#[cfg(target_os = "macos")] +fn wait_for_menu_item(el: &AXElement, pid: Option, target_value: &str) -> bool { + let deadline = std::time::Instant::now() + + std::time::Duration::from_millis(crate::system::wait::menu_timeout_ms()); + loop { + if find_and_press_menu_item(el, target_value) + || pid.is_some_and(|p| find_and_press_open_menu_item(p, target_value)) + { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } +} + +#[cfg(target_os = "macos")] +fn menu_already_open() -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + "Refusing to select from a menu while another menu is already open", + ) + .with_suggestion("Dismiss the open menu and retry the select command.") +} + +#[cfg(target_os = "macos")] +fn menu_opened_after_action(was_open: bool, is_open: bool) -> bool { + !was_open && is_open +} + #[cfg(target_os = "macos")] fn find_and_press_menu_item(el: &AXElement, target_value: &str) -> bool { use accessibility_sys::{kAXPressAction, AXUIElementPerformAction}; @@ -104,293 +215,19 @@ fn select_child_by_name(el: &AXElement, name: &str) -> bool { false } -#[cfg(target_os = "macos")] -pub(crate) fn ax_scroll( - el: &AXElement, - direction: &agent_desktop_core::action::Direction, - amount: u32, -) -> Result<(), AdapterError> { - use accessibility_sys::{ - kAXErrorSuccess, AXUIElementPerformAction, AXUIElementPostKeyboardEvent, - AXUIElementSetAttributeValue, - }; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::menu_opened_after_action; - let scroll_area = find_scroll_area(el); - let target = scroll_area.as_ref().unwrap_or(el); - - let scroll_visible = CFString::new("AXScrollToVisible"); - if unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) } - == kAXErrorSuccess - { - return Ok(()); + #[test] + fn menu_guard_rejects_preexisting_menu() { + assert!(!menu_opened_after_action(true, true)); + assert!(!menu_opened_after_action(true, false)); } - let (bar_attr, inc_action) = match direction { - Direction::Down => ("AXVerticalScrollBar", "AXIncrement"), - Direction::Up => ("AXVerticalScrollBar", "AXDecrement"), - Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"), - Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"), - }; - - if let Some(bar) = get_scroll_bar(target, bar_attr) { - let ax_action = CFString::new(inc_action); - let mut ok = true; - for _ in 0..amount { - if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) } - != kAXErrorSuccess - { - ok = false; - break; - } - } - if ok { - return Ok(()); - } + #[test] + fn menu_guard_requires_closed_to_open_transition() { + assert!(menu_opened_after_action(false, true)); + assert!(!menu_opened_after_action(false, false)); } - - let page_action = match direction { - Direction::Down => "AXScrollDownByPage", - Direction::Up => "AXScrollUpByPage", - Direction::Right => "AXScrollRightByPage", - Direction::Left => "AXScrollLeftByPage", - }; - if crate::actions::ax_helpers::has_ax_action(target, page_action) { - let ax = CFString::new(page_action); - for _ in 0..amount { - unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) }; - } - return Ok(()); - } - - if let Some(bar) = get_scroll_bar(target, bar_attr) { - if try_scroll_bar_value_shift(&bar, direction, amount) { - return Ok(()); - } - if try_scroll_bar_sub_elements(&bar, direction) { - return Ok(()); - } - } - - if try_focus_child_in_direction(target, direction) { - return Ok(()); - } - if try_select_row_in_direction(target, direction) { - return Ok(()); - } - - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let keycode: u16 = match direction { - Direction::Down => 121, - Direction::Up => 116, - Direction::Right => 124, - Direction::Left => 123, - }; - let cf_focused = CFString::new("AXFocused"); - unsafe { - AXUIElementSetAttributeValue( - target.0, - cf_focused.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - std::thread::sleep(std::time::Duration::from_millis(50)); - let app = crate::tree::element_for_pid(pid); - for _ in 0..amount { - unsafe { - AXUIElementPostKeyboardEvent(app.0, 0, keycode, true); - AXUIElementPostKeyboardEvent(app.0, 0, keycode, false); - }; - } - return Ok(()); - } - - if let Some(pid) = crate::system::app_ops::pid_from_element(el) { - let _ = crate::system::app_ops::ensure_app_focused(pid); - if let Some(b) = crate::tree::read_bounds(target) { - let (dy, dx) = match direction { - Direction::Down => (-(amount as i32) * 5, 0), - Direction::Up => (amount as i32 * 5, 0), - Direction::Right => (0, -(amount as i32) * 5), - Direction::Left => (0, amount as i32 * 5), - }; - return crate::input::mouse::synthesize_scroll_at( - b.x + b.width / 2.0, - b.y + b.height / 2.0, - dy, - dx, - ); - } - } - - Err(AdapterError::new( - ErrorCode::ActionNotSupported, - "No scroll mechanism found on element", - ) - .with_suggestion("Element may not be scrollable, or try the parent container.")) -} - -#[cfg(target_os = "macos")] -fn try_scroll_bar_value_shift( - bar: &AXElement, - direction: &agent_desktop_core::action::Direction, - amount: u32, -) -> bool { - use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; - - if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") { - return false; - } - let current = read_scroll_bar_value(bar).unwrap_or(0.0); - let delta = 0.1 * amount as f64; - let new_val = match direction { - Direction::Down | Direction::Right => (current + delta).min(1.0), - Direction::Up | Direction::Left => (current - delta).max(0.0), - }; - let cf_num = CFNumber::from(new_val as f32); - let cf_attr = CFString::new("AXValue"); - let err = unsafe { - AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef()) - }; - err == kAXErrorSuccess -} - -#[cfg(target_os = "macos")] -fn read_scroll_bar_value(bar: &AXElement) -> Option { - use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue}; - use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; - - let cf_attr = CFString::new("AXValue"); - let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut(); - let err = - unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) }; - if err != kAXErrorSuccess || value.is_null() { - return None; - } - let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; - cf.downcast::().and_then(|n| n.to_f64()) -} - -#[cfg(target_os = "macos")] -fn try_scroll_bar_sub_elements( - bar: &AXElement, - direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction}; - use agent_desktop_core::action::Direction; - use core_foundation::{base::TCFType, string::CFString}; - - let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default(); - let target_subroles = match direction { - Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"], - Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"], - }; - let press = CFString::new("AXPress"); - for child in &children { - let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default(); - if target_subroles.iter().any(|t| *t == sr) - && unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) } - == kAXErrorSuccess - { - return true; - } - } - false -} - -#[cfg(target_os = "macos")] -fn try_focus_child_in_direction( - scroll_area: &AXElement, - _direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; - use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; - - let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); - let child = match children.first() { - Some(c) => c, - None => return false, - }; - let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default(); - let target = match grandchildren.last() { - Some(t) => t, - None => return false, - }; - let cf_attr = CFString::new("AXFocused"); - let err = unsafe { - AXUIElementSetAttributeValue( - target.0, - cf_attr.as_concrete_TypeRef(), - CFBoolean::true_value().as_CFTypeRef(), - ) - }; - err == kAXErrorSuccess -} - -#[cfg(target_os = "macos")] -fn try_select_row_in_direction( - scroll_area: &AXElement, - _direction: &agent_desktop_core::action::Direction, -) -> bool { - use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; - use core_foundation::{ - array::CFArray, - base::{CFRetain, CFType, CFTypeRef, TCFType}, - string::CFString, - }; - - let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); - for child in &children { - let role = crate::tree::copy_string_attr(child, kAXRoleAttribute); - if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) { - continue; - } - if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") { - continue; - } - let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default(); - if let Some(last) = rows.last() { - unsafe { CFRetain(last.0 as CFTypeRef) }; - let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) }; - let arr = CFArray::from_CFTypes(&[el_as_cftype]); - let cf_attr = CFString::new("AXSelectedRows"); - let err = unsafe { - AXUIElementSetAttributeValue( - child.0, - cf_attr.as_concrete_TypeRef(), - arr.as_CFTypeRef(), - ) - }; - if err == kAXErrorSuccess { - return true; - } - } - } - false -} - -#[cfg(target_os = "macos")] -fn find_scroll_area(el: &AXElement) -> Option { - use accessibility_sys::kAXRoleAttribute; - let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?; - if role == "AXScrollArea" { - return Some(el.clone()); - } - let mut current = crate::tree::copy_element_attr(el, "AXParent")?; - for _ in 0..5 { - let r = crate::tree::copy_string_attr(¤t, kAXRoleAttribute)?; - if r == "AXScrollArea" { - return Some(current); - } - current = crate::tree::copy_element_attr(¤t, "AXParent")?; - } - None -} - -#[cfg(target_os = "macos")] -fn get_scroll_bar(scroll_area: &AXElement, bar_attr: &str) -> Option { - crate::tree::copy_element_attr(scroll_area, bar_attr) } diff --git a/crates/macos/src/actions/mod.rs b/crates/macos/src/actions/mod.rs index 6fde1b3..185f9a6 100644 --- a/crates/macos/src/actions/mod.rs +++ b/crates/macos/src/actions/mod.rs @@ -1,9 +1,21 @@ -pub mod ax_helpers; -pub mod chain; -pub mod chain_defs; -pub mod chain_steps; -pub mod discovery; -pub mod dispatch; -pub mod extras; +pub(crate) mod ax_helpers; +pub(crate) mod chain; +mod chain_context; +mod chain_def; +pub(crate) mod chain_defs; +pub(crate) mod chain_menu_steps; +mod chain_step; +pub(crate) mod chain_steps; +pub(crate) mod chain_web_steps; +pub(crate) mod discovery; +pub(crate) mod dispatch; +pub(crate) mod extras; +pub(crate) mod post_state; +pub(crate) mod scroll; +pub(crate) mod toggle_state; +pub(crate) mod type_text; -pub use dispatch::perform_action; +#[cfg(test)] +mod chain_steps_tests; + +pub(crate) use dispatch::perform_action; diff --git a/crates/macos/src/actions/post_state.rs b/crates/macos/src/actions/post_state.rs new file mode 100644 index 0000000..7022383 --- /dev/null +++ b/crates/macos/src/actions/post_state.rs @@ -0,0 +1,51 @@ +use agent_desktop_core::action::{Action, ElementState}; + +#[cfg(target_os = "macos")] +pub(crate) fn read_post_state( + el: &crate::tree::AXElement, + action: &Action, +) -> Option { + let delay_ms = match action { + Action::Click | Action::Toggle | Action::Check | Action::Uncheck | Action::TypeText(_) => { + 50 + } + Action::SetValue(_) | Action::Clear | Action::Expand | Action::Collapse => 0, + _ => return None, + }; + if delay_ms > 0 { + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + } + Some(read_element_state(el)) +} + +pub(crate) fn read_element_state(el: &crate::tree::AXElement) -> ElementState { + let value = crate::tree::copy_value_typed(el); + let role = crate::actions::ax_helpers::element_role(el).unwrap_or_default(); + let focused = crate::tree::element::copy_bool_attr(el, "AXFocused").unwrap_or(false); + let enabled = crate::tree::element::copy_bool_attr(el, "AXEnabled").unwrap_or(true); + let expanded = crate::tree::element::copy_bool_attr(el, "AXExpanded") + .or_else(|| crate::tree::element::copy_bool_attr(el, "AXDisclosing")) + .unwrap_or(false); + let mut states = Vec::new(); + if focused { + states.push("focused".into()); + } + if !enabled { + states.push("disabled".into()); + } + if expanded { + states.push("expanded".into()); + } + if crate::tree::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { + states.push("checked".into()); + } + ElementState { + role, + states, + value, + } +} + +fn value_is_checked(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true")) +} diff --git a/crates/macos/src/actions/scroll.rs b/crates/macos/src/actions/scroll.rs new file mode 100644 index 0000000..90edaa7 --- /dev/null +++ b/crates/macos/src/actions/scroll.rs @@ -0,0 +1,328 @@ +#[cfg(target_os = "macos")] +use agent_desktop_core::error::{AdapterError, ErrorCode}; + +#[cfg(target_os = "macos")] +use crate::tree::AXElement; + +#[cfg(target_os = "macos")] +pub(crate) fn ax_scroll( + el: &AXElement, + direction: &agent_desktop_core::action::Direction, + amount: u32, + policy: agent_desktop_core::action::InteractionPolicy, +) -> Result<(), AdapterError> { + use accessibility_sys::{ + kAXErrorSuccess, AXUIElementPerformAction, AXUIElementSetAttributeValue, + }; + use agent_desktop_core::action::Direction; + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + let scroll_area = find_scroll_area(el); + let target = scroll_area.as_ref().unwrap_or(el); + + let scroll_visible = CFString::new("AXScrollToVisible"); + unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) }; + + let (bar_attr, inc_action) = match direction { + Direction::Down => ("AXVerticalScrollBar", "AXIncrement"), + Direction::Up => ("AXVerticalScrollBar", "AXDecrement"), + Direction::Right => ("AXHorizontalScrollBar", "AXIncrement"), + Direction::Left => ("AXHorizontalScrollBar", "AXDecrement"), + }; + + if let Some(bar) = get_scroll_bar(target, bar_attr) { + let ax_action = CFString::new(inc_action); + let mut ok = true; + for _ in 0..amount { + if unsafe { AXUIElementPerformAction(bar.0, ax_action.as_concrete_TypeRef()) } + != kAXErrorSuccess + { + ok = false; + break; + } + } + if ok { + return Ok(()); + } + } + + let page_action = match direction { + Direction::Down => "AXScrollDownByPage", + Direction::Up => "AXScrollUpByPage", + Direction::Right => "AXScrollRightByPage", + Direction::Left => "AXScrollLeftByPage", + }; + if crate::actions::ax_helpers::has_ax_action(target, page_action) { + let ax = CFString::new(page_action); + let mut completed = 0; + for _ in 0..amount { + if unsafe { AXUIElementPerformAction(target.0, ax.as_concrete_TypeRef()) } + != kAXErrorSuccess + { + break; + } + completed += 1; + } + if completed == amount { + return Ok(()); + } + } + + if let Some(bar) = get_scroll_bar(target, bar_attr) { + if try_scroll_bar_value_shift(&bar, direction, amount) { + return Ok(()); + } + if try_scroll_bar_sub_elements(&bar, direction) { + return Ok(()); + } + } + + if policy.allow_focus_steal && try_focus_child_in_direction(target, direction) { + return Ok(()); + } + if try_select_row_in_direction(target, direction) { + return Ok(()); + } + + if policy.allow_focus_steal { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let keycode: u16 = match direction { + Direction::Down => 121, + Direction::Up => 116, + Direction::Right => 124, + Direction::Left => 123, + }; + let _ = crate::system::app_ops::ensure_app_focused(pid); + let cf_focused = CFString::new("AXFocused"); + let focus_err = unsafe { + AXUIElementSetAttributeValue( + target.0, + cf_focused.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if focus_err == kAXErrorSuccess { + std::thread::sleep(std::time::Duration::from_millis(50)); + crate::input::keyboard::synthesize_keycode(keycode, amount)?; + return Ok(()); + } + } + } + + if policy.allow_focus_steal && policy.allow_cursor_move { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + if let Some(b) = crate::tree::read_bounds(target) { + let (dy, dx) = scroll_wheel_delta(direction, amount); + return crate::input::mouse::synthesize_scroll_at( + b.x + b.width / 2.0, + b.y + b.height / 2.0, + dy, + dx, + ); + } + } + } + + if policy.allow_focus_steal && !policy.allow_cursor_move { + return Err(AdapterError::policy_denied( + "Cursor-moving scroll fallback is disabled by the current interaction policy", + )); + } + + Err(AdapterError::new( + ErrorCode::ActionNotSupported, + "No scroll mechanism found on element", + ) + .with_suggestion("Element may not be scrollable, or try the parent container.")) +} + +#[cfg(target_os = "macos")] +fn scroll_wheel_delta( + direction: &agent_desktop_core::action::Direction, + amount: u32, +) -> (i32, i32) { + use agent_desktop_core::action::Direction; + match direction { + Direction::Down => (-(amount as i32) * 5, 0), + Direction::Up => (amount as i32 * 5, 0), + Direction::Right => (0, amount as i32 * 5), + Direction::Left => (0, -(amount as i32) * 5), + } +} + +#[cfg(target_os = "macos")] +fn try_scroll_bar_value_shift( + bar: &AXElement, + direction: &agent_desktop_core::action::Direction, + amount: u32, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use agent_desktop_core::action::Direction; + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + if !crate::actions::ax_helpers::is_attr_settable(bar, "AXValue") { + return false; + } + let current = read_scroll_bar_value(bar).unwrap_or(0.0); + let delta = 0.1 * amount as f64; + let new_val = match direction { + Direction::Down | Direction::Right => (current + delta).min(1.0), + Direction::Up | Direction::Left => (current - delta).max(0.0), + }; + let cf_num = CFNumber::from(new_val as f32); + let cf_attr = CFString::new("AXValue"); + let err = unsafe { + AXUIElementSetAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), cf_num.as_CFTypeRef()) + }; + err == kAXErrorSuccess +} + +#[cfg(target_os = "macos")] +fn read_scroll_bar_value(bar: &AXElement) -> Option { + use accessibility_sys::{kAXErrorSuccess, AXUIElementCopyAttributeValue}; + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + + let cf_attr = CFString::new("AXValue"); + let mut value: core_foundation::base::CFTypeRef = std::ptr::null_mut(); + let err = + unsafe { AXUIElementCopyAttributeValue(bar.0, cf_attr.as_concrete_TypeRef(), &mut value) }; + if err != kAXErrorSuccess || value.is_null() { + return None; + } + let cf = unsafe { core_foundation::base::CFType::wrap_under_create_rule(value) }; + cf.downcast::().and_then(|n| n.to_f64()) +} + +#[cfg(target_os = "macos")] +fn try_scroll_bar_sub_elements( + bar: &AXElement, + direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementPerformAction}; + use agent_desktop_core::action::Direction; + use core_foundation::{base::TCFType, string::CFString}; + + let children = crate::tree::copy_ax_array(bar, "AXChildren").unwrap_or_default(); + let target_subroles = match direction { + Direction::Down | Direction::Right => &["AXIncrementPage", "AXIncrementArrow"], + Direction::Up | Direction::Left => &["AXDecrementPage", "AXDecrementArrow"], + }; + let press = CFString::new("AXPress"); + for child in &children { + let sr = crate::tree::copy_string_attr(child, "AXSubrole").unwrap_or_default(); + if target_subroles.iter().any(|t| *t == sr) + && unsafe { AXUIElementPerformAction(child.0, press.as_concrete_TypeRef()) } + == kAXErrorSuccess + { + return true; + } + } + false +} + +#[cfg(target_os = "macos")] +fn try_focus_child_in_direction( + scroll_area: &AXElement, + _direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, AXUIElementSetAttributeValue}; + use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; + + let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); + let child = match children.first() { + Some(c) => c, + None => return false, + }; + let grandchildren = crate::tree::copy_ax_array(child, "AXChildren").unwrap_or_default(); + let target = match grandchildren.last() { + Some(t) => t, + None => return false, + }; + let cf_attr = CFString::new("AXFocused"); + let err = unsafe { + AXUIElementSetAttributeValue( + target.0, + cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + err == kAXErrorSuccess +} + +#[cfg(target_os = "macos")] +fn try_select_row_in_direction( + scroll_area: &AXElement, + _direction: &agent_desktop_core::action::Direction, +) -> bool { + use accessibility_sys::{kAXErrorSuccess, kAXRoleAttribute, AXUIElementSetAttributeValue}; + use core_foundation::{ + array::CFArray, + base::{CFRetain, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + + let children = crate::tree::copy_ax_array(scroll_area, "AXChildren").unwrap_or_default(); + for child in &children { + let role = crate::tree::copy_string_attr(child, kAXRoleAttribute); + if !matches!(role.as_deref(), Some("AXTable" | "AXOutline" | "AXList")) { + continue; + } + if !crate::actions::ax_helpers::is_attr_settable(child, "AXSelectedRows") { + continue; + } + let rows = crate::tree::copy_ax_array(child, "AXRows").unwrap_or_default(); + if let Some(last) = rows.last() { + unsafe { CFRetain(last.0 as CFTypeRef) }; + let el_as_cftype = unsafe { CFType::wrap_under_create_rule(last.0 as CFTypeRef) }; + let arr = CFArray::from_CFTypes(&[el_as_cftype]); + let cf_attr = CFString::new("AXSelectedRows"); + let err = unsafe { + AXUIElementSetAttributeValue( + child.0, + cf_attr.as_concrete_TypeRef(), + arr.as_CFTypeRef(), + ) + }; + if err == kAXErrorSuccess { + return true; + } + } + } + false +} + +#[cfg(target_os = "macos")] +fn find_scroll_area(el: &AXElement) -> Option { + use accessibility_sys::kAXRoleAttribute; + + let role = crate::tree::copy_string_attr(el, kAXRoleAttribute)?; + if role == "AXScrollArea" { + return Some(el.clone()); + } + let mut current = crate::tree::copy_element_attr(el, "AXParent")?; + for _ in 0..5 { + let r = crate::tree::copy_string_attr(¤t, kAXRoleAttribute)?; + if r == "AXScrollArea" { + return Some(current); + } + current = crate::tree::copy_element_attr(¤t, "AXParent")?; + } + None +} + +#[cfg(target_os = "macos")] +fn get_scroll_bar(scroll_area: &AXElement, bar_attr: &str) -> Option { + crate::tree::copy_element_attr(scroll_area, bar_attr) +} + +#[cfg(test)] +mod tests { + use agent_desktop_core::action::Direction; + + #[test] + fn horizontal_wheel_delta_matches_direction() { + assert_eq!(super::scroll_wheel_delta(&Direction::Right, 2), (0, 10)); + assert_eq!(super::scroll_wheel_delta(&Direction::Left, 2), (0, -10)); + } +} diff --git a/crates/macos/src/actions/toggle_state.rs b/crates/macos/src/actions/toggle_state.rs new file mode 100644 index 0000000..767ceb8 --- /dev/null +++ b/crates/macos/src/actions/toggle_state.rs @@ -0,0 +1,195 @@ +use agent_desktop_core::{ + action::InteractionPolicy, + error::{AdapterError, ErrorCode}, +}; + +use crate::{ + actions::{ + ax_helpers, + chain::{execute_chain, ChainContext}, + chain_defs, discovery, + }, + tree::AXElement, +}; + +const DEFAULT_TOGGLE_TIMEOUT_MS: u64 = 600; +const MAX_TOGGLE_TIMEOUT_MS: u64 = 10_000; +const DEFAULT_TOGGLE_STABLE_MS: u64 = 200; +const MAX_TOGGLE_STABLE_MS: u64 = 2_000; + +pub(crate) fn toggle(el: &AXElement, policy: InteractionPolicy) -> Result<(), AdapterError> { + let role = ax_helpers::element_role(el); + if !role + .as_deref() + .is_some_and(crate::tree::roles::is_toggleable_role) + { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!( + "Toggle not supported on role '{}'", + role.as_deref().unwrap_or("unknown") + ), + ) + .with_suggestion( + "Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements.", + )); + } + let before = crate::tree::copy_value_typed(el); + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + deadline: None, + }; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx, policy)?; + if let Some(before) = before { + wait_for_value_change(el, &before)?; + } + Ok(()) +} + +pub(crate) fn check_uncheck( + el: &AXElement, + want_checked: bool, + policy: InteractionPolicy, +) -> Result<(), AdapterError> { + let role = ax_helpers::element_role(el); + if !role + .as_deref() + .is_some_and(crate::tree::roles::is_toggleable_role) + { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!( + "check/uncheck not supported on role '{}'", + role.as_deref().unwrap_or("unknown") + ), + ) + .with_suggestion("Only works on checkboxes, switches, and radio buttons.")); + } + if checked_state(el) == Some(want_checked) { + return Ok(()); + } + if ax_helpers::is_attr_settable(el, "AXValue") + && ax_helpers::set_ax_bool(el, "AXValue", want_checked) + && wait_for_checked_state(el, want_checked).is_ok() + { + return Ok(()); + } + let caps = discovery::discover(el); + let ctx = ChainContext { + dynamic_value: None, + deadline: None, + }; + execute_chain(el, &caps, &chain_defs::CLICK_CHAIN, &ctx, policy)?; + wait_for_checked_state(el, want_checked) +} + +fn checked_state(el: &AXElement) -> Option { + crate::tree::copy_value_typed(el).and_then(|value| parse_checked_value(&value)) +} + +fn parse_checked_value(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" | "checked" => Some(true), + "0" | "false" | "no" | "off" | "unchecked" => Some(false), + "2" | "mixed" | "indeterminate" => None, + _ => None, + } +} + +fn wait_for_checked_state(el: &AXElement, want_checked: bool) -> Result<(), AdapterError> { + let deadline = std::time::Instant::now() + toggle_timeout(); + loop { + if checked_state(el) == Some(want_checked) { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "check/uncheck did not reach the requested state", + ) + .with_suggestion("Retry after refreshing the snapshot.")); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } +} + +fn wait_for_value_change(el: &AXElement, before: &str) -> Result<(), AdapterError> { + let deadline = std::time::Instant::now() + toggle_timeout(); + let stable_for = toggle_stable_duration(); + let mut candidate: Option<(String, std::time::Instant)> = None; + loop { + if let Some(changed) = crate::tree::copy_value_typed(el) { + if changed != before { + match &mut candidate { + Some((candidate_value, since)) if candidate_value == &changed => { + if since.elapsed() >= stable_for { + return Ok(()); + } + } + _ => { + candidate = Some((changed, std::time::Instant::now())); + } + } + } else { + candidate = None; + } + } + if std::time::Instant::now() >= deadline { + return Err(AdapterError::new( + ErrorCode::ActionFailed, + "toggle did not change the element value", + ) + .with_suggestion("Use 'click' for controls that do not expose stable toggle state.")); + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } +} + +fn toggle_timeout() -> std::time::Duration { + env_duration_ms( + "AGENT_DESKTOP_TOGGLE_TIMEOUT_MS", + DEFAULT_TOGGLE_TIMEOUT_MS, + MAX_TOGGLE_TIMEOUT_MS, + ) +} + +fn toggle_stable_duration() -> std::time::Duration { + env_duration_ms( + "AGENT_DESKTOP_TOGGLE_STABLE_MS", + DEFAULT_TOGGLE_STABLE_MS, + MAX_TOGGLE_STABLE_MS, + ) +} + +fn env_duration_ms(name: &str, default_ms: u64, max_ms: u64) -> std::time::Duration { + let ms = std::env::var(name) + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|ms| *ms > 0) + .map(|ms| ms.min(max_ms)) + .unwrap_or(default_ms); + std::time::Duration::from_millis(ms) +} + +#[cfg(test)] +mod tests { + use super::parse_checked_value; + + #[test] + fn parses_checked_values_from_common_ax_strings() { + for value in ["1", "true", "TRUE", "YES", "on", "checked"] { + assert_eq!(parse_checked_value(value), Some(true)); + } + for value in ["0", "false", "FALSE", "NO", "off", "unchecked"] { + assert_eq!(parse_checked_value(value), Some(false)); + } + } + + #[test] + fn treats_mixed_and_unknown_checked_values_as_indeterminate() { + for value in ["2", "mixed", "indeterminate", "maybe", ""] { + assert_eq!(parse_checked_value(value), None); + } + } +} diff --git a/crates/macos/src/actions/type_text.rs b/crates/macos/src/actions/type_text.rs new file mode 100644 index 0000000..ee08987 --- /dev/null +++ b/crates/macos/src/actions/type_text.rs @@ -0,0 +1,157 @@ +#[cfg(target_os = "macos")] +use agent_desktop_core::{ + action::InteractionPolicy, + error::{AdapterError, ErrorCode}, +}; + +#[cfg(target_os = "macos")] +use crate::tree::AXElement; + +#[cfg(target_os = "macos")] +pub(crate) fn execute_type( + el: &AXElement, + text: &str, + policy: InteractionPolicy, +) -> Result<(), AdapterError> { + match type_via_ax_value(el, text) { + Ok(()) => return Ok(()), + Err(err) if !policy.allow_focus_steal => return Err(err), + Err(_) => {} + } + + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + crate::actions::ax_helpers::ax_focus_or_err(el)?; + std::thread::sleep(std::time::Duration::from_millis(50)); + if !text.is_ascii() { + return type_via_clipboard_paste(el, text); + } + crate::input::keyboard::synthesize_text(text) +} + +#[cfg(target_os = "macos")] +fn type_via_clipboard_paste(el: &AXElement, text: &str) -> Result<(), AdapterError> { + let before = readable_value(el); + let previous = crate::input::clipboard::get().unwrap_or_default(); + crate::input::clipboard::set(text)?; + std::thread::sleep(std::time::Duration::from_millis(50)); + let combo = agent_desktop_core::action::KeyCombo { + key: "v".into(), + modifiers: vec![agent_desktop_core::action::Modifier::Cmd], + }; + let paste_result = crate::input::keyboard::synthesize_key(&combo); + std::thread::sleep(std::time::Duration::from_millis(100)); + let restore_result = crate::input::clipboard::set(&previous); + paste_result?; + restore_result?; + verify_paste_effect(before.as_deref(), readable_value(el).as_deref()) +} + +#[cfg(target_os = "macos")] +fn type_via_ax_value(el: &AXElement, text: &str) -> Result<(), AdapterError> { + if !is_text_target(el) || !crate::actions::ax_helpers::is_attr_settable(el, "AXValue") { + return Err(AdapterError::policy_denied( + "Headless typing requires a settable text value; use set-value or an explicit focus command", + )); + } + + let current = if is_secure_text_field(el) { + String::new() + } else { + crate::tree::copy_value_typed(el).unwrap_or_default() + }; + let next = typed_value(¤t, text); + crate::actions::ax_helpers::ax_set_value(el, &next)?; + if is_secure_text_field(el) { + return Ok(()); + } + let after = crate::tree::copy_value_typed(el).unwrap_or_default(); + if after == next { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::ActionFailed, + "AX value write reported success but the element value did not change", + ) + .with_suggestion( + "Use explicit keyboard input for web-backed fields that ignore AXValue writes", + )) +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn execute_type( + _el: &crate::tree::AXElement, + _text: &str, + _policy: agent_desktop_core::action::InteractionPolicy, +) -> Result<(), agent_desktop_core::error::AdapterError> { + Err(agent_desktop_core::error::AdapterError::new( + agent_desktop_core::error::ErrorCode::PlatformNotSupported, + "type_text is not supported on this platform", + )) +} + +#[cfg(target_os = "macos")] +fn is_text_target(el: &AXElement) -> bool { + matches!( + crate::actions::ax_helpers::element_role(el).as_deref(), + Some("textfield" | "combobox") + ) +} + +#[cfg(target_os = "macos")] +fn is_secure_text_field(el: &AXElement) -> bool { + crate::tree::copy_string_attr(el, "AXRole").as_deref() == Some("AXSecureTextField") +} + +#[cfg(target_os = "macos")] +fn readable_value(el: &AXElement) -> Option { + if is_secure_text_field(el) { + None + } else { + crate::tree::copy_value_typed(el) + } +} + +#[cfg(target_os = "macos")] +fn verify_paste_effect(before: Option<&str>, after: Option<&str>) -> Result<(), AdapterError> { + if before.is_none() || after.is_none() || before != after { + return Ok(()); + } + Err(AdapterError::new( + ErrorCode::ActionFailed, + "Clipboard paste completed but the target value did not change", + ) + .with_suggestion( + "Use set-value for fields that expose AXValue, or retry with physical keyboard input.", + )) +} + +#[cfg(target_os = "macos")] +fn typed_value(current: &str, text: &str) -> String { + let mut value = String::with_capacity(current.len() + text.len()); + value.push_str(current); + value.push_str(text); + value +} + +#[cfg(test)] +mod tests { + #[test] + fn typed_value_appends_without_losing_existing_text() { + assert_eq!(super::typed_value("abc", "123"), "abc123"); + } + + #[test] + fn paste_verification_rejects_readable_no_change() { + let err = super::verify_paste_effect(Some("before"), Some("before")).unwrap_err(); + assert_eq!(err.code, agent_desktop_core::error::ErrorCode::ActionFailed); + } + + #[test] + fn paste_verification_accepts_unreadable_or_changed_values() { + assert!(super::verify_paste_effect(None, Some("after")).is_ok()); + assert!(super::verify_paste_effect(Some("before"), None).is_ok()); + assert!(super::verify_paste_effect(Some("before"), Some("after")).is_ok()); + } +} diff --git a/crates/macos/src/adapter.rs b/crates/macos/src/adapter.rs index 93b5d9a..59bf462 100644 --- a/crates/macos/src/adapter.rs +++ b/crates/macos/src/adapter.rs @@ -1,13 +1,14 @@ use agent_desktop_core::{ - action::{Action, ActionResult, DragParams, MouseEvent, WindowOp}, + action::{ActionRequest, ActionResult, DragParams, ElementState, MouseEvent, WindowOp}, adapter::{ - ImageBuffer, NativeHandle, PermissionStatus, PlatformAdapter, ScreenshotTarget, - SnapshotSurface, TreeOptions, WindowFilter, + ImageBuffer, NativeHandle, PlatformAdapter, ScreenshotTarget, SnapshotSurface, TreeOptions, + WindowFilter, }, error::AdapterError, node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo}, notification::{NotificationFilter, NotificationIdentity, NotificationInfo}, refs::RefEntry, + PermissionReport, }; use rustc_hash::FxHashSet; @@ -26,8 +27,16 @@ impl Default for MacOSAdapter { } impl PlatformAdapter for MacOSAdapter { - fn check_permissions(&self) -> PermissionStatus { - crate::system::permissions::check() + fn permission_report(&self) -> PermissionReport { + crate::system::permissions::report() + } + + fn request_permissions(&self) -> PermissionReport { + crate::system::permissions::request_report() + } + + fn unknown_accessibility_means_unsupported(&self) -> bool { + false } fn get_tree( @@ -51,16 +60,25 @@ impl PlatformAdapter for MacOSAdapter { .ok_or_else(|| AdapterError::element_not_found("No open alert or dialog"))?, }; let mut visited = FxHashSet::default(); - crate::tree::build_subtree(&el, 0, 0, opts.max_depth, &mut visited, opts.skeleton) - .ok_or_else(|| AdapterError::internal("Empty AX tree for surface")) + let context = crate::tree::TreeBuildContext::for_pid(win.pid); + crate::tree::build_subtree( + &el, + 0, + 0, + opts.max_depth, + &mut visited, + opts.skeleton, + &context, + ) + .ok_or_else(|| AdapterError::internal("Empty AX tree for surface")) } fn execute_action( &self, handle: &NativeHandle, - action: Action, + request: ActionRequest, ) -> Result { - execute_action_impl(handle, action) + execute_action_impl(handle, request) } fn resolve_element(&self, entry: &RefEntry) -> Result { @@ -79,11 +97,11 @@ impl PlatformAdapter for MacOSAdapter { } fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError> { - list_windows_impl(filter) + crate::system::window_list::list_windows_impl(filter) } fn list_apps(&self) -> Result, AdapterError> { - crate::system::app_ops::list_apps_impl() + crate::system::app_list::list_apps_impl() } fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError> { @@ -143,17 +161,30 @@ impl PlatformAdapter for MacOSAdapter { #[cfg(target_os = "macos")] { use crate::tree::AXElement; - use accessibility_sys::kAXValueAttribute; use std::mem::ManuallyDrop; let el = ManuallyDrop::new(AXElement( handle.as_raw() as accessibility_sys::AXUIElementRef )); - Ok(crate::tree::copy_string_attr(&el, kAXValueAttribute)) + Ok(crate::tree::copy_value_typed(&el)) } #[cfg(not(target_os = "macos"))] Err(AdapterError::not_supported("get_live_value")) } + fn get_live_state(&self, handle: &NativeHandle) -> Result, AdapterError> { + #[cfg(target_os = "macos")] + { + use crate::tree::AXElement; + use std::mem::ManuallyDrop; + let el = ManuallyDrop::new(AXElement( + handle.as_raw() as accessibility_sys::AXUIElementRef + )); + Ok(Some(crate::actions::post_state::read_element_state(&el))) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("get_live_state")) + } + fn get_element_bounds(&self, handle: &NativeHandle) -> Result, AdapterError> { #[cfg(target_os = "macos")] { @@ -179,6 +210,14 @@ impl PlatformAdapter for MacOSAdapter { crate::input::mouse::synthesize_mouse(event) } + fn key_event( + &self, + combo: &agent_desktop_core::action::KeyCombo, + down: bool, + ) -> Result<(), AdapterError> { + crate::input::keyboard::synthesize_key_state(combo, down) + } + fn drag(&self, params: DragParams) -> Result<(), AdapterError> { crate::input::mouse::synthesize_drag(params) } @@ -230,21 +269,29 @@ impl PlatformAdapter for MacOSAdapter { handle.as_raw() as accessibility_sys::AXUIElementRef )); let mut ancestors = FxHashSet::default(); - crate::tree::build_subtree(&el, 0, 0, opts.max_depth, &mut ancestors, opts.skeleton) - .ok_or_else(|| { - AdapterError::new( - agent_desktop_core::error::ErrorCode::ElementNotFound, - "Element no longer exists in accessibility tree", - ) - .with_suggestion("Run 'snapshot' to refresh refs, then retry.") - }) + let context = crate::tree::TreeBuildContext::empty(); + crate::tree::build_subtree( + &el, + 0, + 0, + opts.max_depth, + &mut ancestors, + opts.skeleton, + &context, + ) + .ok_or_else(|| { + AdapterError::new( + agent_desktop_core::error::ErrorCode::ElementNotFound, + "Element no longer exists in accessibility tree", + ) + .with_suggestion("Run 'snapshot' to refresh refs, then retry.") + }) } } -#[cfg(target_os = "macos")] fn execute_action_impl( handle: &NativeHandle, - action: Action, + request: ActionRequest, ) -> Result { use crate::tree::AXElement; use std::mem::ManuallyDrop; @@ -252,103 +299,5 @@ fn execute_action_impl( let el = ManuallyDrop::new(AXElement( handle.as_raw() as accessibility_sys::AXUIElementRef )); - crate::actions::perform_action(&el, &action) -} - -#[cfg(not(target_os = "macos"))] -fn execute_action_impl( - _handle: &NativeHandle, - _action: Action, -) -> Result { - Err(AdapterError::not_supported("execute_action")) -} - -pub(crate) fn list_windows_impl(filter: &WindowFilter) -> Result, AdapterError> { - #[cfg(target_os = "macos")] - { - use core_foundation::base::{CFType, TCFType}; - use core_foundation::number::CFNumber; - use core_foundation::string::CFString; - use core_foundation_sys::dictionary::CFDictionaryGetValue; - use core_graphics::display::CGDisplay; - use core_graphics::window::{ - kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowName, kCGWindowOwnerName, - kCGWindowOwnerPID, - }; - use rustc_hash::FxHasher; - use std::ffi::c_void; - use std::hash::{Hash, Hasher}; - - unsafe fn dict_string(dict: *const c_void, key: *const c_void) -> Option { - let val = CFDictionaryGetValue(dict as _, key); - if val.is_null() { - return None; - } - CFType::wrap_under_get_rule(val as _) - .downcast::() - .map(|s| s.to_string()) - } - - unsafe fn dict_i64(dict: *const c_void, key: *const c_void) -> Option { - let val = CFDictionaryGetValue(dict as _, key); - if val.is_null() { - return None; - } - CFType::wrap_under_get_rule(val as _) - .downcast::() - .and_then(|n| n.to_i64()) - } - - let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) { - Some(a) => a, - None => return Ok(vec![]), - }; - - let app_filter = filter.app.as_deref().unwrap_or("").to_lowercase(); - let mut windows = Vec::new(); - - for raw in arr.get_all_values() { - if raw.is_null() { - continue; - } - let layer = unsafe { dict_i64(raw, kCGWindowLayer as _) }.unwrap_or(99); - if layer != 0 { - continue; - } - - let app_name = match unsafe { dict_string(raw, kCGWindowOwnerName as _) } { - Some(n) if !n.is_empty() => n, - _ => continue, - }; - if !app_filter.is_empty() && !app_name.to_lowercase().contains(&app_filter) { - continue; - } - - let title = match unsafe { dict_string(raw, kCGWindowName as _) } { - Some(t) if !t.is_empty() => t, - _ => app_name.clone(), - }; - - let pid = unsafe { dict_i64(raw, kCGWindowOwnerPID as _) }.unwrap_or(0) as i32; - let mut h = FxHasher::default(); - pid.hash(&mut h); - title.hash(&mut h); - let id = format!("w-{:x}", h.finish() & 0xFFFFFF); - - windows.push(WindowInfo { - id, - title, - app: app_name, - pid, - bounds: None, - is_focused: windows.is_empty(), - }); - } - Ok(windows) - } - #[cfg(not(target_os = "macos"))] - { - let _ = filter; - Err(AdapterError::not_supported("list_windows")) - } + crate::actions::perform_action(&el, &request) } diff --git a/crates/macos/src/input/keyboard.rs b/crates/macos/src/input/keyboard.rs index afde81a..c5b28f6 100644 --- a/crates/macos/src/input/keyboard.rs +++ b/crates/macos/src/input/keyboard.rs @@ -1,11 +1,16 @@ -use agent_desktop_core::{action::KeyCombo, error::AdapterError}; +use agent_desktop_core::{ + action::KeyCombo, + error::{AdapterError, ErrorCode}, +}; #[cfg(target_os = "macos")] mod imp { use super::*; use accessibility_sys::{ - kAXErrorSuccess, AXUIElementCreateSystemWide, AXUIElementPostKeyboardEvent, + kAXErrorCannotComplete, kAXErrorSuccess, AXUIElementCreateSystemWide, + AXUIElementPostKeyboardEvent, }; + use std::time::Duration; pub fn synthesize_key(combo: &KeyCombo) -> Result<(), AdapterError> { tracing::debug!( @@ -48,8 +53,8 @@ mod imp { } } - let err_down = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, key_code, true) }; - let err_up = unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, key_code, false) }; + let err_down = post_event(sys_wide, key_code, true); + let err_up = post_event(sys_wide, key_code, false); if !combo.modifiers.is_empty() { release_modifiers(sys_wide, &combo.modifiers); @@ -70,8 +75,12 @@ mod imp { Ok(()) } - pub fn synthesize_text(text: &str) -> Result<(), AdapterError> { - tracing::debug!("keyboard: synthesize_text {} chars", text.len()); + pub fn synthesize_key_state(combo: &KeyCombo, down: bool) -> Result<(), AdapterError> { + tracing::debug!( + "keyboard: synthesize_key_state key={} down={down}", + combo.key + ); + let key_code = key_name_to_code(&combo.key)?; let sys_wide = unsafe { AXUIElementCreateSystemWide() }; if sys_wide.is_null() { return Err(AdapterError::internal( @@ -79,31 +88,113 @@ mod imp { )); } - for ch in text.chars() { - if ch == '\n' { - let return_code = 36u16; - unsafe { - AXUIElementPostKeyboardEvent(sys_wide, 0, return_code, true); - AXUIElementPostKeyboardEvent(sys_wide, 0, return_code, false); - }; - } else if let Some(code) = char_to_keycode(ch) { - let needs_shift = ch.is_ascii_uppercase() || is_shifted_char(ch); - if needs_shift { - unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, 56, true) }; + let result = (|| { + if down { + for m in &combo.modifiers { + post_checked(sys_wide, modifier_keycode(m), true, 0, 1)?; } - unsafe { - AXUIElementPostKeyboardEvent(sys_wide, 0, code, true); - AXUIElementPostKeyboardEvent(sys_wide, 0, code, false); - }; - if needs_shift { - unsafe { AXUIElementPostKeyboardEvent(sys_wide, 0, 56, false) }; + post_checked(sys_wide, key_code, true, 0, 1) + } else { + let key_result = post_checked(sys_wide, key_code, false, 0, 1); + for m in combo.modifiers.iter().rev() { + post_checked(sys_wide, modifier_keycode(m), false, 0, 1)?; } + key_result } - std::thread::sleep(std::time::Duration::from_millis(4)); - } + })(); unsafe { core_foundation::base::CFRelease(sys_wide as _) }; - Ok(()) + result + } + + pub fn synthesize_text(text: &str) -> Result<(), AdapterError> { + tracing::debug!("keyboard: synthesize_text {} chars", text.chars().count()); + let sys_wide = unsafe { AXUIElementCreateSystemWide() }; + if sys_wide.is_null() { + return Err(AdapterError::internal( + "Failed to create system-wide AX element", + )); + } + + let total = text.chars().count(); + let mut delivered = 0usize; + let result = (|| { + for ch in text.chars() { + if ch == '\n' { + post_char_key(sys_wide, 36, delivered, total)?; + } else if let Some(code) = char_to_keycode(ch) { + let needs_shift = ch.is_ascii_uppercase() || is_shifted_char(ch); + if needs_shift { + post_checked(sys_wide, 56, true, delivered, total)?; + } + let char_result = post_char_key(sys_wide, code, delivered, total); + if needs_shift { + let shift_up = post_checked(sys_wide, 56, false, delivered, total); + char_result.and(shift_up)?; + } else { + char_result?; + } + } else { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!("Cannot synthesize character '{ch}' with keyboard fallback"), + ) + .with_suggestion("Use set-value for non-ASCII text when supported.")); + } + delivered += 1; + std::thread::sleep(Duration::from_millis(4)); + } + Ok(()) + })(); + + unsafe { core_foundation::base::CFRelease(sys_wide as _) }; + result + } + + pub fn synthesize_keycode(key_code: u16, repeats: u32) -> Result<(), AdapterError> { + let sys_wide = unsafe { AXUIElementCreateSystemWide() }; + if sys_wide.is_null() { + return Err(AdapterError::internal( + "Failed to create system-wide AX element", + )); + } + let result = (|| { + for i in 0..repeats { + post_char_key(sys_wide, key_code, i as usize, repeats as usize)?; + std::thread::sleep(Duration::from_millis(10)); + } + Ok(()) + })(); + unsafe { core_foundation::base::CFRelease(sys_wide as _) }; + result + } + + pub fn synthesize_key_for_element( + el: &crate::tree::AXElement, + combo: &KeyCombo, + ) -> Result<(), AdapterError> { + let key_code = key_name_to_code(&combo.key)?; + let mut pressed = Vec::new(); + for m in &combo.modifiers { + if let Err(err) = post_checked(el.0, modifier_keycode(m), true, 0, 1) { + release_modifiers(el.0, &pressed); + return Err(err); + } + pressed.push(m.clone()); + } + let key_result = post_char_key(el.0, key_code, 0, 1); + let mut release_result = Ok(()); + for m in pressed.iter().rev() { + if let Err(err) = post_checked(el.0, modifier_keycode(m), false, 0, 1) { + if release_result.is_ok() { + release_result = Err(err); + } + } + } + if key_result.is_err() || release_result.is_err() { + release_modifiers_system_wide(&pressed); + } + key_result.and(release_result) } fn release_modifiers( @@ -116,171 +207,75 @@ mod imp { } } - fn modifier_keycode(m: &agent_desktop_core::action::Modifier) -> u16 { - use agent_desktop_core::action::Modifier; - match m { - Modifier::Cmd => 55, - Modifier::Shift => 56, - Modifier::Alt => 58, - Modifier::Ctrl => 59, + fn release_modifiers_system_wide(modifiers: &[agent_desktop_core::action::Modifier]) { + let sys_wide = unsafe { AXUIElementCreateSystemWide() }; + if sys_wide.is_null() { + return; } + release_modifiers(sys_wide, modifiers); + unsafe { core_foundation::base::CFRelease(sys_wide as _) }; + } + + fn post_char_key( + el: accessibility_sys::AXUIElementRef, + code: u16, + delivered: usize, + total: usize, + ) -> Result<(), AdapterError> { + post_checked(el, code, true, delivered, total)?; + post_checked(el, code, false, delivered, total) + } + + fn post_checked( + el: accessibility_sys::AXUIElementRef, + code: u16, + down: bool, + delivered: usize, + total: usize, + ) -> Result<(), AdapterError> { + let err = post_event(el, code, down); + if err == kAXErrorSuccess { + return Ok(()); + } + if err == kAXErrorCannotComplete { + std::thread::sleep(Duration::from_millis(10)); + let retry = post_event(el, code, down); + if retry == kAXErrorSuccess { + return Ok(()); + } + return Err(post_error(retry, delivered, total)); + } + Err(post_error(err, delivered, total)) + } + + fn post_event(el: accessibility_sys::AXUIElementRef, code: u16, down: bool) -> i32 { + unsafe { AXUIElementPostKeyboardEvent(el, 0, code, down) } + } + + fn post_error(err: i32, delivered: usize, total: usize) -> AdapterError { + AdapterError::new( + ErrorCode::ActionFailed, + format!( + "Keyboard synthesis failed after {delivered}/{total} characters delivered (err={err})" + ), + ) + .with_suggestion("Retry after refreshing the target snapshot and confirming focus.") + } + + fn modifier_keycode(m: &agent_desktop_core::action::Modifier) -> u16 { + crate::input::keyboard_map::modifier_keycode(m) } fn is_shifted_char(ch: char) -> bool { - matches!( - ch, - '!' | '@' - | '#' - | '$' - | '%' - | '^' - | '&' - | '*' - | '(' - | ')' - | '_' - | '+' - | '{' - | '}' - | '|' - | ':' - | '"' - | '<' - | '>' - | '?' - | '~' - ) + crate::input::keyboard_map::is_shifted_char(ch) } fn char_to_keycode(ch: char) -> Option { - let lower = ch.to_ascii_lowercase(); - Some(match lower { - 'a' => 0, - 'b' => 11, - 'c' => 8, - 'd' => 2, - 'e' => 14, - 'f' => 3, - 'g' => 5, - 'h' => 4, - 'i' => 34, - 'j' => 38, - 'k' => 40, - 'l' => 37, - 'm' => 46, - 'n' => 45, - 'o' => 31, - 'p' => 35, - 'q' => 12, - 'r' => 15, - 's' => 1, - 't' => 17, - 'u' => 32, - 'v' => 9, - 'w' => 13, - 'x' => 7, - 'y' => 16, - 'z' => 6, - '0' | ')' => 29, - '1' | '!' => 18, - '2' | '@' => 19, - '3' | '#' => 20, - '4' | '$' => 21, - '5' | '%' => 23, - '6' | '^' => 22, - '7' | '&' => 26, - '8' | '*' => 28, - '9' | '(' => 25, - ' ' => 49, - '-' | '_' => 27, - '=' | '+' => 24, - '[' | '{' => 33, - ']' | '}' => 30, - '\\' | '|' => 42, - ';' | ':' => 41, - '\'' | '"' => 39, - ',' | '<' => 43, - '.' | '>' => 47, - '/' | '?' => 44, - '`' | '~' => 50, - '\t' => 48, - _ => return None, - }) + crate::input::keyboard_map::char_to_keycode(ch) } fn key_name_to_code(key: &str) -> Result { - let code = match key { - "a" => 0, - "b" => 11, - "c" => 8, - "d" => 2, - "e" => 14, - "f" => 3, - "g" => 5, - "h" => 4, - "i" => 34, - "j" => 38, - "k" => 40, - "l" => 37, - "m" => 46, - "n" => 45, - "o" => 31, - "p" => 35, - "q" => 12, - "r" => 15, - "s" => 1, - "t" => 17, - "u" => 32, - "v" => 9, - "w" => 13, - "x" => 7, - "y" => 16, - "z" => 6, - "0" => 29, - "1" => 18, - "2" => 19, - "3" => 20, - "4" => 21, - "5" => 23, - "6" => 22, - "7" => 26, - "8" => 28, - "9" => 25, - "return" | "enter" => 36, - "escape" | "esc" => 53, - "tab" => 48, - "space" => 49, - "delete" | "backspace" => 51, - "forwarddelete" => 117, - "home" => 115, - "end" => 119, - "pageup" => 116, - "pagedown" => 121, - "left" => 123, - "right" => 124, - "down" => 125, - "up" => 126, - "f1" => 122, - "f2" => 120, - "f3" => 99, - "f4" => 118, - "f5" => 96, - "f6" => 97, - "f7" => 98, - "f8" => 100, - "f9" => 101, - "f10" => 109, - "f11" => 103, - "f12" => 111, - other => { - return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::InvalidArgs, - format!("Unknown key: '{other}'"), - ) - .with_suggestion("Valid keys: a-z, 0-9, return, escape, tab, space, delete, left, right, up, down, f1-f12")) - } - }; - Ok(code) + crate::input::keyboard_map::key_name_to_code(key) } } @@ -292,9 +287,27 @@ mod imp { Err(AdapterError::not_supported("synthesize_key")) } + pub fn synthesize_key_state(_combo: &KeyCombo, _down: bool) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_key_state")) + } + pub fn synthesize_text(_text: &str) -> Result<(), AdapterError> { Err(AdapterError::not_supported("synthesize_text")) } + + pub fn synthesize_keycode(_key_code: u16, _repeats: u32) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_keycode")) + } + + pub fn synthesize_key_for_element( + _el: &crate::tree::AXElement, + _combo: &KeyCombo, + ) -> Result<(), AdapterError> { + Err(AdapterError::not_supported("synthesize_key_for_element")) + } } -pub use imp::{synthesize_key, synthesize_text}; +pub use imp::{ + synthesize_key, synthesize_key_for_element, synthesize_key_state, synthesize_keycode, + synthesize_text, +}; diff --git a/crates/macos/src/input/keyboard_map.rs b/crates/macos/src/input/keyboard_map.rs new file mode 100644 index 0000000..fbebb58 --- /dev/null +++ b/crates/macos/src/input/keyboard_map.rs @@ -0,0 +1,174 @@ +use agent_desktop_core::{ + action::Modifier, + error::{AdapterError, ErrorCode}, +}; + +pub(crate) fn modifier_keycode(m: &Modifier) -> u16 { + match m { + Modifier::Cmd => 55, + Modifier::Shift => 56, + Modifier::Alt => 58, + Modifier::Ctrl => 59, + } +} + +pub(crate) fn is_shifted_char(ch: char) -> bool { + matches!( + ch, + '!' | '@' + | '#' + | '$' + | '%' + | '^' + | '&' + | '*' + | '(' + | ')' + | '_' + | '+' + | '{' + | '}' + | '|' + | ':' + | '"' + | '<' + | '>' + | '?' + | '~' + ) +} + +pub(crate) fn char_to_keycode(ch: char) -> Option { + let lower = ch.to_ascii_lowercase(); + Some(match lower { + 'a' => 0, + 'b' => 11, + 'c' => 8, + 'd' => 2, + 'e' => 14, + 'f' => 3, + 'g' => 5, + 'h' => 4, + 'i' => 34, + 'j' => 38, + 'k' => 40, + 'l' => 37, + 'm' => 46, + 'n' => 45, + 'o' => 31, + 'p' => 35, + 'q' => 12, + 'r' => 15, + 's' => 1, + 't' => 17, + 'u' => 32, + 'v' => 9, + 'w' => 13, + 'x' => 7, + 'y' => 16, + 'z' => 6, + '0' | ')' => 29, + '1' | '!' => 18, + '2' | '@' => 19, + '3' | '#' => 20, + '4' | '$' => 21, + '5' | '%' => 23, + '6' | '^' => 22, + '7' | '&' => 26, + '8' | '*' => 28, + '9' | '(' => 25, + ' ' => 49, + '-' | '_' => 27, + '=' | '+' => 24, + '[' | '{' => 33, + ']' | '}' => 30, + '\\' | '|' => 42, + ';' | ':' => 41, + '\'' | '"' => 39, + ',' | '<' => 43, + '.' | '>' => 47, + '/' | '?' => 44, + '`' | '~' => 50, + '\t' => 48, + _ => return None, + }) +} + +pub(crate) fn key_name_to_code(key: &str) -> Result { + let code = match key { + "a" => 0, + "b" => 11, + "c" => 8, + "d" => 2, + "e" => 14, + "f" => 3, + "g" => 5, + "h" => 4, + "i" => 34, + "j" => 38, + "k" => 40, + "l" => 37, + "m" => 46, + "n" => 45, + "o" => 31, + "p" => 35, + "q" => 12, + "r" => 15, + "s" => 1, + "t" => 17, + "u" => 32, + "v" => 9, + "w" => 13, + "x" => 7, + "y" => 16, + "z" => 6, + "0" => 29, + "1" => 18, + "2" => 19, + "3" => 20, + "4" => 21, + "5" => 23, + "6" => 22, + "7" => 26, + "8" => 28, + "9" => 25, + "return" | "enter" => 36, + "escape" | "esc" => 53, + "tab" => 48, + "space" => 49, + "delete" | "backspace" => 51, + "forwarddelete" => 117, + "home" => 115, + "end" => 119, + "pageup" => 116, + "pagedown" => 121, + "left" => 123, + "right" => 124, + "down" => 125, + "up" => 126, + "cmd" | "command" => 55, + "shift" => 56, + "alt" | "option" => 58, + "ctrl" | "control" => 59, + "f1" => 122, + "f2" => 120, + "f3" => 99, + "f4" => 118, + "f5" => 96, + "f6" => 97, + "f7" => 98, + "f8" => 100, + "f9" => 101, + "f10" => 109, + "f11" => 103, + "f12" => 111, + other => { + return Err(AdapterError::new( + ErrorCode::InvalidArgs, + format!("Unknown key: '{other}'"), + ) + .with_suggestion("Valid keys: a-z, 0-9, return, escape, tab, space, delete, left, right, up, down, f1-f12")) + } + }; + Ok(code) +} diff --git a/crates/macos/src/input/mod.rs b/crates/macos/src/input/mod.rs index 769ec85..3b72086 100644 --- a/crates/macos/src/input/mod.rs +++ b/crates/macos/src/input/mod.rs @@ -1,3 +1,4 @@ pub mod clipboard; pub mod keyboard; +pub(crate) mod keyboard_map; pub mod mouse; diff --git a/crates/macos/src/notifications/nc_session.rs b/crates/macos/src/notifications/nc_session.rs index c45b682..d641683 100644 --- a/crates/macos/src/notifications/nc_session.rs +++ b/crates/macos/src/notifications/nc_session.rs @@ -20,14 +20,16 @@ impl NcSession { } pub(crate) fn close(self) -> Result<(), AdapterError> { - if !self.was_already_open { - close_nc()?; - } + let close_result = if self.was_already_open { + Ok(()) + } else { + close_nc() + }; if let Some(ref app) = self.previous_app { reactivate_app(app); } std::mem::forget(self); - Ok(()) + close_result } } @@ -46,13 +48,17 @@ impl Drop for NcSession { #[cfg(target_os = "macos")] fn frontmost_app() -> Option { - let output = std::process::Command::new("/usr/bin/osascript") - .args([ - "-e", - "tell application \"System Events\" to get name of first application process whose frontmost is true", - ]) - .output() - .ok()?; + let mut command = std::process::Command::new("/usr/bin/osascript"); + command.args([ + "-e", + "tell application \"System Events\" to get name of first application process whose frontmost is true", + ]); + let output = crate::system::process::run_with_timeout( + &mut command, + "frontmost-app osascript", + std::time::Duration::from_secs(2), + ) + .ok()?; if output.status.success() { let name = String::from_utf8_lossy(&output.stdout).trim().to_string(); if name.is_empty() { @@ -72,15 +78,28 @@ fn frontmost_app() -> Option { #[cfg(target_os = "macos")] fn reactivate_app(name: &str) { - let script = format!( - "tell application \"{}\" to activate", - name.replace('"', "\\\"") + let script = format!("tell application {} to activate", applescript_string(name)); + let mut command = std::process::Command::new("/usr/bin/osascript"); + command.arg("-e").arg(script); + let _ = crate::system::process::run_with_timeout( + &mut command, + "reactivate-app osascript", + std::time::Duration::from_secs(1), ); - let _ = std::process::Command::new("/usr/bin/osascript") - .args(["-e", &script]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .output(); +} + +#[cfg(target_os = "macos")] +fn applescript_string(value: &str) -> String { + let mut escaped = String::with_capacity(value.len() + 2); + escaped.push('"'); + for ch in value.chars() { + if matches!(ch, '\\' | '"') { + escaped.push('\\'); + } + escaped.push(ch); + } + escaped.push('"'); + escaped } #[cfg(not(target_os = "macos"))] @@ -88,11 +107,14 @@ fn reactivate_app(_name: &str) {} #[cfg(target_os = "macos")] pub(super) fn nc_pid() -> Option { - let output = std::process::Command::new("/usr/bin/pgrep") - .arg("-x") - .arg("NotificationCenter") - .output() - .ok()?; + let mut command = std::process::Command::new("/usr/bin/pgrep"); + command.arg("-x").arg("NotificationCenter"); + let output = crate::system::process::run_with_timeout( + &mut command, + "pgrep NotificationCenter", + std::time::Duration::from_secs(1), + ) + .ok()?; String::from_utf8_lossy(&output.stdout) .trim() @@ -125,30 +147,14 @@ fn open_nc() -> Result<(), AdapterError> { click (first menu bar item of menu bar 1 whose description is "Clock") end tell"#; - let mut child = std::process::Command::new("/usr/bin/osascript") - .arg("-e") - .arg(script) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn() - .map_err(|e| AdapterError::internal(format!("Failed to spawn osascript: {e}")))?; - + let mut command = std::process::Command::new("/usr/bin/osascript"); + command.arg("-e").arg(script); + let _ = crate::system::process::run_with_timeout( + &mut command, + "osascript open-nc", + std::time::Duration::from_secs(2), + ); std::thread::sleep(std::time::Duration::from_millis(500)); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); - loop { - match child.try_wait() { - Ok(Some(_)) => break, - Ok(None) => { - if std::time::Instant::now() > deadline { - let _ = child.kill(); - let _ = child.wait(); - break; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - Err(_) => break, - } - } Ok(()) } @@ -171,6 +177,19 @@ fn close_nc() -> Result<(), AdapterError> { Ok(()) } +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::applescript_string; + + #[test] + fn applescript_string_escapes_quotes_and_backslashes() { + assert_eq!( + applescript_string(r#"Bad \ "Name""#), + r#""Bad \\ \"Name\"""# + ); + } +} + #[cfg(not(target_os = "macos"))] fn close_nc() -> Result<(), AdapterError> { Err(AdapterError::not_supported("close_nc")) diff --git a/crates/macos/src/system/app_list.rs b/crates/macos/src/system/app_list.rs new file mode 100644 index 0000000..c63159c --- /dev/null +++ b/crates/macos/src/system/app_list.rs @@ -0,0 +1,363 @@ +use agent_desktop_core::{ + adapter::WindowFilter, + error::AdapterError, + node::{AppInfo, WindowInfo}, +}; +#[cfg(target_os = "macos")] +use core_foundation::base::TCFType; +#[cfg(target_os = "macos")] +use std::sync::OnceLock; +#[cfg(target_os = "macos")] +use std::time::Duration; + +#[cfg(target_os = "macos")] +const PS_TIMEOUT: Duration = Duration::from_secs(2); + +pub fn list_apps_impl() -> Result, AdapterError> { + #[cfg(target_os = "macos")] + { + let mut apps = apps_from_cg_windows(); + let windows = crate::system::window_list::list_windows_impl(&WindowFilter { + focused_only: false, + app: None, + }) + .unwrap_or_default(); + merge_apps(&mut apps, apps_from_windows(windows)); + + merge_apps(&mut apps, running_apps_from_workspace()); + + merge_apps(&mut apps, running_apps_from_process_table()); + + apps.sort_by(|a, b| { + a.name + .to_ascii_lowercase() + .cmp(&b.name.to_ascii_lowercase()) + .then_with(|| a.pid.cmp(&b.pid)) + }); + Ok(apps) + } + #[cfg(not(target_os = "macos"))] + Err(AdapterError::not_supported("list_apps")) +} + +pub fn apps_from_windows(windows: Vec) -> Vec { + let mut seen_pids = std::collections::HashSet::new(); + let mut apps = Vec::new(); + + for window in windows { + if seen_pids.insert(window.pid) { + apps.push(AppInfo { + name: window.app, + pid: window.pid, + bundle_id: None, + }); + } + } + + apps +} + +#[cfg(target_os = "macos")] +pub(crate) fn pid_for_app_name(app_name: &str) -> Option { + let apps = app_sources(); + find_pid_in_apps(&apps, app_name) +} + +fn find_pid_in_apps(apps: &[AppInfo], app_name: &str) -> Option { + let wanted = app_name.to_ascii_lowercase(); + apps.iter() + .find(|app| app.name.eq_ignore_ascii_case(app_name)) + .or_else(|| { + apps.iter() + .find(|app| app.name.to_ascii_lowercase().contains(&wanted)) + }) + .map(|app| app.pid) +} + +#[cfg(target_os = "macos")] +fn app_sources() -> Vec { + let mut apps = apps_from_cg_windows(); + merge_apps(&mut apps, running_apps_from_workspace()); + merge_apps(&mut apps, running_apps_from_process_table()); + apps +} + +#[cfg(target_os = "macos")] +fn running_apps_from_workspace() -> Vec { + use core_foundation::{base::TCFType, string::CFString}; + use std::ffi::c_void; + + type Id = *mut c_void; + type Class = *mut c_void; + type Sel = *mut c_void; + + #[link(name = "AppKit", kind = "framework")] + extern "C" { + fn objc_getClass(name: *const core::ffi::c_char) -> Class; + fn sel_registerName(name: *const core::ffi::c_char) -> Sel; + fn objc_msgSend(receiver: Id, sel: Sel, ...) -> Id; + } + + unsafe fn ns_string(id: Id) -> Option { + if id.is_null() { + return None; + } + Some( + CFString::wrap_under_get_rule(id as core_foundation_sys::string::CFStringRef) + .to_string(), + ) + } + + unsafe { + if !appkit_loaded() { + return Vec::new(); + } + + let workspace_cls = objc_getClass(c"NSWorkspace".as_ptr()); + if workspace_cls.is_null() { + return Vec::new(); + } + + let shared_sel = sel_registerName(c"sharedWorkspace".as_ptr()); + let send_class: unsafe extern "C" fn(Class, Sel) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let workspace = send_class(workspace_cls, shared_sel); + if workspace.is_null() { + return Vec::new(); + } + + let running_sel = sel_registerName(c"runningApplications".as_ptr()); + let send_id: unsafe extern "C" fn(Id, Sel) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let running = send_id(workspace, running_sel); + if running.is_null() { + return Vec::new(); + } + + let count_sel = sel_registerName(c"count".as_ptr()); + let send_count: unsafe extern "C" fn(Id, Sel) -> usize = + std::mem::transmute(objc_msgSend as *const c_void); + let count = send_count(running, count_sel); + + let object_sel = sel_registerName(c"objectAtIndex:".as_ptr()); + let send_object: unsafe extern "C" fn(Id, Sel, usize) -> Id = + std::mem::transmute(objc_msgSend as *const c_void); + let policy_sel = sel_registerName(c"activationPolicy".as_ptr()); + let send_policy: unsafe extern "C" fn(Id, Sel) -> isize = + std::mem::transmute(objc_msgSend as *const c_void); + let pid_sel = sel_registerName(c"processIdentifier".as_ptr()); + let send_pid: unsafe extern "C" fn(Id, Sel) -> i32 = + std::mem::transmute(objc_msgSend as *const c_void); + let name_sel = sel_registerName(c"localizedName".as_ptr()); + let bundle_sel = sel_registerName(c"bundleIdentifier".as_ptr()); + + let mut seen_pids = std::collections::HashSet::new(); + let mut apps = Vec::new(); + for idx in 0..count { + let app = send_object(running, object_sel, idx); + if app.is_null() || send_policy(app, policy_sel) != 0 { + continue; + } + + let pid = send_pid(app, pid_sel); + if pid <= 0 || !seen_pids.insert(pid) { + continue; + } + + let name = ns_string(send_id(app, name_sel)); + if let Some(name) = name { + apps.push(AppInfo { + name, + pid, + bundle_id: ns_string(send_id(app, bundle_sel)), + }); + } + } + + apps + } +} + +#[cfg(target_os = "macos")] +fn appkit_loaded() -> bool { + use std::ffi::c_void; + + type Id = *mut c_void; + + extern "C" { + fn dlopen(filename: *const core::ffi::c_char, flag: i32) -> Id; + } + + static APPKIT_LOADED: OnceLock = OnceLock::new(); + *APPKIT_LOADED.get_or_init(|| unsafe { + !dlopen( + c"/System/Library/Frameworks/AppKit.framework/AppKit".as_ptr(), + 1, + ) + .is_null() + }) +} + +fn merge_apps(apps: &mut Vec, incoming: Vec) { + let mut seen_pids = apps + .iter() + .map(|app| app.pid) + .collect::>(); + + for app in incoming { + if seen_pids.insert(app.pid) { + apps.push(app); + } else if let Some(existing) = apps.iter_mut().find(|existing| existing.pid == app.pid) { + if existing.bundle_id.is_none() { + existing.bundle_id = app.bundle_id; + } + } + } +} + +#[cfg(target_os = "macos")] +fn apps_from_cg_windows() -> Vec { + use core_foundation::{ + array::CFArray, + base::{CFType, CFTypeRef, TCFType}, + dictionary::CFDictionary, + string::CFString, + }; + + extern "C" { + fn CGWindowListCopyWindowInfo(option: u32, window_id: u32) -> CFTypeRef; + } + + let info_ref = unsafe { CGWindowListCopyWindowInfo(17, 0) }; + if info_ref.is_null() { + return Vec::new(); + } + + let array = unsafe { CFArray::::wrap_under_create_rule(info_ref as _) }; + let mut seen_pids = std::collections::HashSet::new(); + let mut apps = Vec::new(); + + for item in array.iter() { + let dict = unsafe { + CFDictionary::::wrap_under_get_rule(item.as_concrete_TypeRef() as _) + }; + let Some(layer) = cg_int_field(&dict, "kCGWindowLayer") else { + continue; + }; + if layer != 0 { + continue; + } + let Some(pid) = cg_int_field(&dict, "kCGWindowOwnerPID").map(|p| p as i32) else { + continue; + }; + if !seen_pids.insert(pid) { + continue; + } + let Some(name) = cg_string_field(&dict, "kCGWindowOwnerName") else { + continue; + }; + apps.push(AppInfo { + name, + pid, + bundle_id: None, + }); + } + + apps +} + +#[cfg(target_os = "macos")] +fn cg_int_field( + dict: &core_foundation::dictionary::CFDictionary< + core_foundation::string::CFString, + core_foundation::base::CFType, + >, + key: &str, +) -> Option { + let key = core_foundation::string::CFString::new(key); + dict.find(&key).and_then(|value| { + let number = unsafe { + core_foundation::number::CFNumber::wrap_under_get_rule(value.as_concrete_TypeRef() as _) + }; + number.to_i64() + }) +} + +#[cfg(target_os = "macos")] +fn cg_string_field( + dict: &core_foundation::dictionary::CFDictionary< + core_foundation::string::CFString, + core_foundation::base::CFType, + >, + key: &str, +) -> Option { + let key = core_foundation::string::CFString::new(key); + dict.find(&key).map(|value| unsafe { + core_foundation::string::CFString::wrap_under_get_rule(value.as_concrete_TypeRef() as _) + .to_string() + }) +} + +#[cfg(target_os = "macos")] +fn running_apps_from_process_table() -> Vec { + let mut command = std::process::Command::new("/bin/ps"); + command.args(["-axo", "pid=,comm="]); + let output = match crate::system::process::run_with_timeout(&mut command, "ps", PS_TIMEOUT) { + Ok(output) if output.status.success() => output, + Ok(_) | Err(_) => return Vec::new(), + }; + let text = String::from_utf8_lossy(&output.stdout); + let mut seen_pids = std::collections::HashSet::new(); + let mut apps = Vec::new(); + + for line in text.lines() { + let line = line.trim_start(); + let mut fields = line.splitn(2, char::is_whitespace); + let Some(pid_text) = fields.next() else { + continue; + }; + let Some(command) = fields.next().map(str::trim) else { + continue; + }; + let Ok(pid) = pid_text.parse::() else { + continue; + }; + let Some(name) = app_name_from_command(command) else { + continue; + }; + if seen_pids.insert(pid) { + apps.push(AppInfo { + name, + pid, + bundle_id: None, + }); + } + } + + apps +} + +#[cfg(target_os = "macos")] +fn app_name_from_command(command: &str) -> Option { + if command.contains("/Contents/Frameworks/") + || command.contains("/Contents/PlugIns/") + || command.contains("/XPCServices/") + || command.contains(".appex/") + { + return None; + } + + let marker = ".app/Contents/MacOS"; + let marker_start = command.find(marker)?; + let app_path = &command[..marker_start + ".app".len()]; + let app_name = app_path.rsplit('/').next()?.strip_suffix(".app")?; + if app_name.is_empty() { + None + } else { + Some(app_name.to_string()) + } +} + +#[cfg(test)] +#[path = "app_list_tests.rs"] +mod tests; diff --git a/crates/macos/src/system/app_list_tests.rs b/crates/macos/src/system/app_list_tests.rs new file mode 100644 index 0000000..b38b6f0 --- /dev/null +++ b/crates/macos/src/system/app_list_tests.rs @@ -0,0 +1,64 @@ +use super::*; + +fn app(name: &str, pid: i32) -> AppInfo { + AppInfo { + name: name.to_string(), + pid, + bundle_id: None, + } +} + +fn app_with_bundle(name: &str, pid: i32, bundle_id: &str) -> AppInfo { + AppInfo { + name: name.to_string(), + pid, + bundle_id: Some(bundle_id.to_string()), + } +} + +#[test] +fn merge_apps_does_not_duplicate_same_pid_with_different_name() { + let mut apps = vec![app("Preview", 42)]; + + merge_apps(&mut apps, vec![app("Preview Helper", 42)]); + + assert_eq!(apps.len(), 1); + assert_eq!(apps[0].name, "Preview"); +} + +#[test] +fn merge_apps_adds_bundle_id_for_existing_pid() { + let mut apps = vec![app("Preview", 42)]; + + merge_apps( + &mut apps, + vec![app_with_bundle("Preview Helper", 42, "com.apple.Preview")], + ); + + assert_eq!(apps.len(), 1); + assert_eq!(apps[0].bundle_id.as_deref(), Some("com.apple.Preview")); +} + +#[test] +fn merge_apps_keeps_distinct_pids_with_same_name() { + let mut apps = vec![app("Terminal", 10)]; + + merge_apps(&mut apps, vec![app("Terminal", 11)]); + + assert_eq!(apps.len(), 2); + assert_eq!(apps[1].pid, 11); +} + +#[test] +fn find_pid_in_apps_prefers_exact_case_insensitive_match() { + let apps = vec![app("Finder Helper", 10), app("Finder", 11)]; + + assert_eq!(find_pid_in_apps(&apps, "finder"), Some(11)); +} + +#[test] +fn find_pid_in_apps_falls_back_to_contains_match() { + let apps = vec![app("Preview", 10), app("Docker Desktop", 11)]; + + assert_eq!(find_pid_in_apps(&apps, "Docker"), Some(11)); +} diff --git a/crates/macos/src/system/app_ops.rs b/crates/macos/src/system/app_ops.rs index aa7a4e6..fcc10ac 100644 --- a/crates/macos/src/system/app_ops.rs +++ b/crates/macos/src/system/app_ops.rs @@ -1,8 +1,4 @@ -use agent_desktop_core::{ - adapter::WindowFilter, - error::AdapterError, - node::{AppInfo, WindowInfo}, -}; +use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo}; #[cfg(target_os = "macos")] pub fn pid_from_element(el: &crate::tree::AXElement) -> Option { @@ -96,10 +92,12 @@ pub fn focus_window_impl(_win: &WindowInfo) -> Result<(), AdapterError> { #[cfg(target_os = "macos")] pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result { tracing::debug!("system: launch app={id:?} timeout={timeout_ms}ms"); - use crate::adapter::list_windows_impl; + use crate::system::window_list::list_windows_impl; use std::process::Command; use std::time::{Duration, Instant}; + const OPEN_TIMEOUT: Duration = Duration::from_secs(5); + if id.contains("..") || id.starts_with('/') { return Err(AdapterError::new( agent_desktop_core::error::ErrorCode::InvalidArgs, @@ -118,11 +116,9 @@ pub fn launch_app_impl(id: &str, timeout_ms: u64) -> Result Result Result<(), AdapterError> { tracing::debug!("system: close app={id:?} force={force}"); use std::process::Command; + use std::time::Duration; + + const QUIT_TIMEOUT: Duration = Duration::from_secs(3); + if force { - Command::new("pkill") - .arg("-x") - .arg(id) - .output() - .map_err(|e| AdapterError::internal(format!("pkill failed: {e}")))?; + let mut command = Command::new("/usr/bin/pkill"); + command.arg("-x").arg(id); + crate::system::process::run_with_timeout(&mut command, "pkill", QUIT_TIMEOUT)?; } else { let pid = crate::system::key_dispatch::find_pid_by_name(id)?; let app_ax = crate::tree::element_for_pid(pid); @@ -189,11 +187,9 @@ pub fn close_app_impl(id: &str, force: bool) -> Result<(), AdapterError> { tell theProc to quit end tell"# ); - Command::new("osascript") - .arg("-e") - .arg(script) - .output() - .map_err(|e| AdapterError::internal(format!("quit failed: {e}")))?; + let mut command = Command::new("/usr/bin/osascript"); + command.arg("-e").arg(script); + crate::system::process::run_with_timeout(&mut command, "osascript", QUIT_TIMEOUT)?; } } Ok(()) @@ -238,80 +234,3 @@ fn try_quit_via_menu_bar(app_el: &crate::tree::AXElement) -> bool { pub fn close_app_impl(_id: &str, _force: bool) -> Result<(), AdapterError> { Err(AdapterError::not_supported("close_app")) } - -pub fn list_apps_impl() -> Result, AdapterError> { - #[cfg(target_os = "macos")] - { - use core_foundation::base::{CFType, TCFType}; - use core_foundation::number::CFNumber; - use core_foundation::string::CFString; - use core_foundation_sys::dictionary::CFDictionaryGetValue; - use core_graphics::display::CGDisplay; - use core_graphics::window::{ - kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowOwnerName, kCGWindowOwnerPID, - }; - - let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) { - Some(a) => a, - None => return Ok(vec![]), - }; - - let mut seen_pids = std::collections::HashSet::new(); - let mut apps = Vec::new(); - - for raw in arr.get_all_values() { - if raw.is_null() { - continue; - } - - let layer = unsafe { - let v = CFDictionaryGetValue(raw as _, kCGWindowLayer as _); - if v.is_null() { - continue; - } - CFType::wrap_under_get_rule(v as _) - .downcast::() - .and_then(|n| n.to_i64()) - .unwrap_or(99) - }; - if layer != 0 { - continue; - } - - let pid = unsafe { - let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerPID as _); - if v.is_null() { - continue; - } - CFType::wrap_under_get_rule(v as _) - .downcast::() - .and_then(|n| n.to_i64()) - .unwrap_or(0) as i32 - }; - if !seen_pids.insert(pid) { - continue; - } - - let name = unsafe { - let v = CFDictionaryGetValue(raw as _, kCGWindowOwnerName as _); - if v.is_null() { - continue; - } - CFType::wrap_under_get_rule(v as _) - .downcast::() - .map(|s| s.to_string()) - }; - - if let Some(n) = name { - apps.push(AppInfo { - name: n, - pid, - bundle_id: None, - }); - } - } - Ok(apps) - } - #[cfg(not(target_os = "macos"))] - Err(AdapterError::not_supported("list_apps")) -} diff --git a/crates/macos/src/system/key_dispatch.rs b/crates/macos/src/system/key_dispatch.rs index 55717eb..9a83bcf 100644 --- a/crates/macos/src/system/key_dispatch.rs +++ b/crates/macos/src/system/key_dispatch.rs @@ -297,14 +297,20 @@ pub(crate) fn find_pid_by_name(app_name: &str) -> Result { focused_only: false, app: Some(app_name.to_string()), }; - let windows = crate::adapter::list_windows_impl(&filter)?; - windows.first().map(|w| w.pid).ok_or_else(|| { - AdapterError::new( - agent_desktop_core::error::ErrorCode::AppNotFound, - format!("App '{app_name}' not found"), - ) - .with_suggestion("Verify the app is running. Use 'list-apps' to see running applications.") - }) + let windows = crate::system::window_list::list_windows_impl(&filter)?; + windows + .first() + .map(|w| w.pid) + .or_else(|| crate::system::app_list::pid_for_app_name(app_name)) + .ok_or_else(|| { + AdapterError::new( + agent_desktop_core::error::ErrorCode::AppNotFound, + format!("App '{app_name}' not found"), + ) + .with_suggestion( + "Verify the app is running. Use 'list-apps' to see running applications.", + ) + }) } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/system/mod.rs b/crates/macos/src/system/mod.rs index 81662af..7eda061 100644 --- a/crates/macos/src/system/mod.rs +++ b/crates/macos/src/system/mod.rs @@ -1,6 +1,9 @@ +pub mod app_list; pub mod app_ops; pub mod key_dispatch; pub mod permissions; +pub(crate) mod process; pub mod screenshot; pub mod wait; +pub mod window_list; pub mod window_ops; diff --git a/crates/macos/src/system/permissions.rs b/crates/macos/src/system/permissions.rs index 21b6870..39e9588 100644 --- a/crates/macos/src/system/permissions.rs +++ b/crates/macos/src/system/permissions.rs @@ -1,4 +1,7 @@ -use agent_desktop_core::adapter::PermissionStatus; +use agent_desktop_core::{PermissionReport, PermissionState}; + +const ACCESSIBILITY_SUGGESTION: &str = "Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop, such as Terminal, iTerm, or Codex. If macOS lists the built binary separately, add that binary too."; +const SCREEN_RECORDING_SUGGESTION: &str = "Open System Settings > Privacy & Security > Screen Recording and add the app that launches agent-desktop, such as Terminal, iTerm, or Codex. If macOS lists the built binary separately, add that binary too."; #[cfg(target_os = "macos")] mod imp { @@ -21,6 +24,20 @@ mod imp { AXIsProcessTrustedWithOptions(dict.as_concrete_TypeRef()) } } + + #[link(name = "CoreGraphics", kind = "framework")] + extern "C" { + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; + } + + pub fn screen_recording_granted() -> bool { + unsafe { CGPreflightScreenCaptureAccess() } + } + + pub fn request_screen_recording() -> bool { + unsafe { CGRequestScreenCaptureAccess() } + } } #[cfg(not(target_os = "macos"))] @@ -31,25 +48,51 @@ mod imp { pub fn request_trust() -> bool { false } + pub fn screen_recording_granted() -> bool { + false + } + pub fn request_screen_recording() -> bool { + false + } } -pub fn check() -> PermissionStatus { +pub fn report() -> PermissionReport { + PermissionReport { + accessibility: accessibility_report_state(), + screen_recording: screen_recording_report_state(), + automation: PermissionState::NotRequired, + } +} + +pub fn request_report() -> PermissionReport { + PermissionReport { + accessibility: permission_state(imp::request_trust(), ACCESSIBILITY_SUGGESTION), + screen_recording: permission_state( + imp::request_screen_recording(), + SCREEN_RECORDING_SUGGESTION, + ), + automation: PermissionState::NotRequired, + } +} + +fn permission_state(granted: bool, suggestion: &'static str) -> PermissionState { + if granted { + PermissionState::Granted + } else { + PermissionState::Denied { + suggestion: suggestion.into(), + } + } +} + +fn accessibility_report_state() -> PermissionState { if imp::is_trusted() { - PermissionStatus::Granted - } else { - PermissionStatus::Denied { - suggestion: "Open System Settings > Privacy & Security > Accessibility and add your terminal application".into(), - } + return PermissionState::Granted; } + + permission_state(false, ACCESSIBILITY_SUGGESTION) } -pub fn check_with_request() -> PermissionStatus { - let trusted = imp::request_trust(); - if trusted { - PermissionStatus::Granted - } else { - PermissionStatus::Denied { - suggestion: "Grant accessibility permission in the system dialog that appeared".into(), - } - } +fn screen_recording_report_state() -> PermissionState { + permission_state(imp::screen_recording_granted(), SCREEN_RECORDING_SUGGESTION) } diff --git a/crates/macos/src/system/process.rs b/crates/macos/src/system/process.rs new file mode 100644 index 0000000..c6add20 --- /dev/null +++ b/crates/macos/src/system/process.rs @@ -0,0 +1,119 @@ +use agent_desktop_core::error::AdapterError; +use std::io::Read; +use std::process::{Command, ExitStatus, Output, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +pub(crate) fn run_with_timeout( + command: &mut Command, + label: &str, + timeout: Duration, +) -> Result { + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + let mut child = command + .spawn() + .map_err(|e| AdapterError::internal(format!("{label}: {e}")))?; + + let stdout_handle = child.stdout.take().map(spawn_drain); + let stderr_handle = child.stderr.take().map(spawn_drain); + let started = Instant::now(); + + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if started.elapsed() >= timeout => { + let _ = child.kill(); + let _ = child.wait(); + join_drain(stdout_handle); + join_drain(stderr_handle); + return Err(AdapterError::timeout(format!("{label} timed out")) + .with_platform_detail(format!("timeout after {timeout:?}"))); + } + Ok(None) => thread::sleep(Duration::from_millis(20)), + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + join_drain(stdout_handle); + join_drain(stderr_handle); + return Err(AdapterError::internal(format!("{label} status: {e}"))); + } + } + }; + + let stdout = join_drain(stdout_handle); + let stderr = join_drain(stderr_handle); + Ok(make_output(status, stdout, stderr)) +} + +fn spawn_drain(mut reader: R) -> thread::JoinHandle> +where + R: Read + Send + 'static, +{ + thread::spawn(move || { + let mut buf = Vec::new(); + let _ = reader.read_to_end(&mut buf); + buf + }) +} + +fn join_drain(handle: Option>>) -> Vec { + handle.and_then(|h| h.join().ok()).unwrap_or_default() +} + +fn make_output(status: ExitStatus, stdout: Vec, stderr: Vec) -> Output { + Output { + status, + stdout, + stderr, + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[test] + fn run_with_timeout_returns_output_for_successful_process() { + let mut command = Command::new("/bin/echo"); + command.arg("ok"); + + let output = run_with_timeout(&mut command, "echo", Duration::from_secs(1)).unwrap(); + + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); + } + + #[test] + fn run_with_timeout_kills_slow_process() { + let mut command = Command::new("/bin/sleep"); + command.arg("1"); + + let err = run_with_timeout(&mut command, "sleep", Duration::from_millis(10)).unwrap_err(); + + assert_eq!(err.code.as_str(), "TIMEOUT"); + } + + #[test] + fn run_with_timeout_drains_large_stdout_without_deadlock() { + let mut command = Command::new("/bin/sh"); + command.args(["-c", "yes ABCDEFGHIJ | head -c 200000"]); + + let output = run_with_timeout(&mut command, "yes-head", Duration::from_secs(5)).unwrap(); + + assert!(output.status.success()); + assert!( + output.stdout.len() >= 200_000, + "expected >=200000 bytes of drained stdout, got {}", + output.stdout.len() + ); + } + + #[test] + fn run_with_timeout_returns_internal_for_missing_binary() { + let mut command = Command::new("/nonexistent/binary-zzz"); + + let err = run_with_timeout(&mut command, "missing", Duration::from_secs(1)).unwrap_err(); + + assert_eq!(err.code.as_str(), "INTERNAL"); + } +} diff --git a/crates/macos/src/system/screenshot.rs b/crates/macos/src/system/screenshot.rs index fc4215e..da4f53b 100644 --- a/crates/macos/src/system/screenshot.rs +++ b/crates/macos/src/system/screenshot.rs @@ -1,60 +1,98 @@ -use agent_desktop_core::{adapter::ImageBuffer, adapter::ImageFormat, error::AdapterError}; +use agent_desktop_core::{ + adapter::{ImageBuffer, ImageFormat}, + error::{AdapterError, ErrorCode}, +}; #[cfg(target_os = "macos")] mod imp { use super::*; - use std::process::Command; + use std::os::unix::fs::DirBuilderExt; + use std::path::{Path, PathBuf}; + use std::process::{Command, Output}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + const SCREENCAPTURE: &str = "/usr/sbin/screencapture"; + #[cfg(not(test))] + const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(5); + #[cfg(test)] + const SCREENSHOT_TIMEOUT: Duration = Duration::from_millis(20); + static NEXT_TEMP_ID: AtomicU64 = AtomicU64::new(0); + + fn capture(window_id: Option) -> Result { + let temp = TempPng::new()?; + let mut command = Command::new(SCREENCAPTURE); + command.args(["-x", "-t", "png"]); + + if let Some(wid) = window_id { + command.args(["-l", &wid.to_string()]); + } + + command.arg(temp.path()); + let output = run_screencapture(&mut command)?; + + if !output.status.success() { + return Err(map_screencapture_error(&output)); + } + + read_png(temp.path()) + } pub fn capture_app(pid: i32) -> Result { tracing::debug!("system: screenshot app pid={pid}"); - let temp = temp_path(); - let cg_id = find_cg_window_id_for_pid(pid); - - let status = if let Some(wid) = cg_id { - Command::new("screencapture") - .args(["-x", "-t", "png", "-l"]) - .arg(wid.to_string()) - .arg(&temp) - .status() - } else { - Command::new("screencapture") - .args(["-x", "-t", "png"]) - .arg(&temp) - .status() - } - .map_err(|e| AdapterError::internal(format!("screencapture: {e}")))?; - - if !status.success() { - return Err(AdapterError::internal("screencapture exited with error")); - } - - read_png(&temp) + capture(find_cg_window_id_for_pid(pid)) } pub fn capture_screen(_idx: usize) -> Result { tracing::debug!("system: screenshot screen"); - let temp = temp_path(); - let status = Command::new("screencapture") - .args(["-x", "-t", "png"]) - .arg(&temp) - .status() - .map_err(|e| AdapterError::internal(format!("screencapture: {e}")))?; + capture(None) + } - if !status.success() { - return Err(AdapterError::internal("screencapture exited with error")); + struct TempPng { + dir: PathBuf, + path: PathBuf, + } + + impl TempPng { + fn new() -> Result { + let mut dir = std::env::temp_dir(); + dir.push(format!("agent-desktop-screenshot-{}", unique_suffix())); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&dir) + .map_err(|e| AdapterError::internal(format!("create screenshot temp dir: {e}")))?; + let path = dir.join("capture.png"); + Ok(Self { dir, path }) } - read_png(&temp) + fn path(&self) -> &Path { + &self.path + } } - fn temp_path() -> String { - format!("/tmp/agent-desktop-ss-{}.png", std::process::id()) + impl Drop for TempPng { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + let _ = std::fs::remove_dir(&self.dir); + } } - fn read_png(path: &str) -> Result { + fn unique_suffix() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let seq = NEXT_TEMP_ID.fetch_add(1, Ordering::Relaxed); + format!("{}-{nanos}-{seq}", std::process::id()) + } + + fn run_screencapture(command: &mut Command) -> Result { + crate::system::process::run_with_timeout(command, "screencapture", SCREENSHOT_TIMEOUT) + } + + fn read_png(path: &Path) -> Result { let data = std::fs::read(path) .map_err(|e| AdapterError::internal(format!("read screenshot: {e}")))?; - let _ = std::fs::remove_file(path); let (width, height) = png_dimensions(&data); Ok(ImageBuffer { data, @@ -64,6 +102,32 @@ mod imp { }) } + fn map_screencapture_error(output: &Output) -> AdapterError { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + let combined = format!("{stderr}\n{stdout}"); + let lower = combined.to_lowercase(); + if lower.contains("screen recording") + || lower.contains("not authorized") + || lower.contains("permission") + || lower.contains("denied") + { + return AdapterError::new(ErrorCode::PermDenied, "Screen Recording permission denied") + .with_suggestion( + "Open System Settings > Privacy & Security > Screen Recording and add the app that launches agent-desktop. If macOS lists the built binary separately, add that binary too.", + ) + .with_platform_detail(combined.trim()); + } + + let detail = combined.trim(); + let detail = if detail.is_empty() { + "screencapture produced no diagnostic output" + } else { + detail + }; + AdapterError::internal("screencapture exited with error").with_platform_detail(detail) + } + fn png_dimensions(data: &[u8]) -> (u32, u32) { if data.len() < 24 { return (0, 0); @@ -151,6 +215,50 @@ mod imp { best_id } + + #[cfg(test)] + mod tests { + use super::*; + use std::os::unix::process::ExitStatusExt; + use std::process::ExitStatus; + + fn output(stderr: &str) -> Output { + Output { + status: ExitStatus::from_raw(1 << 8), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } + } + + #[test] + fn maps_screen_recording_error_to_permission_denied() { + let err = map_screencapture_error(&output("Screen Recording is not authorized")); + + assert_eq!(err.code, ErrorCode::PermDenied); + assert!(err.suggestion.is_some()); + } + + #[test] + fn maps_unknown_screencapture_error_to_internal() { + let err = map_screencapture_error(&output("display server unavailable")); + + assert_eq!(err.code, ErrorCode::Internal); + assert_eq!( + err.platform_detail.as_deref(), + Some("display server unavailable") + ); + } + + #[test] + fn run_screencapture_kills_timed_out_process() { + let mut command = Command::new("/bin/sleep"); + command.arg("10"); + + let err = run_screencapture(&mut command).unwrap_err(); + + assert_eq!(err.code, ErrorCode::Timeout); + } + } } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/system/wait.rs b/crates/macos/src/system/wait.rs index a60133f..f41a804 100644 --- a/crates/macos/src/system/wait.rs +++ b/crates/macos/src/system/wait.rs @@ -6,6 +6,9 @@ mod imp { use super::*; use crate::tree::surfaces::is_menu_open; + const DEFAULT_MENU_TIMEOUT_MS: u64 = 750; + const MAX_MENU_TIMEOUT_MS: u64 = 10_000; + pub fn wait_for_menu(pid: i32, open: bool, timeout_ms: u64) -> Result<(), AdapterError> { let deadline = Instant::now() + Duration::from_millis(timeout_ms); loop { @@ -23,6 +26,15 @@ mod imp { std::thread::sleep(Duration::from_millis(50)); } } + + pub fn menu_timeout_ms() -> u64 { + std::env::var("AGENT_DESKTOP_MENU_TIMEOUT_MS") + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|ms| *ms > 0) + .map(|ms| ms.min(MAX_MENU_TIMEOUT_MS)) + .unwrap_or(DEFAULT_MENU_TIMEOUT_MS) + } } #[cfg(not(target_os = "macos"))] @@ -32,6 +44,10 @@ mod imp { pub fn wait_for_menu(_pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> { Err(AdapterError::not_supported("wait_for_menu")) } + + pub fn menu_timeout_ms() -> u64 { + 750 + } } -pub use imp::wait_for_menu; +pub use imp::{menu_timeout_ms, wait_for_menu}; diff --git a/crates/macos/src/system/window_list.rs b/crates/macos/src/system/window_list.rs new file mode 100644 index 0000000..0e36f7e --- /dev/null +++ b/crates/macos/src/system/window_list.rs @@ -0,0 +1,196 @@ +use agent_desktop_core::{adapter::WindowFilter, error::AdapterError, node::WindowInfo}; + +pub(crate) fn list_windows_impl(filter: &WindowFilter) -> Result, AdapterError> { + #[cfg(target_os = "macos")] + { + for attempt in 0..3 { + let windows = list_windows_once(filter); + if !windows.is_empty() || attempt == 2 { + return Ok(windows); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + Ok(vec![]) + } + #[cfg(not(target_os = "macos"))] + { + let _ = filter; + Err(AdapterError::not_supported("list_windows")) + } +} + +#[cfg(target_os = "macos")] +fn list_windows_once(filter: &WindowFilter) -> Vec { + #[cfg(target_os = "macos")] + { + use core_foundation::base::{CFType, TCFType}; + use core_foundation::number::CFNumber; + use core_foundation::string::CFString; + use core_foundation_sys::dictionary::CFDictionaryGetValue; + use core_graphics::display::CGDisplay; + use core_graphics::window::{ + kCGWindowLayer, kCGWindowListOptionOnScreenOnly, kCGWindowName, kCGWindowNumber, + kCGWindowOwnerName, kCGWindowOwnerPID, + }; + use std::{collections::HashMap, ffi::c_void}; + + unsafe fn dict_string(dict: *const c_void, key: *const c_void) -> Option { + let val = unsafe { CFDictionaryGetValue(dict as _, key) }; + if val.is_null() { + return None; + } + unsafe { CFType::wrap_under_get_rule(val as _) } + .downcast::() + .map(|s| s.to_string()) + } + + unsafe fn dict_i64(dict: *const c_void, key: *const c_void) -> Option { + let val = unsafe { CFDictionaryGetValue(dict as _, key) }; + if val.is_null() { + return None; + } + unsafe { CFType::wrap_under_get_rule(val as _) } + .downcast::() + .and_then(|n| n.to_i64()) + } + + let arr = match CGDisplay::window_list_info(kCGWindowListOptionOnScreenOnly, None) { + Some(a) => a, + None => return vec![], + }; + + let app_filter = filter.app.as_deref().unwrap_or("").to_lowercase(); + let mut candidates = Vec::new(); + + for raw in arr.get_all_values() { + if raw.is_null() { + continue; + } + let layer = unsafe { dict_i64(raw, kCGWindowLayer as _) }.unwrap_or(99); + if layer != 0 { + continue; + } + + let app_name = match unsafe { dict_string(raw, kCGWindowOwnerName as _) } { + Some(n) if !n.is_empty() => n, + _ => continue, + }; + if !app_filter.is_empty() && !app_name.to_lowercase().contains(&app_filter) { + continue; + } + + let title = match unsafe { dict_string(raw, kCGWindowName as _) } { + Some(t) if !t.is_empty() => t, + _ => app_name.clone(), + }; + + let pid = unsafe { dict_i64(raw, kCGWindowOwnerPID as _) }.unwrap_or(0) as i32; + let window_number = unsafe { dict_i64(raw, kCGWindowNumber as _) }.unwrap_or(0); + + candidates.push((app_name, title, pid, window_number)); + } + + let mut title_counts: HashMap<(i32, String), usize> = HashMap::new(); + for (_, title, pid, _) in &candidates { + *title_counts.entry((*pid, title.clone())).or_insert(0) += 1; + } + + let mut focus_cache: HashMap = HashMap::new(); + let mut windows = Vec::new(); + let mut focused_seen = false; + + for (app_name, title, pid, window_number) in candidates { + let title_count = title_counts + .get(&(pid, title.clone())) + .copied() + .unwrap_or(0); + let identity = focus_cache + .entry(pid) + .or_insert_with(|| focused_window_identity(pid)); + let is_focused = !focused_seen + && matches_focused_window(&title, window_number, identity, title_count); + if filter.focused_only && !is_focused { + continue; + } + focused_seen |= is_focused; + + windows.push(WindowInfo { + id: format!("w-{window_number}"), + title, + app: app_name, + pid, + bounds: None, + is_focused, + }); + } + if windows.is_empty() { + if let Some(app_name) = filter.app.as_deref() { + if let Some(window) = ax_window_for_app(app_name) { + if !filter.focused_only || window.is_focused { + windows.push(window); + } + } + } + } + windows + } +} + +#[cfg(target_os = "macos")] +fn ax_window_for_app(app_name: &str) -> Option { + let pid = crate::system::app_list::pid_for_app_name(app_name)?; + let app = crate::tree::element_for_pid(pid); + let window = crate::tree::copy_element_attr(&app, "AXFocusedWindow") + .or_else(|| crate::tree::copy_element_attr(&app, "AXMainWindow")) + .or_else(|| { + crate::tree::copy_ax_array(&app, "AXWindows") + .and_then(|windows| windows.into_iter().next()) + })?; + if crate::tree::copy_string_attr(&window, "AXRole").as_deref() != Some("AXWindow") { + return None; + } + let title = + crate::tree::copy_string_attr(&window, "AXTitle").unwrap_or_else(|| app_name.into()); + let window_number = crate::tree::copy_i64_attr(&window, "AXWindowNumber").unwrap_or(0); + let is_focused = crate::tree::copy_bool_attr(&app, "AXFrontmost") == Some(true); + Some(WindowInfo { + id: format!("w-{window_number}"), + title, + app: app_name.to_string(), + pid, + bounds: None, + is_focused, + }) +} + +#[cfg(target_os = "macos")] +type FocusedWindowIdentity = Option<(Option, Option)>; + +#[cfg(target_os = "macos")] +fn focused_window_identity(pid: i32) -> FocusedWindowIdentity { + let app = crate::tree::element_for_pid(pid); + if crate::tree::copy_bool_attr(&app, "AXFrontmost") != Some(true) { + return None; + } + let window = crate::tree::copy_element_attr(&app, "AXFocusedWindow")?; + Some(( + crate::tree::copy_string_attr(&window, "AXTitle"), + crate::tree::copy_i64_attr(&window, "AXWindowNumber"), + )) +} + +#[cfg(target_os = "macos")] +fn matches_focused_window( + title: &str, + window_number: i64, + identity: &FocusedWindowIdentity, + same_title_count: usize, +) -> bool { + let Some((focused_title, focused_number)) = identity else { + return false; + }; + if let Some(number) = focused_number { + return *number == window_number; + } + focused_title.as_deref() == Some(title) && same_title_count == 1 +} diff --git a/crates/macos/src/system/window_ops.rs b/crates/macos/src/system/window_ops.rs index 5014e18..85302cf 100644 --- a/crates/macos/src/system/window_ops.rs +++ b/crates/macos/src/system/window_ops.rs @@ -5,7 +5,7 @@ mod imp { use super::*; use accessibility_sys::{ kAXErrorSuccess, kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, - kAXValueTypeCGSize, AXUIElementPerformAction, AXUIElementSetAttributeValue, + kAXValueTypeCGSize, AXUIElementSetAttributeValue, }; use agent_desktop_core::error::ErrorCode; use core_foundation::{base::TCFType, boolean::CFBoolean, string::CFString}; @@ -28,11 +28,17 @@ mod imp { WindowOp::Resize { width, height } => set_size(&win_el, width, height), WindowOp::Move { x, y } => set_position(&win_el, x, y), WindowOp::Minimize => set_minimized(&win_el, true), - WindowOp::Maximize => press_zoom_button(&win_el), + WindowOp::Maximize => maximize_to_main_display(&win_el), WindowOp::Restore => set_minimized(&win_el, false), } } + fn maximize_to_main_display(el: &crate::tree::AXElement) -> Result<(), AdapterError> { + let bounds = core_graphics::display::CGDisplay::main().bounds(); + set_position(el, bounds.origin.x, bounds.origin.y)?; + set_size(el, bounds.size.width, bounds.size.height) + } + fn set_size(el: &crate::tree::AXElement, width: f64, height: f64) -> Result<(), AdapterError> { let size = CGSize::new(width, height); let ax_value = @@ -101,29 +107,6 @@ mod imp { } Ok(()) } - - fn press_zoom_button(el: &crate::tree::AXElement) -> Result<(), AdapterError> { - let zoom = crate::tree::copy_element_attr(el, "AXZoomButton"); - match zoom { - Some(btn) => { - let action = CFString::new("AXPress"); - let err = unsafe { AXUIElementPerformAction(btn.0, action.as_concrete_TypeRef()) }; - if err != kAXErrorSuccess { - return Err(AdapterError::new( - ErrorCode::ActionFailed, - format!("Zoom button press failed (err={err})"), - ) - .with_suggestion("Try 'resize-window' with explicit dimensions instead.")); - } - Ok(()) - } - None => Err(AdapterError::new( - ErrorCode::ActionNotSupported, - "Window has no zoom button", - ) - .with_suggestion("Window may not support maximizing")), - } - } } #[cfg(not(target_os = "macos"))] diff --git a/crates/macos/src/tree/action_list.rs b/crates/macos/src/tree/action_list.rs new file mode 100644 index 0000000..b783578 --- /dev/null +++ b/crates/macos/src/tree/action_list.rs @@ -0,0 +1,68 @@ +use super::capabilities::{copy_action_names, is_attr_settable}; +use super::AXElement; + +#[cfg(target_os = "macos")] +use accessibility_sys::{kAXFocusedAttribute, kAXValueAttribute}; + +#[cfg(target_os = "macos")] +pub(crate) fn platform_available_actions(el: &AXElement, role: &str) -> Vec { + let ax_actions = copy_action_names(el); + let has = |name: &str| ax_actions.iter().any(|a| a == name); + let mut actions = Vec::new(); + + if has("AXPress") { + push_unique(&mut actions, "Click"); + if crate::tree::roles::is_toggleable_role(role) { + push_unique(&mut actions, "Toggle"); + } + if matches!(role, "combobox" | "menuitem" | "tab") { + push_unique(&mut actions, "Select"); + } + } + if has("AXShowMenu") && role_allows_context_menu_action(role) { + push_unique(&mut actions, "RightClick"); + } + if has("AXScrollToVisible") { + push_unique(&mut actions, "ScrollTo"); + } + if has("AXIncrement") || has("AXDecrement") || is_attr_settable(el, kAXValueAttribute) { + push_unique(&mut actions, "SetValue"); + } + if is_attr_settable(el, kAXFocusedAttribute) { + push_unique(&mut actions, "SetFocus"); + } + if is_attr_settable(el, "AXExpanded") { + push_unique(&mut actions, "Expand"); + push_unique(&mut actions, "Collapse"); + } + + actions +} + +#[cfg(not(target_os = "macos"))] +pub(crate) fn platform_available_actions(_el: &AXElement, _role: &str) -> Vec { + Vec::new() +} + +fn push_unique(actions: &mut Vec, action: &str) { + if !actions.iter().any(|a| a == action) { + actions.push(action.to_string()); + } +} + +fn role_allows_context_menu_action(role: &str) -> bool { + !matches!(role, "combobox" | "menubutton") +} + +#[cfg(test)] +mod tests { + use super::role_allows_context_menu_action; + + #[test] + fn menu_opening_controls_do_not_advertise_right_click() { + assert!(!role_allows_context_menu_action("combobox")); + assert!(!role_allows_context_menu_action("menubutton")); + assert!(role_allows_context_menu_action("textfield")); + assert!(role_allows_context_menu_action("button")); + } +} diff --git a/crates/macos/src/tree/ax_element.rs b/crates/macos/src/tree/ax_element.rs new file mode 100644 index 0000000..73215e1 --- /dev/null +++ b/crates/macos/src/tree/ax_element.rs @@ -0,0 +1,41 @@ +#[cfg(target_os = "macos")] +mod imp { + use accessibility_sys::AXUIElementRef; + use core_foundation::base::{CFRelease, CFRetain, CFTypeRef}; + + pub struct AXElement(pub(crate) AXUIElementRef); + + impl Drop for AXElement { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { CFRelease(self.0 as CFTypeRef) } + } + } + } + + impl Clone for AXElement { + fn clone(&self) -> Self { + if !self.0.is_null() { + unsafe { CFRetain(self.0 as CFTypeRef) }; + } + AXElement(self.0) + } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + pub struct AXElement(pub(crate) *const std::ffi::c_void); + + impl Drop for AXElement { + fn drop(&mut self) {} + } + + impl Clone for AXElement { + fn clone(&self) -> Self { + AXElement(self.0) + } + } +} + +pub use imp::AXElement; diff --git a/crates/macos/src/tree/build_context.rs b/crates/macos/src/tree/build_context.rs new file mode 100644 index 0000000..71c3385 --- /dev/null +++ b/crates/macos/src/tree/build_context.rs @@ -0,0 +1,18 @@ +use super::AXElement; + +pub struct TreeBuildContext { + pub(crate) focused: Option, +} + +impl TreeBuildContext { + pub fn for_pid(pid: i32) -> Self { + let app = super::element_for_pid(pid); + Self { + focused: super::copy_element_attr(&app, "AXFocusedUIElement"), + } + } + + pub fn empty() -> Self { + Self { focused: None } + } +} diff --git a/crates/macos/src/tree/builder.rs b/crates/macos/src/tree/builder.rs index cdd1131..86da272 100644 --- a/crates/macos/src/tree/builder.rs +++ b/crates/macos/src/tree/builder.rs @@ -1,10 +1,15 @@ use agent_desktop_core::node::AccessibilityNode; use rustc_hash::FxHashSet; +use super::action_list::platform_available_actions; +use super::build_context::TreeBuildContext; +use super::capabilities::same_element; use super::element::{ - child_attributes, copy_ax_array, copy_string_attr, count_children, element_for_pid, - fetch_node_attrs, read_bounds, AXElement, ABSOLUTE_MAX_DEPTH, + child_attributes, copy_ax_array, copy_bool_attr, copy_string_attr, count_children, + element_for_pid, fetch_node_attrs, ABSOLUTE_MAX_DEPTH, }; +use super::element_bounds::read_bounds; +use super::AXElement; #[cfg(target_os = "macos")] use accessibility_sys::{ @@ -48,28 +53,36 @@ pub fn build_subtree( max_depth: u8, ancestors: &mut FxHashSet, skeleton: bool, + context: &TreeBuildContext, ) -> Option { if depth > max_depth { return None; } if raw_depth >= ABSOLUTE_MAX_DEPTH { - let (ax_role, title, ax_desc, value, _, _) = fetch_node_attrs(el); + let (ax_role, title, ax_desc, value, _) = fetch_node_attrs(el); let role = ax_role .as_deref() .map(crate::tree::roles::ax_role_to_str) .unwrap_or("unknown") .to_string(); + let is_secure_text = is_secure_text_role(ax_role.as_deref()); + let value = redact_secure_value(ax_role.as_deref(), value); let name = title.or(ax_desc); let child_count = count_children(el, ax_role.as_deref()); let bounds = read_bounds(el); + let mut states = Vec::new(); + if is_secure_text { + states.push("secure".into()); + } return Some(AccessibilityNode { ref_id: None, - role, + available_actions: platform_available_actions(el, &role), name, value, description: None, hint: None, - states: vec![], + states, + role, bounds, children_count: if child_count > 0 { Some(child_count) @@ -84,15 +97,21 @@ pub fn build_subtree( return None; } - let (ax_role, title, ax_desc, value, enabled, focused) = fetch_node_attrs(el); + let (ax_role, title, ax_desc, value, enabled) = fetch_node_attrs(el); - let role = ax_role - .as_deref() - .map(crate::tree::roles::ax_role_to_str) - .unwrap_or("unknown") - .to_string(); + let (role, promoted_label) = + crate::tree::roles::normalized_role_and_label(el, ax_role.as_deref()); + let is_secure_text = is_secure_text_role(ax_role.as_deref()); + let value = redact_secure_value(ax_role.as_deref(), value); + let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default(); + let is_promoted_item = promoted_label.is_some(); + let available_actions = if is_promoted_item { + vec!["Click".into(), "RightClick".into()] + } else { + platform_available_actions(el, &role) + }; - let name = title.clone().or_else(|| ax_desc.clone()); + let name = promoted_label.or_else(|| title.clone().or_else(|| ax_desc.clone())); let description = if title.is_some() { ax_desc } else { None }; let name = if name.is_none() && ax_role.as_deref() == Some("AXStaticText") { @@ -102,12 +121,25 @@ pub fn build_subtree( }; let mut states = Vec::new(); - if focused { + if context + .focused + .as_ref() + .is_some_and(|focused| same_element(el, focused)) + { states.push("focused".into()); } if !enabled { states.push("disabled".into()); } + if is_secure_text { + states.push("secure".into()); + } + if element_is_expanded(el) { + states.push("expanded".into()); + } + if super::roles::is_toggleable_role(&role) && value_is_checked(value.as_deref()) { + states.push("checked".into()); + } let bounds = read_bounds(el); @@ -117,10 +149,6 @@ pub fn build_subtree( ) && title.as_deref().is_none_or(str::is_empty) && value.as_deref().is_none_or(str::is_empty); - // Web wrappers do not consume a logical depth slot so that Electron/Chromium - // structural layers (AXGroup/AXGenericElement with no label) are transparent to - // agents. A chain of wrappers only stops at ABSOLUTE_MAX_DEPTH, not max_depth. - // This is intentional: skeleton depth tracks semantic content depth, not raw DOM depth. let child_depth = if is_web_wrapper { depth } else { depth + 1 }; let child_raw_depth = raw_depth + 1; @@ -147,28 +175,33 @@ pub fn build_subtree( description, hint: None, states, + available_actions, bounds, children_count, children: vec![], }); } - let children_raw = copy_children(el, ax_role.as_deref()).unwrap_or_default(); let name = name.or_else(|| label_from_children(&children_raw)); - let children = children_raw - .into_iter() - .filter_map(|child| { - build_subtree( - &child, - child_depth, - child_raw_depth, - max_depth, - ancestors, - skeleton, - ) - }) - .collect(); + let children = if is_promoted_item { + Vec::new() + } else { + children_raw + .into_iter() + .filter_map(|child| { + build_subtree( + &child, + child_depth, + child_raw_depth, + max_depth, + ancestors, + skeleton, + context, + ) + }) + .collect() + }; ancestors.remove(&ptr_key); @@ -180,12 +213,35 @@ pub fn build_subtree( description, hint: None, states, + available_actions, bounds, children_count: None, children, }) } +fn is_secure_text_role(ax_role: Option<&str>) -> bool { + ax_role == Some("AXSecureTextField") +} + +fn redact_secure_value(ax_role: Option<&str>, value: Option) -> Option { + if is_secure_text_role(ax_role) { + None + } else { + value + } +} + +fn element_is_expanded(el: &AXElement) -> bool { + copy_bool_attr(el, "AXExpanded") + .or_else(|| copy_bool_attr(el, "AXDisclosing")) + .unwrap_or(false) +} + +fn value_is_checked(value: Option<&str>) -> bool { + matches!(value, Some("1" | "true")) +} + pub fn label_from_children(children: &[AXElement]) -> Option { #[cfg(target_os = "macos")] { @@ -250,27 +306,11 @@ pub fn build_subtree( _max_depth: u8, _visited: &mut FxHashSet, _skeleton: bool, + _context: &TreeBuildContext, ) -> Option { None } #[cfg(test)] -mod tests { - use super::child_attributes; - - #[test] - fn test_browser_children_use_columns() { - assert_eq!( - child_attributes(Some("AXBrowser")), - ["AXColumns", "AXContents"] - ); - } - - #[test] - fn test_default_children_follow_fallback_order() { - assert_eq!( - child_attributes(Some("AXGroup")), - ["AXChildren", "AXContents", "AXChildrenInNavigationOrder"] - ); - } -} +#[path = "builder_tests.rs"] +mod tests; diff --git a/crates/macos/src/tree/builder_tests.rs b/crates/macos/src/tree/builder_tests.rs new file mode 100644 index 0000000..5951cc8 --- /dev/null +++ b/crates/macos/src/tree/builder_tests.rs @@ -0,0 +1,29 @@ +use super::{child_attributes, redact_secure_value}; + +#[test] +fn test_browser_children_use_columns() { + assert_eq!( + child_attributes(Some("AXBrowser")), + ["AXColumns", "AXContents"] + ); +} + +#[test] +fn test_default_children_follow_fallback_order() { + assert_eq!( + child_attributes(Some("AXGroup")), + ["AXChildren", "AXContents", "AXChildrenInNavigationOrder"] + ); +} + +#[test] +fn test_secure_text_value_is_redacted() { + assert_eq!( + redact_secure_value(Some("AXSecureTextField"), Some("secret".into())), + None + ); + assert_eq!( + redact_secure_value(Some("AXTextField"), Some("visible".into())), + Some("visible".into()) + ); +} diff --git a/crates/macos/src/tree/capabilities.rs b/crates/macos/src/tree/capabilities.rs new file mode 100644 index 0000000..3e20448 --- /dev/null +++ b/crates/macos/src/tree/capabilities.rs @@ -0,0 +1,62 @@ +#[cfg(target_os = "macos")] +mod imp { + use crate::tree::AXElement; + use accessibility_sys::{ + kAXErrorSuccess, AXUIElementCopyActionNames, AXUIElementIsAttributeSettable, + }; + use core_foundation::{ + array::CFArray, + base::{CFEqual, CFType, CFTypeRef, TCFType}, + string::CFString, + }; + use std::os::raw::c_uchar; + + pub fn is_attr_settable(el: &AXElement, attr: &str) -> bool { + let cf_attr = CFString::new(attr); + let mut settable: c_uchar = 0; + let err = unsafe { + AXUIElementIsAttributeSettable(el.0, cf_attr.as_concrete_TypeRef(), &mut settable) + }; + err == kAXErrorSuccess && settable != 0 + } + + pub fn copy_action_names(el: &AXElement) -> Vec { + let mut actions_ref: core_foundation_sys::array::CFArrayRef = std::ptr::null(); + let err = unsafe { AXUIElementCopyActionNames(el.0, &mut actions_ref) }; + if err != kAXErrorSuccess || actions_ref.is_null() { + return Vec::new(); + } + + let actions: CFArray = unsafe { TCFType::wrap_under_create_rule(actions_ref) }; + let mut result = Vec::with_capacity(actions.len() as usize); + for i in 0..actions.len() { + if let Some(name) = actions.get(i).and_then(|v| v.downcast::()) { + result.push(name.to_string()); + } + } + result + } + + pub fn same_element(a: &AXElement, b: &AXElement) -> bool { + unsafe { CFEqual(a.0 as CFTypeRef, b.0 as CFTypeRef) != 0 } + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::tree::AXElement; + + pub fn is_attr_settable(_el: &AXElement, _attr: &str) -> bool { + false + } + + pub fn copy_action_names(_el: &AXElement) -> Vec { + Vec::new() + } + + pub fn same_element(_a: &AXElement, _b: &AXElement) -> bool { + false + } +} + +pub use imp::{copy_action_names, is_attr_settable, same_element}; diff --git a/crates/macos/src/tree/element.rs b/crates/macos/src/tree/element.rs index c19de24..3cf07a5 100644 --- a/crates/macos/src/tree/element.rs +++ b/crates/macos/src/tree/element.rs @@ -1,10 +1,10 @@ -use agent_desktop_core::node::Rect; - pub const ABSOLUTE_MAX_DEPTH: u8 = 50; pub(crate) fn child_attributes(ax_role: Option<&str>) -> &'static [&'static str] { if ax_role == Some("AXBrowser") { &["AXColumns", "AXContents"] + } else if ax_role == Some("AXApplication") { + &["AXWindows", "AXFocusedWindow", "AXMainWindow", "AXChildren"] } else { &["AXChildren", "AXContents", "AXChildrenInNavigationOrder"] } @@ -13,9 +13,10 @@ pub(crate) fn child_attributes(ax_role: Option<&str>) -> &'static [&'static str] #[cfg(target_os = "macos")] mod imp { use super::*; + use crate::tree::ax_element::AXElement; use accessibility_sys::{ - kAXDescriptionAttribute, kAXEnabledAttribute, kAXErrorSuccess, kAXFocusedAttribute, - kAXRoleAttribute, kAXTitleAttribute, kAXValueAttribute, AXUIElementCopyAttributeValue, + kAXDescriptionAttribute, kAXEnabledAttribute, kAXErrorSuccess, kAXRoleAttribute, + kAXTitleAttribute, kAXValueAttribute, AXUIElementCopyAttributeValue, AXUIElementCopyMultipleAttributeValues, AXUIElementCreateApplication, AXUIElementRef, AXUIElementSetMessagingTimeout, }; @@ -27,25 +28,6 @@ mod imp { string::CFString, }; - pub struct AXElement(pub(crate) AXUIElementRef); - - impl Drop for AXElement { - fn drop(&mut self) { - if !self.0.is_null() { - unsafe { CFRelease(self.0 as CFTypeRef) } - } - } - } - - impl Clone for AXElement { - fn clone(&self) -> Self { - if !self.0.is_null() { - unsafe { CFRetain(self.0 as CFTypeRef) }; - } - AXElement(self.0) - } - } - pub fn element_for_pid(pid: i32) -> AXElement { let el = AXElement(unsafe { AXUIElementCreateApplication(pid) }); if !el.0.is_null() { @@ -62,7 +44,6 @@ mod imp { Option, Option, bool, - bool, ) { let attr_names = [ kAXRoleAttribute, @@ -70,7 +51,6 @@ mod imp { kAXDescriptionAttribute, kAXValueAttribute, kAXEnabledAttribute, - kAXFocusedAttribute, ]; let cf_names: Vec = attr_names.iter().map(|a| CFString::new(a)).collect(); let cf_refs: Vec<_> = cf_names.iter().map(|s| s.as_concrete_TypeRef()).collect(); @@ -92,8 +72,7 @@ mod imp { let desc = copy_string_attr(el, kAXDescriptionAttribute); let val = copy_value_typed(el); let enabled = copy_bool_attr(el, kAXEnabledAttribute).unwrap_or(true); - let focused = copy_bool_attr(el, kAXFocusedAttribute).unwrap_or(false); - return (role, title, desc, val, enabled, focused); + return (role, title, desc, val, enabled); } let arr = unsafe { CFArray::::wrap_under_create_rule(result_ref as _) }; @@ -119,7 +98,7 @@ mod imp { } None } - 4 | 5 => item + 4 => item .downcast::() .map(|b| bool::from(b).to_string()), _ => None, @@ -133,9 +112,8 @@ mod imp { let desc = get(2); let val = get(3); let enabled = get(4).map(|s| s == "true").unwrap_or(true); - let focused = get(5).map(|s| s == "true").unwrap_or(false); - (role, title, desc, val, enabled, focused) + (role, title, desc, val, enabled) } pub fn resolve_element_name(el: &AXElement) -> Option { @@ -212,6 +190,19 @@ mod imp { cf_type.downcast::().map(|b| b.into()) } + pub fn copy_i64_attr(el: &AXElement, attr: &str) -> Option { + let cf_attr = CFString::new(attr); + let mut value: CFTypeRef = std::ptr::null_mut(); + let err = unsafe { + AXUIElementCopyAttributeValue(el.0, cf_attr.as_concrete_TypeRef(), &mut value) + }; + if err != kAXErrorSuccess || value.is_null() { + return None; + } + let cf_type = unsafe { CFType::wrap_under_create_rule(value) }; + cf_type.downcast::().and_then(|n| n.to_i64()) + } + pub fn copy_ax_array(el: &AXElement, attr: &str) -> Option> { let cf_attr = CFString::new(attr); let mut value: CFTypeRef = std::ptr::null_mut(); @@ -271,118 +262,48 @@ mod imp { 0 } } - - pub fn read_bounds(el: &AXElement) -> Option { - use accessibility_sys::{ - kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, kAXValueTypeCGSize, - AXValueGetValue, - }; - use core_graphics::geometry::{CGPoint, CGSize}; - use std::ffi::c_void; - - let pos_cf = CFString::new(kAXPositionAttribute); - let mut pos_ref: CFTypeRef = std::ptr::null_mut(); - let pos_ok = unsafe { - AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref) - }; - if pos_ok != kAXErrorSuccess || pos_ref.is_null() { - return None; - } - - let mut point = CGPoint::new(0.0, 0.0); - let got_pos = unsafe { - AXValueGetValue( - pos_ref as _, - kAXValueTypeCGPoint, - &mut point as *mut _ as *mut c_void, - ) - }; - unsafe { CFRelease(pos_ref) }; - if !got_pos { - return None; - } - - let size_cf = CFString::new(kAXSizeAttribute); - let mut size_ref: CFTypeRef = std::ptr::null_mut(); - let size_ok = unsafe { - AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref) - }; - if size_ok != kAXErrorSuccess || size_ref.is_null() { - return None; - } - - let mut size = CGSize::new(0.0, 0.0); - let got_size = unsafe { - AXValueGetValue( - size_ref as _, - kAXValueTypeCGSize, - &mut size as *mut _ as *mut c_void, - ) - }; - unsafe { CFRelease(size_ref) }; - if !got_size { - return None; - } - - if !point.x.is_finite() - || !point.y.is_finite() - || !size.width.is_finite() - || !size.height.is_finite() - { - return None; - } - - Some(Rect { - x: point.x, - y: point.y, - width: size.width, - height: size.height, - }) - } } #[cfg(not(target_os = "macos"))] mod imp { - use super::*; - - pub struct AXElement(pub(crate) *const std::ffi::c_void); - - impl Drop for AXElement { - fn drop(&mut self) {} - } - impl Clone for AXElement { - fn clone(&self) -> Self { - AXElement(self.0) - } - } + use crate::tree::ax_element::AXElement; pub fn element_for_pid(_pid: i32) -> AXElement { AXElement(std::ptr::null()) } + pub fn copy_ax_array(_el: &AXElement, _attr: &str) -> Option> { None } + pub fn copy_string_attr(_el: &AXElement, _attr: &str) -> Option { None } + pub fn copy_bool_attr(_el: &AXElement, _attr: &str) -> Option { None } + + pub fn copy_i64_attr(_el: &AXElement, _attr: &str) -> Option { + None + } + pub fn copy_element_attr(_el: &AXElement, _attr: &str) -> Option { None } + pub fn count_children(_element: &AXElement, _ax_role: Option<&str>) -> u32 { 0 } - pub fn read_bounds(_el: &AXElement) -> Option { - None - } + pub fn resolve_element_name(_el: &AXElement) -> Option { None } + pub fn copy_value_typed(_el: &AXElement) -> Option { None } + pub fn fetch_node_attrs( _el: &AXElement, ) -> ( @@ -391,14 +312,12 @@ mod imp { Option, Option, bool, - bool, ) { - (None, None, None, None, true, false) + (None, None, None, None, true) } } pub use imp::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed, - count_children, element_for_pid, fetch_node_attrs, read_bounds, resolve_element_name, - AXElement, + copy_ax_array, copy_bool_attr, copy_element_attr, copy_i64_attr, copy_string_attr, + copy_value_typed, count_children, element_for_pid, fetch_node_attrs, resolve_element_name, }; diff --git a/crates/macos/src/tree/element_bounds.rs b/crates/macos/src/tree/element_bounds.rs new file mode 100644 index 0000000..f8d1f09 --- /dev/null +++ b/crates/macos/src/tree/element_bounds.rs @@ -0,0 +1,80 @@ +use agent_desktop_core::node::Rect; + +use super::AXElement; + +#[cfg(target_os = "macos")] +pub fn read_bounds(el: &AXElement) -> Option { + use accessibility_sys::{ + kAXErrorSuccess, kAXPositionAttribute, kAXSizeAttribute, kAXValueTypeCGPoint, + kAXValueTypeCGSize, AXUIElementCopyAttributeValue, AXValueGetValue, + }; + use core_foundation::{ + base::{CFRelease, CFTypeRef, TCFType}, + string::CFString, + }; + use core_graphics::geometry::{CGPoint, CGSize}; + use std::ffi::c_void; + + let pos_cf = CFString::new(kAXPositionAttribute); + let mut pos_ref: CFTypeRef = std::ptr::null_mut(); + let pos_ok = + unsafe { AXUIElementCopyAttributeValue(el.0, pos_cf.as_concrete_TypeRef(), &mut pos_ref) }; + if pos_ok != kAXErrorSuccess || pos_ref.is_null() { + return None; + } + + let mut point = CGPoint::new(0.0, 0.0); + let got_pos = unsafe { + AXValueGetValue( + pos_ref as _, + kAXValueTypeCGPoint, + &mut point as *mut _ as *mut c_void, + ) + }; + unsafe { CFRelease(pos_ref) }; + if !got_pos { + return None; + } + + let size_cf = CFString::new(kAXSizeAttribute); + let mut size_ref: CFTypeRef = std::ptr::null_mut(); + let size_ok = unsafe { + AXUIElementCopyAttributeValue(el.0, size_cf.as_concrete_TypeRef(), &mut size_ref) + }; + if size_ok != kAXErrorSuccess || size_ref.is_null() { + return None; + } + + let mut size = CGSize::new(0.0, 0.0); + let got_size = unsafe { + AXValueGetValue( + size_ref as _, + kAXValueTypeCGSize, + &mut size as *mut _ as *mut c_void, + ) + }; + unsafe { CFRelease(size_ref) }; + if !got_size { + return None; + } + + if !point.x.is_finite() + || !point.y.is_finite() + || !size.width.is_finite() + || !size.height.is_finite() + { + return None; + } + + Some(Rect { + x: point.x, + y: point.y, + width: size.width, + height: size.height, + }) +} + +#[cfg(not(target_os = "macos"))] +pub fn read_bounds(_el: &AXElement) -> Option { + None +} diff --git a/crates/macos/src/tree/mod.rs b/crates/macos/src/tree/mod.rs index d1897f3..0c43faf 100644 --- a/crates/macos/src/tree/mod.rs +++ b/crates/macos/src/tree/mod.rs @@ -1,18 +1,26 @@ +pub mod action_list; +pub mod ax_element; +pub mod build_context; pub mod builder; +pub mod capabilities; pub mod element; +pub mod element_bounds; pub mod resolve; pub mod roles; pub mod surfaces; +pub use ax_element::AXElement; +pub use build_context::TreeBuildContext; pub use builder::{build_subtree, window_element_for}; +pub use capabilities::{copy_action_names, is_attr_settable, same_element}; pub use element::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, copy_value_typed, - count_children, element_for_pid, read_bounds, resolve_element_name, AXElement, - ABSOLUTE_MAX_DEPTH, + copy_ax_array, copy_bool_attr, copy_element_attr, copy_i64_attr, copy_string_attr, + copy_value_typed, count_children, element_for_pid, resolve_element_name, ABSOLUTE_MAX_DEPTH, }; +pub use element_bounds::read_bounds; pub use resolve::{find_element_recursive, resolve_element_impl}; pub use roles::{ax_role_to_str, is_interactive_role}; pub use surfaces::{ alert_for_pid, focused_surface_for_pid, is_menu_open, list_surfaces_for_pid, - menu_element_for_pid, popover_for_pid, sheet_for_pid, + menu_element_for_pid, menubar_for_pid, popover_for_pid, sheet_for_pid, }; diff --git a/crates/macos/src/tree/resolve.rs b/crates/macos/src/tree/resolve.rs index 08565e6..885cb4a 100644 --- a/crates/macos/src/tree/resolve.rs +++ b/crates/macos/src/tree/resolve.rs @@ -1,10 +1,16 @@ -use agent_desktop_core::{adapter::NativeHandle, error::AdapterError, refs::RefEntry}; +use agent_desktop_core::{ + adapter::NativeHandle, + error::{AdapterError, ErrorCode}, + refs::RefEntry, +}; use rustc_hash::FxHashSet; +use super::builder::window_element_for; use super::element::{ - child_attributes, copy_ax_array, copy_string_attr, element_for_pid, resolve_element_name, - AXElement, + child_attributes, copy_ax_array, copy_element_attr, copy_string_attr, element_for_pid, + resolve_element_name, }; +use super::AXElement; #[cfg(target_os = "macos")] pub fn resolve_element_impl(entry: &RefEntry) -> Result { @@ -15,33 +21,30 @@ pub fn resolve_element_impl(entry: &RefEntry) -> Result Result bool { + entry.root_ref.is_none() && entry.bounds_hash.is_some() +} + +#[cfg(target_os = "macos")] +fn candidate_roots(entry: &RefEntry) -> Vec { + let root = element_for_pid(entry.pid); + let mut roots = Vec::new(); + if let Some(source_window_title) = entry.source_window_title.as_deref() { + roots.push(window_element_for(entry.pid, source_window_title)); + } + if let Some(focused) = copy_element_attr(&root, "AXFocusedWindow") { + roots.push(focused); + } + if let Some(main) = copy_element_attr(&root, "AXMainWindow") { + roots.push(main); + } + roots.extend(copy_ax_array(&root, "AXWindows").unwrap_or_default()); + if let Some(menubar) = crate::tree::menubar_for_pid(entry.pid) { + roots.push(menubar); + } + if let Some(menu) = crate::tree::menu_element_for_pid(entry.pid) { + roots.push(menu); + } + if roots.is_empty() { + roots.push(root); + } + roots +} + +#[cfg(target_os = "macos")] +fn find_entry_by_path(roots: &[AXElement], entry: &RefEntry) -> Result { + use core_foundation::base::{CFRetain, CFTypeRef}; + + if entry.path.is_empty() { + return Err(AdapterError::element_not_found("element")); + } + + for root in roots { + let Some(candidate) = element_at_path(root, &entry.path) else { + continue; + }; + if element_matches_entry(&candidate, entry) { + unsafe { CFRetain(candidate.0 as CFTypeRef) }; + return Ok(NativeHandle::from_ptr(candidate.0 as *const _)); + } + } + + Err(AdapterError::element_not_found("element")) +} + +#[cfg(target_os = "macos")] +fn element_at_path(root: &AXElement, path: &[usize]) -> Option { + let mut current = root.clone(); + for idx in path { + let ax_role = copy_string_attr(¤t, accessibility_sys::kAXRoleAttribute); + let children = resolve_children(¤t, ax_role.as_deref()); + current = children.get(*idx)?.clone(); + } + Some(current) +} + +#[cfg(target_os = "macos")] +fn find_entry_in_roots( + roots: &[AXElement], + entry: &RefEntry, + resolve_depth: u8, + deadline: std::time::Instant, +) -> Result { + for root in roots { + let mut visited = FxHashSet::default(); + if let Ok(handle) = + find_element_recursive(root, entry, 0, resolve_depth, &mut visited, deadline) + { + return Ok(handle); + } + } + Err(AdapterError::element_not_found("element")) +} + /// Depth-first search for a single element matching `entry`. /// -/// `deadline` is shared across the full resolve call (exact + relaxed passes) -/// so the total wall-clock time is bounded at five seconds. Subtrees whose -/// spatial bounds do not contain the target's centre point are pruned, which -/// makes resolution over large documents (e.g. dense spreadsheets) fast even -/// when the AX tree has thousands of nodes. +/// `deadline` is shared across retry attempts so the total wall-clock time is +/// bounded at five seconds. #[cfg(target_os = "macos")] pub fn find_element_recursive( el: &AXElement, @@ -71,11 +152,10 @@ pub fn find_element_recursive( use core_foundation::base::{CFRetain, CFTypeRef}; if std::time::Instant::now() > deadline { - return Err(AdapterError::new( - agent_desktop_core::error::ErrorCode::StaleRef, - "Element resolution timed out", - ) - .with_suggestion("Run 'snapshot' to refresh, then retry with the updated ref.")); + return Err( + AdapterError::new(ErrorCode::Timeout, "Element resolution timed out") + .with_suggestion("Retry the command, or run 'snapshot' if the UI changed."), + ); } let ptr_key = el.0 as usize; @@ -84,30 +164,12 @@ pub fn find_element_recursive( } let ax_role = copy_string_attr(el, kAXRoleAttribute); - let normalized = ax_role - .as_deref() - .map(crate::tree::roles::ax_role_to_str) - .unwrap_or("unknown"); + let normalized = crate::tree::roles::normalized_role_for_element(el, ax_role.as_deref()); - if normalized == entry.role { - let elem_name = resolve_element_name(el); - let name_match = match (&entry.name, &elem_name) { - (Some(en), Some(nn)) => en == nn, - (None, None) => true, - _ => false, - }; - let bounds_match = match entry.bounds_hash { - Some(expected) => { - let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash()); - actual.map(|h| h == expected).unwrap_or(false) - } - None => true, - }; - if name_match && bounds_match { - ancestors.remove(&ptr_key); - unsafe { CFRetain(el.0 as CFTypeRef) }; - return Ok(NativeHandle::from_ptr(el.0 as *const _)); - } + if normalized == entry.role && element_matches_entry(el, entry) { + ancestors.remove(&ptr_key); + unsafe { CFRetain(el.0 as CFTypeRef) }; + return Ok(NativeHandle::from_ptr(el.0 as *const _)); } if depth >= max_depth { @@ -115,27 +177,12 @@ pub fn find_element_recursive( return Err(AdapterError::element_not_found("element")); } - if let Some(target) = &entry.bounds { - if let Some(el_bounds) = crate::tree::read_bounds(el) { - if el_bounds.width > 0.0 && el_bounds.height > 0.0 { - let cx = target.x + target.width / 2.0; - let cy = target.y + target.height / 2.0; - if cx < el_bounds.x - || cx > el_bounds.x + el_bounds.width - || cy < el_bounds.y - || cy > el_bounds.y + el_bounds.height - { - ancestors.remove(&ptr_key); - return Err(AdapterError::element_not_found("element")); - } - } - } + if should_prune_by_bounds(el, entry, depth) { + ancestors.remove(&ptr_key); + return Err(AdapterError::element_not_found("element")); } - let children = child_attributes(ax_role.as_deref()) - .iter() - .find_map(|attr| copy_ax_array(el, attr).filter(|v| !v.is_empty())) - .unwrap_or_default(); + let children = resolve_children(el, ax_role.as_deref()); for child in &children { if let Ok(handle) = @@ -150,6 +197,123 @@ pub fn find_element_recursive( Err(AdapterError::element_not_found("element")) } +#[cfg(target_os = "macos")] +fn identity_matches( + entry: &RefEntry, + actual_name: Option<&str>, + actual_value: Option<&str>, +) -> bool { + match (entry.name.as_deref(), entry.value.as_deref()) { + (Some(expected), _) => Some(expected) == actual_name || Some(expected) == actual_value, + (None, Some(expected)) => Some(expected) == actual_value || Some(expected) == actual_name, + (None, None) => actual_name.is_none() && actual_value.is_none(), + } +} + +#[cfg(target_os = "macos")] +fn element_matches_entry(el: &AXElement, entry: &RefEntry) -> bool { + element_matches_path_entry(el, entry) && bounds_match(el, entry) +} + +#[cfg(target_os = "macos")] +fn element_matches_path_entry(el: &AXElement, entry: &RefEntry) -> bool { + let ax_role = copy_string_attr(el, accessibility_sys::kAXRoleAttribute); + let (normalized, promoted_label) = + crate::tree::roles::normalized_role_and_label(el, ax_role.as_deref()); + if normalized != entry.role { + return false; + } + + let elem_name = promoted_label.or_else(|| resolve_element_name(el)); + let elem_value = crate::tree::copy_value_typed(el); + identity_matches(entry, elem_name.as_deref(), elem_value.as_deref()) +} + +#[cfg(target_os = "macos")] +fn bounds_match(el: &AXElement, entry: &RefEntry) -> bool { + match entry.bounds_hash { + Some(expected) => { + let actual = crate::tree::read_bounds(el).map(|b| b.bounds_hash()); + actual.map(|h| h == expected).unwrap_or(false) + } + None => true, + } +} + +#[cfg(target_os = "macos")] +fn should_prune_by_bounds(el: &AXElement, entry: &RefEntry, depth: u8) -> bool { + if depth == 0 || entry.bounds.is_none() || entry.bounds_hash.is_none() { + return false; + } + let Some(candidate) = crate::tree::read_bounds(el) else { + return false; + }; + let Some(target) = entry.bounds.as_ref() else { + return false; + }; + !rects_overlap(&candidate, target) +} + +#[cfg(target_os = "macos")] +fn rects_overlap( + candidate: &agent_desktop_core::node::Rect, + target: &agent_desktop_core::node::Rect, +) -> bool { + let candidate_right = candidate.x + candidate.width; + let candidate_bottom = candidate.y + candidate.height; + let target_right = target.x + target.width; + let target_bottom = target.y + target.height; + candidate.x <= target_right + && candidate_right >= target.x + && candidate.y <= target_bottom + && candidate_bottom >= target.y +} + +#[cfg(target_os = "macos")] +fn resolve_children(el: &AXElement, ax_role: Option<&str>) -> Vec { + let mut seen = FxHashSet::default(); + let mut result = Vec::new(); + for attr in child_attributes(ax_role) { + if let Some(children) = copy_ax_array(el, attr) { + for child in children { + if seen.insert(child.0 as usize) { + result.push(child); + } + } + } + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(bounds_hash: Option, root_ref: Option<&str>) -> RefEntry { + RefEntry { + pid: 1, + role: "cell".into(), + name: Some("Investors".into()), + value: None, + states: vec![], + bounds: None, + bounds_hash, + available_actions: vec![], + source_app: None, + source_window_title: None, + root_ref: root_ref.map(String::from), + path: vec![0, 1], + } + } + + #[test] + fn path_fast_path_requires_bounds_identity() { + assert!(!can_use_path_fast_path(&entry(None, None))); + assert!(!can_use_path_fast_path(&entry(Some(42), Some("@e1")))); + assert!(can_use_path_fast_path(&entry(Some(42), None))); + } +} + #[cfg(not(target_os = "macos"))] pub fn resolve_element_impl(_entry: &RefEntry) -> Result { Err(AdapterError::not_supported("resolve_element")) diff --git a/crates/macos/src/tree/roles.rs b/crates/macos/src/tree/roles.rs index a469d26..3cff7a4 100644 --- a/crates/macos/src/tree/roles.rs +++ b/crates/macos/src/tree/roles.rs @@ -3,7 +3,7 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str { "AXApplication" => "application", "AXButton" => "button", "AXMenuButton" => "menubutton", - "AXTextField" | "AXTextArea" | "AXSearchField" => "textfield", + "AXTextField" | "AXTextArea" | "AXSearchField" | "AXSecureTextField" => "textfield", "AXCheckBox" => "checkbox", "AXSwitch" | "AXToggle" => "switch", "AXLink" => "link", @@ -52,24 +52,46 @@ pub fn ax_role_to_str(ax_role: &str) -> &'static str { } } -pub fn is_interactive_role(role: &str) -> bool { - matches!( - role, - "button" - | "menubutton" - | "textfield" - | "checkbox" - | "switch" - | "link" - | "menuitem" - | "tab" - | "slider" - | "combobox" - | "treeitem" - | "cell" - | "radiobutton" - | "incrementor" - | "colorwell" - | "dockitem" - ) +pub fn normalized_role_for_element(el: &crate::tree::AXElement, ax_role: Option<&str>) -> String { + normalized_role_and_label(el, ax_role).0 } + +pub fn normalized_role_and_label( + el: &crate::tree::AXElement, + ax_role: Option<&str>, +) -> (String, Option) { + let promoted_label = promoted_item_label(ax_role, el); + let role = if promoted_label.is_some() { + "cell" + } else { + ax_role.map(ax_role_to_str).unwrap_or("unknown") + }; + (role.to_string(), promoted_label) +} + +pub fn promoted_item_label(ax_role: Option<&str>, el: &crate::tree::AXElement) -> Option { + if ax_role != Some("AXGroup") { + return None; + } + let children = crate::tree::element::child_attributes(ax_role) + .iter() + .find_map(|attr| { + crate::tree::copy_ax_array(el, attr).filter(|children| !children.is_empty()) + }) + .unwrap_or_default(); + let has_icon = children + .iter() + .any(|child| crate::tree::copy_string_attr(child, "AXRole").as_deref() == Some("AXImage")); + if !has_icon { + return None; + } + children.iter().find_map(|child| { + if crate::tree::copy_string_attr(child, "AXRole").as_deref() == Some("AXTextField") { + crate::tree::copy_string_attr(child, "AXValue").filter(|value| !value.is_empty()) + } else { + None + } + }) +} + +pub use agent_desktop_core::roles::{is_interactive_role, is_toggleable_role}; diff --git a/crates/macos/src/tree/surfaces.rs b/crates/macos/src/tree/surfaces.rs index 50f813d..ae23949 100644 --- a/crates/macos/src/tree/surfaces.rs +++ b/crates/macos/src/tree/surfaces.rs @@ -1,6 +1,7 @@ use super::element::{ - copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid, AXElement, + copy_ax_array, copy_bool_attr, copy_element_attr, copy_string_attr, element_for_pid, }; +use super::AXElement; use agent_desktop_core::node::SurfaceInfo; #[cfg(target_os = "macos")] @@ -37,20 +38,43 @@ mod imp { fn context_menu_from_app(pid: i32) -> Option { let app = element_for_pid(pid); + if let Some(menu) = + copy_ax_array(&app, "AXMenus").and_then(|menus| menus.into_iter().find(is_menu)) + { + return Some(menu); + } if let Some(focused) = copy_element_attr(&app, "AXFocusedUIElement") { - if let Some(children) = copy_ax_array(&focused, "AXChildren") { - if let Some(menu) = children - .into_iter() - .find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu")) - { - return Some(menu); - } + if let Some(menu) = find_menu_descendant(&focused, 0) { + return Some(menu); } } let children = copy_ax_array(&app, "AXChildren")?; - children - .into_iter() - .find(|ch| copy_string_attr(ch, "AXRole").as_deref() == Some("AXMenu")) + for child in children { + if let Some(menu) = find_menu_descendant(&child, 0) { + return Some(menu); + } + } + None + } + + fn is_menu(el: &AXElement) -> bool { + copy_string_attr(el, "AXRole").as_deref() == Some("AXMenu") + && copy_bool_attr(el, "AXVisible").unwrap_or(true) + } + + fn find_menu_descendant(el: &AXElement, depth: usize) -> Option { + if depth > 8 { + return None; + } + if is_menu(el) { + return Some(el.clone()); + } + for child in copy_ax_array(el, "AXChildren").unwrap_or_default() { + if let Some(menu) = find_menu_descendant(&child, depth + 1) { + return Some(menu); + } + } + None } pub fn menu_element_for_pid(pid: i32) -> Option { diff --git a/docs/phases.md b/docs/phases.md index da916fc..fcd6efe 100644 --- a/docs/phases.md +++ b/docs/phases.md @@ -10,14 +10,15 @@ Most recent shipments against this roadmap: | Version | Date | What shipped | |---------|------------|--------------| -| v0.1.13 | 2026-04-17 | FFI cdylib on 5 platforms (aarch64/x86_64 macOS + Linux, x86_64 Windows MSVC), Sigstore build-provenance attestations, FFI review hardening (#26 — 50 commits) | +| v0.1.14 | 2026-05 | Phase 1 unified core: typed batch/CLI path, `CommandPolicy`, `PermissionReport`, snapshot-scoped `RefStore`, headless `ActionRequest`, macOS screenshot backend boundary | +| v0.1.13 | 2026-04-17 | FFI cdylib on 5 platforms (aarch64/x86_64 macOS + Linux, x86_64 Windows MSVC), Sigstore build-provenance attestations, FFI review fixes (#26 — 50 commits) | | v0.1.12 | 2026-03–04 | Progressive skeleton traversal + ref-rooted drill-down (#20) | | v0.1.11 | 2026-02–03 | Skill-install prompt fix on all success paths | | v0.1.9 | 2026-01–02 | Scalable skill architecture + ClawHub auto-publish (#14) | | v0.1.8 | 2026-01 | `--compact` flag to collapse single-child unnamed nodes | | v0.1.7 | 2025-12 | Electron / web app accessibility-tree compatibility | -- Phase 1 completion: incremental across v0.1.0 – v0.1.8 (macOS MVP, 53 commands, core engine). +- Phase 1 completion: incremental across v0.1.0 – v0.1.14 (macOS MVP, 54 commands, unified core engine). - Phase 1.5 completion: v0.1.13 (FFI cdylib on 5 platforms). - Phase 2: planned. Full scope defined in `docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md` (superseding `docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md` and `docs/plans/2026-02-25-feat-windows-adapter-phase2-plan.md`). Research-driven refinements to the brainstorm are captured in the plan's §Headless-First Invariant, §Key Technical Decisions, and §Review-Driven Refinements sections. - Phase 3+: planned. See Phase 2 plan for trait method defaults that Phase 3 backfills. @@ -28,141 +29,62 @@ Most recent shipments against this roadmap: | Phase | Name | Status | Platforms | |-------|------|--------|-----------| -| 1 | Foundation + macOS MVP | **Completed** (v0.1.0 – v0.1.12) | macOS | +| 1 | Foundation + macOS MVP | **Completed** (v0.1.0 – v0.1.14) | macOS | | 1.5 | FFI Distribution (C-ABI cdylib) | **Completed** (v0.1.13) | macOS, Windows, Linux | | 2 | Windows Adapter | Planned | macOS, Windows | | 3 | Linux Adapter | Planned | macOS, Windows, Linux | | 4 | MCP Server Mode | Planned | All | -| 5 | Production Hardening | Planned | All | +| 5 | Production Readiness | Planned | All | -Each phase is strictly additive. Core engine, CLI parser, JSON contract, error types, snapshot engine, and command registry are never modified — only new `PlatformAdapter` implementations, new transports, and new modes are added. +Future platform phases are additive against the Phase 1 contracts: typed command args, `CommandPolicy`, `PermissionReport`, snapshot-scoped refs, `ActionRequest`, and the `PlatformAdapter` boundary. Core can still gain explicitly planned additive trait methods, but Windows/Linux should not fork command semantics or duplicate transport dispatch. --- ## Command Surface Architecture (DRY invariant) -Every command in agent-desktop lives **exactly once** in `crates/core/src/commands/` and flows automatically to every transport (CLI, FFI, MCP) and every platform (macOS, Windows, Linux). No transport has per-platform code; no platform has per-transport code. The only place a new command branches is into the `PlatformAdapter` trait method calls it makes — and even those are written once per adapter in each platform crate, never per transport. +Every command in agent-desktop has one shared semantic path. CLI and batch both parse into the same typed `Commands` enum, run the same `CommandPolicy` preflight, and enter the same `dispatch()` match. Platform crates implement primitives through `PlatformAdapter`; they do not own command semantics. -### Layering +### Current Layering -``` - ┌─────────────────────────────────┐ - │ crates/core/src/commands/ │ - │ one file per command, │ - │ operates on &dyn PlatformAdapter │ - └─────────┬───────────┬───────────┘ - │ │ - ┌──────────────────┼───────────┼──────────────────┐ - │ │ │ │ - ▼ ▼ ▼ ▼ - ┌─────────────────┐ ┌───────────────┐ ┌────────────┐ ┌─────────────────┐ - │ src/cli.rs │ │ crates/ffi/ │ │ crates/mcp/│ │ (future) │ - │ (clap derive) │ │ ad_* extern C │ │ #[tool] │ │ HTTP / gRPC │ - └────────┬────────┘ └───────┬───────┘ └──────┬─────┘ └────────┬────────┘ - │ │ │ │ - └───────────────────┴─────────────────┴─────────────────┘ - │ - ▼ - ┌─────────────────────────────────┐ - │ PlatformAdapter trait │ - │ (defined in crates/core) │ - └─────────┬──────────┬────────────┘ - │ │ - ┌──────────────────┼──────────┼──────────────────┐ - ▼ ▼ ▼ ▼ - crates/macos/ crates/windows/ crates/linux/ (future platforms) -``` +| Layer | Scope | Invariant | +|-------|-------|-----------| +| `crates/core/src/commands/.rs` | Platform-agnostic command behavior and args passed to `&dyn PlatformAdapter` | One command implementation | +| `src/cli.rs` / `src/cli_args*.rs` | Clap command enum and transport args | CLI shape only, no platform behavior | +| `src/command_policy.rs` | Permissions, ref usage, side-effect classification | One policy source of truth for CLI, batch, and tests | +| `src/batch.rs` | JSON batch parser and executor | Deserializes into `Commands`; no separate command interpretation | +| `src/dispatch.rs` | Direct command match | Shared CLI/batch execution path | +| `crates/{macos,windows,linux}/` | Adapter method implementations | Same trait signatures across platforms | +| `crates/ffi/` | C ABI wrappers around adapter/core types | ABI marshaling only | -### What each layer contains +### Add a Command -| Layer | Scope | Per-command cost | Per-platform cost | -|-------|-------|------------------|-------------------| -| `crates/core/src/commands/.rs` | Args struct + `execute(args, adapter)` function, platform-agnostic | **1 file** | 0 | -| `crates/core/src/adapter.rs` (trait) | One method per distinct low-level operation (e.g. `watch_element`, `set_text_selection`) | 0 for most commands; ≤1 trait method when the command needs a new primitive | 0 | -| `crates/macos/`, `crates/windows/`, `crates/linux/` | Trait method implementations, one file per operation domain | 0 | **1 implementation per new trait method, per adapter** (real per-platform work) | -| `src/cli.rs` (clap enum) + `src/dispatch.rs` (match arm) | CLI transport | 2 lines (enum variant + match arm) | 0 | -| `crates/ffi/src//.rs` | One `ad_*` extern "C" wrapper per command | ~30 lines of marshaling | 0 | -| `crates/mcp/src/tools.rs` (registry) | One `#[tool]`-annotated wrapper per command | 1 annotation on the core `execute` function | 0 | +1. Add `crates/core/src/commands/{name}.rs`. +2. Register it in `crates/core/src/commands/mod.rs`. +3. Add the CLI args/variant in `src/cli_args*.rs` and `src/cli.rs`. +4. Add a single arm in `src/dispatch.rs`. +5. Add a `CommandPolicy` arm. +6. If needed, add one `PlatformAdapter` method with a `not_supported()` default, then implement it per adapter. -**Concretely:** adding `text select-range` in Phase 2 means: +Batch receives the command automatically once `src/batch.rs::parse_command` maps the JSON command name to that same CLI enum variant. There is no separate batch-only behavior. -1. Write `crates/core/src/commands/text_select_range.rs` with `TextSelectRangeArgs` + `execute(args, adapter)`. Calls `adapter.set_text_selection(handle, range)`. -2. Add one method `set_text_selection` to `PlatformAdapter` with a `not_supported` default. -3. Implement it in `crates/macos/src/actions/text_ops.rs` via `kAXSelectedTextRangeAttribute`. -4. Implement it in `crates/windows/src/actions/text_ops.rs` via `TextPattern.SetSelection`. -5. Implement it in `crates/linux/src/actions/text_ops.rs` via `org.a11y.atspi.Text.AddSelection` (Phase 3). -6. CLI: add 1 variant to `cli.rs` enum + 1 arm to `dispatch.rs`. -7. FFI: add `crates/ffi/src/observation/text_select_range.rs` with `ad_text_select_range(adapter, ref, start, length)` — ~30 lines of marshaling. -8. MCP: no file change. The `#[mcp_tool]` registry sees the new command automatically via the inventory submit below. +### Headless Contract -### Shared command registry pattern +Ref actions use `ActionRequest { action, policy }`. The default `InteractionPolicy` forbids focus stealing and cursor movement. macOS is the reference adapter: -Every command's `execute` function is annotated once; the registry is populated at link time via `inventory::submit!` (or `linkme`). Transports iterate the registry — they do not hand-maintain a list: +- Semantic AX steps run first. +- Physical fallbacks are explicit and policy-gated. +- Non-mouse ref commands must not silently focus apps or move the cursor. +- Expected OS denials return specific error codes such as `PERM_DENIED`, `SNAPSHOT_NOT_FOUND`, or `POLICY_DENIED`, not generic `INTERNAL`. -```rust -// crates/core/src/commands/click.rs — the SINGLE source of truth for Click - -use schemars::JsonSchema; - -#[derive(Debug, clap::Parser, JsonSchema, serde::Deserialize)] -pub struct ClickArgs { - /// The ref to click, e.g. @e5 - #[arg(value_name = "REF")] - pub ref_id: String, -} - -pub fn execute(args: ClickArgs, adapter: &dyn PlatformAdapter) -> Result { - let entry = RefMap::load()?.resolve(&args.ref_id)?; - let handle = adapter.resolve_element(&entry)?; - let result = adapter.execute_action(&handle, Action::Click)?; - Ok(json!({ "action": result.action })) -} - -// Registered once, visible to every transport. -inventory::submit! { - CommandDescriptor { - cli_name: "click", - mcp_name: "desktop_click", - description: "Press the element identified by @ref (AXPress / UIA InvokePattern / AT-SPI Action.DoAction(0))", - args_schema: || schema_for!(ClickArgs), - invoke: invoke_typed::(execute), - annotations: ToolAnnotations { - read_only: false, - destructive: false, - idempotent: false, - ..Default::default() - }, - } -} -``` - -- `crates/core/src/command_registry.rs` defines `CommandDescriptor` and the `invoke_typed` generic helper (parse JSON → Args → execute → JSON back). -- `src/dispatch.rs` walks `inventory::iter::` and matches on `cli_name` — replaces the hand-maintained match when Phase 2 lands the registry (today's dispatch is the bootstrap form). -- `crates/ffi/src/...` walks the same registry to generate `ad_` wrappers via a `build.rs` codegen step (Phase 2 work) — so adding a CLI command *also* emits the FFI entry automatically. Bespoke marshaling lives in per-type `convert/` helpers, not per-command. -- `crates/mcp/src/server.rs` walks the same registry and registers each as an `rmcp` tool with the auto-generated JSON Schema. **Zero per-platform code in the MCP crate. Zero per-command MCP code beyond the one-line `inventory::submit!`.** - -### Why this works - -- **Rust has no runtime reflection**, but the `inventory` / `linkme` crates provide compile-time plugin registration with zero runtime overhead. -- **`schemars` derives JSON Schema** from the same Args struct that clap derives CLI parsing from — one type, two bindings for free, a third for MCP. -- **`rmcp` (the official MCP Rust SDK) accepts `JsonSchema`-derived types directly** via its `#[tool]` macro, so MCP registration is a trivial pass-through. -- **`PlatformAdapter` is dyn-compatible** and passed as `&dyn PlatformAdapter` to every `execute` function; the binary crate's `build_adapter()` is the one-and-only place a concrete adapter is chosen per `#[cfg(target_os)]`. - -### What this means for the phases below - -- **Phase 2 ships the registry migration** as part of the core-extension work. After Phase 2, adding a new command is additive in exactly the places listed in the table above — never per-transport-per-platform. -- **Phase 3's Linux adapter** implements the new trait methods from Phase 2 but writes zero command files, zero CLI dispatch, zero FFI wrappers, zero MCP tool code. Pure platform impl. -- **Phase 4 (MCP)** is a new crate + stdio/HTTP transport + registry walker. It does not enumerate 53 tools by hand. The tool table in Phase 4 is a snapshot of what the registry produces, not a manual list. - -Every objective in Phases 2–5 below assumes this invariant. If any task description implies per-transport or per-platform command-surface duplication, it's a wording bug — the actual implementation follows the registry pattern. +Windows and Linux should implement the same signatures rather than copying macOS-specific fallback decisions. --- ## Phase 1 — Foundation + macOS MVP -**Status: Completed** — shipped incrementally across v0.1.0 – v0.1.12. +**Status: Completed** — shipped incrementally across v0.1.0 – v0.1.14. -Phase 1 is the load-bearing phase. It establishes every shared abstraction, every trait boundary, every output contract, every error type, the complete command trait and registry, and the full workspace structure. All subsequent phases build on top of this foundation without modifying core. +Phase 1 is the load-bearing phase. It establishes the shared command path, trait boundaries, output contract, error types, permission model, ref lifecycle, and full workspace structure. All subsequent platform phases build on top of this foundation without duplicating command semantics. ### Objectives @@ -173,9 +95,9 @@ Phase 1 is the load-bearing phase. It establishes every shared abstraction, ever | P1-O3 | Ref-based interaction | `click @e3` successfully invokes AXPress on the resolved element | | P1-O4 | Context efficiency | Typical Finder snapshot < 500 tokens (measured via tiktoken) | | P1-O5 | Typed JSON contract | Output envelope carries `version: "1.0"`. **Partial**: dedicated `schemas/` JSON-Schema files were never delivered — deferred to Phase 5 quality gates. | -| P1-O6 | Permission detection | Missing Accessibility permission prints specific macOS setup instructions | +| P1-O6 | Permission detection | Permission report covers Accessibility, Screen Recording, and Automation with recovery suggestions | | P1-O7 | Command extensibility | Adding a new command is ~4 registration points: `commands/{name}.rs` + `commands/mod.rs` + `src/cli.rs` variant + `src/dispatch.rs` match arm | -| P1-O8 | 53 working commands | All commands pass integration tests | +| P1-O8 | 54 working commands | All commands pass integration tests | | P1-O9 | CI pipeline | GitHub Actions macOS runner executes full test suite on every PR | | P1-O10 | Progressive skeleton traversal | Skeleton + drill-down workflow achieves 78%+ token savings on Electron apps | @@ -193,8 +115,9 @@ agent-desktop/ │ │ ├── lib.rs # public re-exports only │ │ ├── node.rs # AccessibilityNode, Rect, WindowInfo │ │ ├── adapter.rs # PlatformAdapter trait -│ │ ├── action.rs # Action enum, ActionResult -│ │ ├── refs.rs # RefMap, RefEntry (persisted at ~/.agent-desktop/last_refmap.json) +│ │ ├── action.rs # Action enum, ActionRequest, ActionResult +│ │ ├── refs.rs # RefMap and RefEntry +│ │ ├── refs_store.rs # Snapshot-scoped ref persistence │ │ ├── ref_alloc.rs # INTERACTIVE_ROLES, allocate_refs, is_collapsible, transform_tree │ │ ├── snapshot_ref.rs # Ref-rooted drill-down (run_from_ref) │ │ ├── snapshot.rs # SnapshotEngine (filter, allocate, serialize) @@ -209,8 +132,12 @@ agent-desktop/ │ ├── main.rs │ ├── cli.rs │ ├── cli_args.rs +│ ├── cli_args_actions.rs +│ ├── cli_args_system.rs +│ ├── command_policy.rs +│ ├── batch.rs │ ├── dispatch.rs -│ └── batch_dispatch.rs +│ └── dispatch_parse.rs └── tests/ ├── fixtures/ └── integration/ @@ -230,7 +157,7 @@ pub trait PlatformAdapter: Send + Sync { fn list_surfaces(&self, pid: i32) -> Result, AdapterError>; // Interaction - fn execute_action(&self, handle: &NativeHandle, action: Action) -> Result; + fn execute_action(&self, handle: &NativeHandle, request: ActionRequest) -> Result; fn resolve_element(&self, entry: &RefEntry) -> Result; fn release_handle(&self, handle: &NativeHandle) -> Result<(), AdapterError>; fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError>; @@ -238,9 +165,9 @@ pub trait PlatformAdapter: Send + Sync { fn press_key_for_app(&self, pid: i32, combo: KeyCombo) -> Result<(), AdapterError>; // Lifecycle + windowing - fn check_permissions(&self) -> PermissionStatus; + fn permission_report(&self) -> PermissionReport; + fn request_permissions(&self) -> PermissionReport; fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError>; - fn focused_window(&self) -> Result, AdapterError>; fn launch_app(&self, id: &str, timeout_ms: u64) -> Result; fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError>; fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError>; @@ -267,6 +194,8 @@ pub trait PlatformAdapter: Send + Sync { ### Key Supporting Types - `Action` — `#[non_exhaustive]` enum. Current variants: Click, DoubleClick, TripleClick, RightClick, SetValue(String), SetFocus, Expand, Collapse, Select(String), Toggle, Check, Uncheck, Scroll(Direction, Amount), ScrollTo, PressKey(KeyCombo), KeyDown(KeyCombo), KeyUp(KeyCombo), TypeText(String), Clear, Hover, Drag(DragParams) +- `ActionRequest` — `{ action, policy }`; default policy forbids focus stealing and cursor movement +- `PermissionReport` — `{ accessibility, screen_recording, automation }`, each `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }` - `MouseEvent`, `DragParams`, `KeyCombo` — dedicated types (not unified under an `InputEvent` enum) - `WindowOp` — Resize{w,h}, Move{x,y}, Minimize, Maximize, Restore, Close - `ScreenshotTarget` — FullScreen, Window(WindowInfo), Element(NativeHandle) @@ -286,6 +215,7 @@ crates/macos/src/ ├── tree/ │ ├── mod.rs # re-exports │ ├── element.rs # AXElement struct + attribute readers +│ ├── capabilities.rs # AX-supported actions and settable attributes │ ├── builder.rs # build_subtree, tree traversal │ ├── roles.rs # AXRole string → unified role enum mapping │ ├── resolve.rs # Element re-identification for ref resolution @@ -293,8 +223,10 @@ crates/macos/src/ ├── actions/ │ ├── mod.rs # re-exports │ ├── dispatch.rs # perform_action match arms -│ ├── activate.rs # Smart AX-first activation chain (15-step) -│ └── extras.rs # select_value, ax_scroll +│ ├── chain.rs # policy-aware AX-first activation chain +│ ├── extras.rs # select_value helpers +│ ├── scroll.rs # scroll semantics and explicit physical policy paths +│ └── type_text.rs # headless AX text insertion and physical typing ├── input/ │ ├── mod.rs # re-exports │ ├── keyboard.rs # CGEventCreateKeyboardEvent, key synthesis, text typing @@ -310,8 +242,8 @@ crates/macos/src/ ├── app_ops.rs # launch, close, focus via NSWorkspace / AppleScript ├── window_ops.rs # window resize, move, minimize, maximize, restore ├── key_dispatch.rs # app-targeted key press - ├── permissions.rs # AXIsProcessTrusted(), AXIsProcessTrustedWithOptions(prompt: true) - ├── screenshot.rs # CGWindowListCreateImage + ├── permissions.rs # PermissionReport probe/request + ├── screenshot.rs # ScreenshotBackend + secure screencapture path └── wait.rs # wait utilities ``` @@ -325,19 +257,22 @@ crates/macos/src/ - Bounds: `kAXPositionAttribute` + `kAXSizeAttribute` combined to Rect **Action execution:** -- Click: `AXUIElementPerformAction(kAXPressAction)` -- SetValue: `AXUIElementSetAttributeValue(kAXValueAttribute, value)` -- SetFocus: `AXUIElementSetAttributeValue(kAXFocusedAttribute, true)` -- Expand/Collapse: Toggle `kAXExpandedAttribute` -- Select: `AXUIElementSetAttributeValue(kAXSelectedAttribute, true)` on child +- Ref actions take `ActionRequest`, not bare `Action` +- Default policy forbids focus stealing and cursor movement +- Click/right-click/scroll chains run semantic AX steps first and return structured errors instead of silently using physical/headed paths +- Type in headless mode mutates settable AX text values; physical keyboard typing is policy-gated +- SetValue/Clear: `AXUIElementSetAttributeValue(kAXValueAttribute, value)` +- SetFocus/Press/Hover/Drag/Mouse: explicit focus/cursor/physical commands - Keyboard/Mouse: `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` via CoreGraphics - Clipboard: `NSPasteboard.generalPasteboard` read/write via Cocoa FFI -- Screenshot: `CGWindowListCreateImage` for window-specific or full-screen capture +- Screenshot: `ScreenshotBackend` boundary with secure temporary files; Screen Recording denial maps to `PERM_DENIED` **Permission detection:** -- Call `AXIsProcessTrusted()` on startup -- If false, return `PERM_DENIED` with guidance: "Open System Settings > Privacy > Accessibility and add your terminal" -- Optionally call `AXIsProcessTrustedWithOptions(prompt: true)` to trigger system dialog +- Probe once per CLI process into `PermissionReport` +- Accessibility: `AXIsProcessTrusted()` / `AXIsProcessTrustedWithOptions(prompt: true)` +- Screen Recording: platform screen-capture preflight/request path +- Automation: currently `{ "state": "not_required" }` because the shipped command set does not use Apple Events; future Apple Event paths must report a real granted/denied probe +- `status`, `permissions`, preflight, and `batch` share the same report; `permissions --request` invokes the request path **Notification management:** - Open Notification Center via AX: target the `NotificationCenter` process (bundleId: `com.apple.notificationcenterui`) @@ -369,11 +304,11 @@ Platform-agnostic, lives in `agent-desktop-core`: 4. Serialize: Omit null fields. Omit empty arrays. Omit bounds in compact mode 5. Estimate tokens: Optionally warn if exceeding threshold -RefMap persisted at `~/.agent-desktop/last_refmap.json` with `0o600` permissions, directory at `0o700`. Each snapshot replaces the refmap file entirely (atomic write via temp + rename). Action commands use optimistic re-identification: `(pid, role, name, bounds_hash)`. Return `STALE_REF` on mismatch. +Snapshot refs persist through `RefStore` under `~/.agent-desktop/snapshots/{snapshot_id}/refmap.json`, with a `latest_snapshot_id` pointer for commands that omit `--snapshot`. `~/.agent-desktop/last_refmap.json` remains only as a latest-snapshot inspection artifact. Action commands resolve through `RefStore` and use `ResolvedElement` RAII so native handles are released after ref-consuming commands. Return `STALE_REF` on live re-identification mismatch and `SNAPSHOT_NOT_FOUND` when the requested snapshot does not exist. **Progressive Skeleton Traversal:** - `--skeleton` flag clamps depth to `min(max_depth, 3)`, annotates truncated containers with `children_count` for agent discovery -- `--root ` flag starts traversal from a previously-discovered ref instead of window root +- `--root ` flag starts traversal from a previously-discovered ref instead of window root; `--snapshot ` selects the ref namespace - Named or described containers at skeleton boundary receive refs as drill-down targets (with empty `available_actions`) - Scoped invalidation: re-drilling a ref replaces only that ref's subtree refs, preserving all others - Core modules: `ref_alloc.rs` (canonical `allocate_refs` + `RefAllocConfig`), `snapshot_ref.rs` (drill-down flow that delegates allocation to `ref_alloc`) @@ -410,7 +345,7 @@ The `wait` command has been extended with notification and menu support: - `wait --notification --text "Download complete"` — Wait for a notification containing specific text - `wait --menu` / `wait --menu-closed` — Wait for context menu open/close -### Commands Shipped (53) +### Commands Shipped (54) | Category | Commands | Count | |----------|----------|-------| @@ -423,7 +358,7 @@ The `wait` command has been extended with notification and menu support: | Clipboard | `clipboard-get`, `clipboard-set`, `clipboard-clear` | 3 | | Notification (macOS) | `list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action` | 4 | | Wait | `wait` (with `--element`, `--window`, `--text`, `--menu`, `--notification` flags) | 1 | -| System | `status`, `permissions`, `version` | 3 | +| System | `status`, `permissions`, `version`, `skills` | 4 | | Batch | `batch` | 1 | > System Tray / Menu Bar Extras commands are listed under "Not Yet Implemented" above — they never shipped in Phase 1. @@ -435,7 +370,7 @@ All commands produce a response envelope. Schema files versioned in `schemas/`. Success: ```json { - "version": "1.0", + "version": "2.0", "ok": true, "command": "snapshot", "data": { @@ -450,12 +385,12 @@ Success: Error: ```json { - "version": "1.0", + "version": "2.0", "ok": false, "command": "click", "error": { "code": "STALE_REF", - "message": "RefMap is from a previous snapshot", + "message": "Element could not be resolved from the requested snapshot", "suggestion": "Run 'snapshot' to refresh, then retry with updated ref" } } @@ -465,21 +400,23 @@ Serialization rules: omit null/None fields (`skip_serializing_if`), omit empty a ### Error Taxonomy -The `ErrorCode` enum in `crates/core/src/error.rs` exposes exactly 12 variants: +The `ErrorCode` enum in `crates/core/src/error.rs` exposes these machine-readable variants: | Code | Category | Example | Recovery Suggestion | |------|----------|---------|---------------------| -| `PERM_DENIED` | Permission | Accessibility not granted | Open System Settings > Privacy > Accessibility and add your terminal | +| `PERM_DENIED` | Permission | Accessibility not granted | Open System Settings > Privacy > Accessibility and add the app that launches agent-desktop | | `ELEMENT_NOT_FOUND` | Ref | @e12 could not be resolved | Run 'snapshot' to refresh, then retry with updated ref | | `APP_NOT_FOUND` | Application | --app 'Photoshop' not running | Launch the application first | | `ACTION_FAILED` | Execution | AXPress returned error on disabled button | Element may be disabled. Check states before acting | | `ACTION_NOT_SUPPORTED` | Execution | Expand on a button | This element does not support the requested action | -| `STALE_REF` | Ref | RefMap is from a previous snapshot | Run 'snapshot' (or `snapshot --skeleton`) to refresh | +| `STALE_REF` | Ref | Element could not be re-identified from the requested snapshot | Run 'snapshot' (or `snapshot --skeleton`) to refresh | | `WINDOW_NOT_FOUND` | Window | --window w-999 does not exist | Run 'list-windows' to see available windows | | `PLATFORM_NOT_SUPPORTED` | Platform | Windows/Linux adapter not yet shipped | This platform ships in Phase 2/3 | | `TIMEOUT` | Wait / Traversal | wait --element exceeded timeout | Increase --timeout or check app state | | `INVALID_ARGS` | Input | Bad CLI argument or unknown ref format | Fix the argument per CLI help | | `NOTIFICATION_NOT_FOUND` | Notification | Notification ID not found / NC reordered | Run 'list-notifications' to see current notifications | +| `SNAPSHOT_NOT_FOUND` | Ref | Requested snapshot ID is missing | Run 'snapshot' again and use the returned snapshot_id | +| `POLICY_DENIED` | Action policy | Physical fallback blocked by headless policy | Use an explicit focus/cursor/mouse command if physical interaction is intended | | `INTERNAL` | Internal | Unexpected error or caught panic | Re-run with verbose logging | Exit codes: `0` success, `1` structured error (JSON on stdout), `2` argument/parse error. @@ -668,7 +605,7 @@ These items are tracked in the Phase 2 plan (`docs/plans/2026-04-18-001-feat-pha 2. **Skeleton traversal is platform-agnostic.** The novel progressive skeleton pattern (depth-3 clamp + `children_count` annotation + drill-down via `--root @ref` + scoped invalidation via `RefMap::remove_by_root_ref`) lives entirely in `crates/core/src/snapshot_ref.rs`. Windows adapter contributes ~50 LOC glue: `ControlViewWalker` (NOT `RawViewWalker` or `ContentViewWalker`) + `FindAll(TreeScope_Children, TrueCondition)` for `children_count` + fresh `UICacheRequest` per drill-down. 3. **Asymmetric event threading.** `watch_element` uses main-thread `AXObserver` on macOS (research-confirmed: Apple DTS says all AX is main-thread-only; AXSwift / Hammerspoon / Phoenix all do this); worker-thread MTA `IUIAutomation` event handler on Windows (Microsoft 2025 threading doc: UIA supports cross-thread event delivery). 4. **No `inventory` / `linkme` command registry.** Research confirmed neither survives link-GC reliably across ld64, ld-prime, GNU ld, lld, MSVC for cdylib consumers. Phase 2 uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe, zero linker magic. The "one command per file" CLAUDE.md rule becomes the codegen contract. -5. **v0.1.14 prep release.** Ships `#[non_exhaustive]` on `ErrorCode` + `ad_abi_version()` + `ad_init(expected_major)` + `AD_RESULT_UNKNOWN` sentinel before any ABI break, giving consumers time to adapt. v0.2.0 then ships only unavoidable breaks (`PermissionStatus` tri-state, MSRV 1.82, new variants). +5. **FFI compatibility gates.** v0.1.14 adds explicit FFI result codes for snapshot-not-found and policy-denied paths. Phase 2 still owns `ad_abi_version()`, `ad_init(expected_major)`, and any broader ABI-version handshake before new cross-platform ABI surface ships. 6. **`DeliverFiles` replaces `FileDrop`.** Headless-first forbids `NSDraggingSession` on macOS; the new action uses a 4-tier fallback (URL scheme → `NSWorkspace.open` with `activates: false` → pasteboard + `Cmd-V` → AppleScript). Windows keeps `IDataObject + DoDragDrop` (OLE drag is headless on Windows). ### Windows Engineering Invariants (from Phase 2 plan Unit 3) @@ -692,7 +629,7 @@ Phase 2 brings agent-desktop to Windows. It is also the phase that closes the cr Core engine, CLI parser, JSON contract invariants, and command-registration pattern are preserved. What Phase 2 legitimately changes: `AccessibilityNode` field set, `Action` enum variants, `ErrorCode` variants, `PlatformAdapter` trait size. Every change is additive (`#[non_exhaustive]` already guards the enums) and every macOS backfill lands atomically with the Windows implementation so the two platforms never drift. -Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, every new command added in Phase 2 (`watch`, `text select-range`, `text get-selection`, `text insert-at-caret`, etc.) lives in **exactly one file** under `crates/core/src/commands/` and auto-registers into the CLI, FFI, and MCP transports via `inventory::submit!`. The per-platform work is the three `PlatformAdapter` method implementations (one each in `crates/macos/`, `crates/windows/`, `crates/linux/`) — nothing repeats across transports. +Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant, every new command added in Phase 2 (`watch`, `text select-range`, `text get-selection`, `text insert-at-caret`, etc.) lives in **exactly one file** under `crates/core/src/commands/` and is wired through the shared typed command path. If Phase 2 adds codegen, it uses deterministic `build.rs` filesystem enumeration, not linker registries. The per-platform work is the three `PlatformAdapter` method implementations (one each in `crates/macos/`, `crates/windows/`, `crates/linux/`) — nothing repeats across transports. P2-O16 (FFI parity expansion) also migrates the FFI wrappers from hand-written to codegen: a `build.rs` step in `crates/ffi/` walks the registry and emits one `ad_` extern "C" function per `CommandDescriptor`, using the per-type marshaling helpers in `crates/ffi/src/convert/`. After this migration, the FFI crate holds marshaling primitives, not command wrappers. The `crates/mcp/` crate follows the same walk-the-registry pattern with `rmcp`'s `#[tool]` shape — so Phase 4 can ship its MCP server without hand-maintaining the tool list. @@ -723,7 +660,7 @@ Cross-platform core extensions (new, landed alongside Windows): | P2-O14 | Toolbar and missing surfaces | Both platforms add `SnapshotSurface::Toolbar`. macOS additionally adds `Spotlight` (pid of `/System/Library/CoreServices/Spotlight.app`), `Dock` (pid of `/System/Library/CoreServices/Dock.app`), and `MenuBarExtras` (enumerates `SystemUIServer`, `ControlCenter`, and per-app `AXExtrasMenuBar`). Windows adds `SystemTray` (as structured surface, not just tray commands) | | P2-O15 | Electron / WebView2 deep-tree toggles | macOS: `build_subtree` writes `AXEnhancedUserInterface = YES` on app root for known Electron bundle IDs (VS Code, Cursor, Slack post-Sept-2024, Teams, Discord, Figma Desktop, Notion). Windows: detect Edge WebView2 via UIA `ClassName = "Chrome_WidgetWin_1"` and the equivalent flag; apply same web-wrapper depth-skip. Both: new `--force-electron-a11y` CLI override | | P2-O16 | FFI registry migration + parity expansion | Migrate `crates/ffi/` from hand-written `ad_*` wrappers to a `build.rs` codegen step that walks the compile-time `CommandDescriptor` registry and emits one wrapper per command. After this, adding a CLI command automatically produces the FFI entry (plus its JSON Schema via `schemars` and its MCP tool in Phase 4). Marshaling helpers stay in `crates/ffi/src/convert/` — these are per-type, not per-command. In the same migration: backfill `ad_snapshot` (full refmap pipeline), `ad_execute_by_ref(adapter, "@e5", action, out)`, `ad_wait(…)`, `ad_version`, `ad_abi_version() -> u32` with `AD_ABI_VERSION_MAJOR` cbindgen `[defines]` export, `ad_status`, `ad_set_log_callback(fn(level, msg))` installing a `tracing_subscriber` layer so dlopen consumers see debug output | -| P2-O17 | Screen Recording / Automation permission detection (macOS backfill) | `check_permissions()` returns a richer `PermissionStatus` with a tri-state for AX, Screen Recording (`CGPreflightScreenCaptureAccess` / `CGRequestScreenCaptureAccess`), and Automation (`AEDeterminePermissionToAutomateTarget`). Failures surface as `PermDenied` / `AutomationPermissionDenied` with concrete System Settings paths | +| P2-O17 | Screen Recording / Automation permission detection | macOS Phase 1 already exposes `PermissionReport { accessibility, screen_recording, automation }`. Phase 2 decides whether a distinct `AutomationPermissionDenied` code is still needed once Apple Event automation paths exist | ### Cross-Platform Trait Extensions @@ -761,7 +698,7 @@ New supporting types (land in `crates/core/src/`): - `TextRange` — `{ start: u32, length: u32 }` (UTF-16 code units to match both AX CFRange and UIA TextRange conventions) - `TextSelection` — `{ range: TextRange, caret_offset: u32, lines_in_view: Vec }` - `ScreenshotBackend` — `Modern` (ScreenCaptureKit / Windows.Graphics.Capture / PipeWire) or `Legacy` (preserves Phase 1 subprocess path as fallback for restricted environments) -- `PermissionStatus` extends to `{ accessibility: TriState, screen_recording: TriState, automation: TriState }` where `TriState = Granted | Denied { suggestion } | NotDetermined` +- `PermissionReport` is `{ accessibility, screen_recording, automation }` where each field is `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }` ### Cross-platform capability map (P2-O8 through O17) @@ -839,12 +776,12 @@ crates/windows/src/ | Tree root | `IUIAutomation.ElementFromHandle()` | Via `uiautomation` crate (v0.24+) wrapping UIA COM APIs via `windows` crate | | Children | `IUIAutomationTreeWalker.GetFirstChild` / `GetNextSibling` | With `CacheRequest` for batch attribute retrieval (3-5x faster) | | Role mapping | `UIA ControlType` integers | Map to unified role enum in `tree/roles.rs` — e.g. `UIA_ButtonControlTypeId` → `button` | -| Click | `InvokePattern.Invoke()` | Pattern-based, falls back to `TogglePattern.Toggle()`, then coordinate click via SendInput | -| Set text | `ValuePattern.SetValue()` | Falls back to SelectAll + SendInput keystroke sequence | +| Click | `InvokePattern.Invoke()` | Pattern-based; coordinate click via SendInput only under explicit physical policy | +| Set text | `ValuePattern.SetValue()` | Headless value write by default; SendInput only under explicit focus/physical policy | | Expand/Collapse | `ExpandCollapsePattern.Expand()` / `.Collapse()` | Native UIA pattern | | Select | `SelectionItemPattern.Select()` | For combobox, listbox, tab items | | Toggle | `TogglePattern.Toggle()` | For checkboxes, switches | -| Scroll | `ScrollPattern.Scroll()` / `ScrollPattern.SetScrollPercent()` | Native UIA scroll, falls back to mouse wheel | +| Scroll | `ScrollPattern.Scroll()` / `ScrollPattern.SetScrollPercent()` | Native UIA scroll; mouse wheel only under explicit physical policy | | Keyboard | `SendInput` API | `INPUT_KEYBOARD` structs with virtual key codes and scan codes | | Mouse | `SendInput` API | `INPUT_MOUSE` structs with `MOUSEEVENTF_*` flags | | Clipboard | `OpenClipboard` / `GetClipboardData` / `SetClipboardData` | Win32 APIs, handle `CF_UNICODETEXT` format | @@ -1337,7 +1274,7 @@ Per [Skill Maintenance Addendum](./prd-addendum-skill-maintenance.md): Phase 4 adds a new I/O layer. Core engine and all three platform adapters are unchanged. The MCP server wraps existing command logic in JSON-RPC tool definitions, enabling agent-desktop to work as an MCP-native desktop automation server for Claude Desktop, Cursor, VS Code Copilot, Gemini CLI, Microsoft Agent Framework 1.0, and any other MCP-compatible host. -By Phase 4 the CLI already covers 53+ commands on three platforms, the FFI ships as a shared library for in-process consumers, and the cross-platform event / text-range / stable-selector primitives from Phase 2 / 3 are in place. MCP mode is a **transport + discovery layer**, nothing more. Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant at the top of this document, the MCP crate contains zero per-tool and zero per-platform code — it walks the same compile-time `inventory` registry the CLI and FFI use, and dispatches to the same `execute(args, adapter)` functions. New commands added in Phase 2 or Phase 5 (e.g. `watch_element`, `text select-range`, `find --visual`) become MCP tools automatically with no changes to `crates/mcp/`. +By Phase 4 the CLI already covers the shared command surface on three platforms, the FFI ships as a shared library for in-process consumers, and the cross-platform event / text-range / stable-selector primitives from Phase 2 / 3 are in place. MCP mode is a **transport + discovery layer**, nothing more. Per the [Command Surface Architecture](#command-surface-architecture-dry-invariant) invariant at the top of this document, the MCP crate contains zero per-tool and zero per-platform code — it walks the same deterministic command descriptor registry the CLI and FFI use, and dispatches to the same `execute(args, adapter)` functions. New commands added in Phase 2 or Phase 5 (e.g. `watch_element`, `text select-range`, `find --visual`) become MCP tools automatically with no changes to `crates/mcp/`. ### Objectives @@ -1366,7 +1303,7 @@ This is the invariant: every MCP tool maps 1:1 to a CLI command. `agent-desktop ### New Crate: `agent-desktop-mcp` (platform-agnostic, no per-command code) -The MCP crate is small and generic by design. It contains **zero per-tool files and zero per-platform code**. Per the Command Surface Architecture invariant at the top of this document, every CLI command auto-registers into a shared `inventory` registry; the MCP server iterates that registry at startup and exposes each entry as an MCP tool. +The MCP crate is small and generic by design. It contains **zero per-tool files and zero per-platform code**. Per the Command Surface Architecture invariant at the top of this document, every CLI command is described through deterministic command metadata; the MCP server iterates those descriptors at startup and exposes each entry as an MCP tool. ``` crates/mcp/src/ @@ -1379,7 +1316,7 @@ crates/mcp/src/ └── schema.rs # Translates CommandDescriptor → rmcp tool definition ``` -That's the whole crate. It doesn't know what `desktop_click` does — it reads the `CommandDescriptor` registered by `crates/core/src/commands/click.rs` and forwards invocations through the same `execute(args, adapter)` function the CLI uses. Adding a command in Phase 2 (`text select-range`, `watch_element`) or Phase 5 (`find --visual`, `audit tail`) means **zero lines of MCP code** — the new command shows up automatically in `tools/list`. +That's the whole crate. It doesn't know what `desktop_click` does — it reads generated command descriptors and forwards invocations through the same command execution function the CLI uses. Adding a command in Phase 2 (`text select-range`, `watch_element`) or Phase 5 (`find --visual`, `audit tail`) should mean **zero lines of MCP-specific behavior** — only shared command metadata and adapter methods change. ### MCP tool registration — the one-time rewrite @@ -1389,10 +1326,10 @@ That's the whole crate. It doesn't know what `desktop_click` does — it reads t pub async fn serve(adapter: Box) -> Result<()> { let mut server = rmcp::ServerBuilder::new("agent-desktop", env!("CARGO_PKG_VERSION")); - // Walk the compile-time registry. No hand-maintained tool list. - for cmd in inventory::iter::() { + // Walk generated descriptors. No hand-maintained tool list. + for cmd in command_descriptors() { // Skip tools disallowed by current permission set (P4-O11). - if !cmd.available_under(&adapter.check_permissions()) { continue; } + if !cmd.available_under(&adapter.permission_report()) { continue; } server.tool(rmcp::Tool { name: cmd.mcp_name, @@ -1508,7 +1445,7 @@ Each host gets a ~30-line config + a 60-second "hello agent" demo (launch Calcul On receiving MCP `initialize`: 1. Detect platform (macOS / Windows / Linux) -2. Check accessibility permissions (`check_permissions()`) +2. Check permissions (`permission_report()`) 3. Report capabilities: list of available tools, platform, permission status 4. If permissions not granted, include guidance in capabilities response @@ -1622,7 +1559,7 @@ Per [Skill Maintenance Addendum](./prd-addendum-skill-maintenance.md): --- -## Phase 5 — Production Hardening +## Phase 5 — Production Readiness **Status: Planned** @@ -1908,7 +1845,7 @@ The README is updated at the end of each phase to reflect the current state: | Phase | README Changes | |-------|---------------| -| Phase 1 | Initial README: npm + source installation, core workflow, all 53 commands, JSON output, ref system, error codes, platform support table (macOS only) | +| Phase 1 | Initial README: npm + source installation, core workflow, all 54 commands, JSON output, ref system, error codes, platform support table (macOS only) | | Phase 1.5 | Add "Language bindings (FFI)" section: platform→artifact table, 5-line Python dlopen snippet, `shasum -a 256 -c checksums.txt` + `gh attestation verify` verification, link to `skills/agent-desktop-ffi/` | | Phase 2 | Add Windows: `.exe` installation, Windows permissions, update platform table, Windows build instructions | | Phase 3 | Add Linux: binary installation, AT-SPI2 setup, update platform table, Linux build instructions, minimum OS versions | @@ -1930,8 +1867,8 @@ Per the [Skill Maintenance Addendum](./prd-addendum-skill-maintenance.md): See [Command Surface Architecture](#command-surface-architecture-dry-invariant) for the full layering. Summary of the invariant enforced on every PR: - A new command creates exactly **one** file under `crates/core/src/commands/`. -- That file registers itself via `inventory::submit! { CommandDescriptor { … } }`. -- The CLI, FFI, and MCP transports each walk that registry at startup / build time; none of them hand-maintains a command list. +- CLI and batch must share the typed `Commands` enum, `CommandPolicy`, and `dispatch()` path. +- Any future registry/codegen must be deterministic `build.rs` filesystem enumeration, not `inventory` or `linkme`. - Per-platform work is limited to the `PlatformAdapter` trait implementations in `crates/{macos,windows,linux}/` — never per-transport, never per-command. - PRs that add a command to a single transport without updating the shared registry fail review. If a task in this document sounds like it requires per-transport duplication, it's a wording bug — the actual implementation follows the registry pattern. @@ -1984,7 +1921,7 @@ All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test -- | Set text | `AXValue = val` | `ValuePattern.SetValue()` | `Text.InsertText` | | Keyboard | `CGEventCreateKeyboard` | `SendInput` | `xdotool` / `ydotool` | | Clipboard | `NSPasteboard` | Win32 Clipboard API | `wl-clipboard` / `xclip` | -| Screenshot | `CGWindowListCreateImage` | `BitBlt` / `PrintWindow` | `PipeWire` / `XGetImage` | +| Screenshot | `ScreenshotBackend` over secure `screencapture` path today; ScreenCaptureKit planned | `BitBlt` / `PrintWindow` legacy, Windows.Graphics.Capture planned | `PipeWire` / `XGetImage` | | Permissions | `AXIsProcessTrusted()` | COM security / UAC | Bus availability | | Notifications | Notification Center AX tree (`com.apple.notificationcenterui`) | UIA tree of Action Center / Toast Manager | D-Bus `org.freedesktop.Notifications` + daemon-specific history | | System tray | `SystemUIServer` AX tree + `ControlCenter` AX tree | UIA tree of `Shell_TrayWnd` + overflow window | D-Bus `StatusNotifierWatcher` + XEmbed fallback | @@ -2003,6 +1940,6 @@ All runners enforce: `cargo clippy --all-targets -- -D warnings`, `cargo test -- | R6 | MCP spec changes break compat | Low | Medium | Pin `rmcp` version. Monitor spec under Linux Foundation governance. | | R7 | Tree traversal too slow (>5s) | Medium | Medium | Depth limiting via `--max-depth`. Focused-window-only. Cached subtrees in Phase 5 daemon. Progressive skeleton traversal (`--skeleton` + `--root`) reduces token consumption 78-96% for dense apps. | | R8 | Ref instability confuses agents | Medium | High | Clear docs: refs are snapshot-scoped. `STALE_REF` error with recovery hint. Stable hashing in Phase 5. Progressive skeleton traversal with scoped invalidation provides a stable drill-down workflow for navigating complex UIs. **Phase 2**: stable-selector fields (`identifier`, `subrole`, `role_description`, `placeholder`, `dom_id`, `dom_classes` via `StableSelectors` flatten) + identifier-preferred resolver drop `STALE_REF` rate on Electron / localized apps. | -| R9 | Headless operation requirement | High | Critical | **Phase 2 plan §Headless-First Invariant** codifies the contract: UIA-first / AX-first paths for all actions; `NSDraggingSession` rejected for FileDrop (replaced by `DeliverFiles` 4-tier fallback); `ad_init()` version handshake; integration tests assert no focus steal and no cursor movement for non-mouse commands. | +| R9 | Headless operation requirement | High | Critical | Phase 1 introduced `ActionRequest`/`InteractionPolicy`, default no focus steal/cursor movement, and explicit physical/headed policy paths. Phase 2 must preserve the same contract for Windows/Linux. | | R10 | Command registry link-GC | Medium | High | Research Topic B confirmed `inventory`/`linkme` are unreliable across linkers for cdylib consumers. Resolved by pure `build.rs` filesystem enumeration — zero linker magic. | | R11 | Skeleton traversal cross-platform | Low | High | Core is already platform-agnostic (`crates/core/src/snapshot_ref.rs`); Windows needs ~50 LOC glue (`ControlViewWalker` + `FindAll(TreeScope_Children, TrueCondition)` + fresh `UICacheRequest` per drill-down). Research Topic 4 confirmed `ElementFromHandle(hwnd)` is headless-safe. | diff --git a/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md b/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md new file mode 100644 index 0000000..49b5034 --- /dev/null +++ b/docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md @@ -0,0 +1,62 @@ +--- +title: Bump envelope version when top-level response contracts change +date: 2026-05-13 +category: best-practices +module: crates/core, src +problem_type: best_practice +component: output-contract +severity: high +applies_when: + - Top-level JSON response fields change + - Permission or status envelopes change shape + - Error code semantics change for an existing command + - CLI and batch output contracts are unified +tags: + - json-contract + - release-please + - cli + - batch + - compatibility +--- + +# Bump envelope version when top-level response contracts change + +## Context + +Phase 1 moved CLI and batch through one typed command path and changed several +machine-visible contracts at the same time: permissions became a nested +`PermissionReport`, snapshot refs became snapshot-scoped, new error codes became +observable, and command outputs started sharing the same envelope version. + +The response envelope is how clients decide which parser branch to use. If the +envelope remains unchanged while top-level fields or error semantics change, +pinned clients cannot distinguish an old response from a new one. + +## Guidance + +Keep `crates/core/src/output.rs::ENVELOPE_VERSION` as the single source of +truth. Bump it whenever a client that already parses successful JSON needs a +different parser branch for the same command name. + +Bump the envelope version for: + +- top-level response shape changes +- nested shape changes in always-present fields such as permissions/status +- command success becoming a structured error for a previously successful path +- new required fields in success payloads +- error-code changes that callers reasonably branch on + +Do not bump it for additive optional fields, new commands, new flags, or more +specific human-readable messages. + +## Release Notes + +Every envelope bump must have a `BREAKING CHANGE:` footer in the commit body. +When CHANGELOG output is generated by release-please, do not hand-edit +`CHANGELOG.md`; make the footer precise enough for release-please to generate +the release note. + +## Tests + +Tests should assert the version through `ENVELOPE_VERSION`, not string literals, +so `main`, `batch`, and unit-level envelope tests stay aligned after a bump. diff --git a/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md new file mode 100644 index 0000000..d673a86 --- /dev/null +++ b/docs/solutions/best-practices/keep-ffi-action-policy-aligned-with-cli-2026-05-12.md @@ -0,0 +1,44 @@ +--- +title: Keep FFI action policy aligned with CLI action policy +date: 2026-05-12 +category: best-practices +module: crates/ffi +problem_type: best_practice +component: ffi +severity: high +applies_when: + - FFI exposes an action path that mirrors CLI ref commands + - A core action gains an InteractionPolicy or other side-effect gate + - A new FFI wrapper calls PlatformAdapter directly instead of command dispatch +tags: + - ffi + - interaction-policy + - cli-parity + - abi +--- + +# Keep FFI action policy aligned with CLI action policy + +## Context + +The CLI action path moved to `ActionRequest { action, policy }`, but the FFI +`ad_execute_action` wrapper initially constructed `ActionRequest::physical` for +every action. That meant C, Swift, Python, Go, and Node consumers received +focus-stealing and cursor-moving behavior for actions that the CLI treats as +headless by default. + +## Guidance + +FFI wrappers that mirror CLI actions must use the same default side-effect +contract as the CLI. For direct action execution, the default is headless: +no focus stealing and no cursor movement. + +When a lower-level FFI consumer intentionally wants broader behavior, expose the +policy as an explicit ABI value. Do not hide it inside a wrapper default. + +## Review Rule + +Any change to `ActionRequest`, `InteractionPolicy`, or command preflight must +include a pass over `crates/ffi/src/actions/`. If the CLI and FFI can perform the +same action, they must document the same default and expose any divergence as an +explicit parameter. diff --git a/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md b/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md new file mode 100644 index 0000000..9fc5ca6 --- /dev/null +++ b/docs/solutions/best-practices/preserve-command-policy-semantics-during-refactor-2026-05-12.md @@ -0,0 +1,86 @@ +--- +title: Preserve command policy semantics during shared ref-action refactors +date: 2026-05-12 +category: best-practices +module: crates/core, crates/macos +problem_type: best_practice +component: command-policy +severity: high +applies_when: + - Ref-consuming commands are moved onto a shared execution helper + - A command has semantic AX steps plus explicit keyboard, clipboard, focus, or cursor paths + - A command reports success after AXValue writes without verifying app-observable state + - A DRY cleanup changes ActionRequest or InteractionPolicy construction +tags: + - command-policy + - ref-actions + - interaction-policy + - macos + - regression-prevention +--- + +# Preserve command policy semantics during shared ref-action refactors + +## Context + +The unified ref-action helper removed repeated `resolve_ref + execute_action + to_value` +boilerplate across commands. That cleanup was correct, but two command-specific +policy choices were accidentally flattened during earlier review rounds: + +- `clear` initially reported success after AXValue writes without verifying + app-observable state, so web-backed controls could remain unchanged. +- `type` initially rejected non-ASCII text after an AXValue failure instead of + using the explicit focus-permitted paste path. + +Both commands still compiled and returned structured responses. The regression +was semantic: web-backed fields can report AXValue success while leaving the app's +JS model unchanged, so post-condition verification is part of the command +contract. + +## Guidance + +Shared ref-action dispatch should only remove repeated mechanics. It must not +choose the `InteractionPolicy` for a command, and it must not drop +post-condition verification that decides whether fallback steps should run. +Default CLI ref commands must stay headless: no focus stealing, no cursor +movement, and no synthetic keyboard or pasteboard use unless an explicit policy +path opted into it. + +Each command owns its policy: + +- Use `ActionRequest::headless` when the command is purely semantic AX work and + must not focus, move the cursor, or synthesize input. +- Use `ActionRequest::focus_fallback` only for APIs that have explicitly opted + into focus-changing behavior, such as CLI `type` after AXValue failure or FFI + callers selecting `AD_POLICY_KIND_FOCUS_FALLBACK`. +- Use `ActionRequest::physical` only for explicit physical interaction commands + or FFI callers selecting `AD_POLICY_KIND_PHYSICAL`. + +Do not infer policy from the fact that a command consumes a ref. `click`, +`check`, `expand`, `collapse`, `scroll-to`, `clear`, and `type` all consume refs, +but they still need command-specific success verification and error guidance. + +## Review Rule + +When a patch consolidates ref-consuming commands, review each call site for the +specific `ActionRequest` constructor. A helper like `execute_ref_action` is safe +only when the caller passes the already-chosen request. If a helper internally +constructs a default policy for many commands, treat that as a regression risk. + +## Regression Tests + +Backfill tests at the command or adapter boundary for commands whose policy is +part of correctness: + +- `clear` must dispatch headlessly from the CLI and must verify AXValue writes + before reporting success. +- `type` must attempt AXValue first from the CLI, then use only its explicit + focus-fallback tier when AXValue cannot update the target. +- FFI policy-specific tests must prove focus and physical paths are available + only when the caller explicitly selects that policy. +- A generic ref-action helper should preserve both the `Action` variant and the + caller's `InteractionPolicy`. + +For AX value writes, treat "set returned success" as incomplete evidence on +web-backed controls. Read back the value when the field is not secure; a +mismatch must be a failed step so the next command-specific fallback can run. diff --git a/docs/solutions/logic-errors/progressive-snapshot-review-hardening-2026-04-16.md b/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md similarity index 96% rename from docs/solutions/logic-errors/progressive-snapshot-review-hardening-2026-04-16.md rename to docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md index 96a9dd9..b3b8be8 100644 --- a/docs/solutions/logic-errors/progressive-snapshot-review-hardening-2026-04-16.md +++ b/docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md @@ -1,6 +1,6 @@ --- -## title: Harden progressive snapshot after review (skeleton reset, drill window, validation, depth) +## title: Progressive snapshot contract fixes after review (skeleton reset, drill window, validation, depth) date: 2026-04-16 category: logic-errors module: agent-desktop-core @@ -23,7 +23,7 @@ tags: - macos - agent-contract -# Harden progressive snapshot after review (skeleton reset, drill window, validation, depth) +# Progressive snapshot contract fixes after review (skeleton reset, drill window, validation, depth) ## Problem @@ -67,4 +67,4 @@ A focused review of the `feat/progressive-skeleton-traversal` work found several ## Related Issues - [Known pattern] DRY ref allocation — `[docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md](../best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md)` -- Plan: `docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md` \ No newline at end of file +- Plan: `docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md` diff --git a/skills/agent-desktop-ffi/SKILL.md b/skills/agent-desktop-ffi/SKILL.md index da0c925..1769b4f 100644 --- a/skills/agent-desktop-ffi/SKILL.md +++ b/skills/agent-desktop-ffi/SKILL.md @@ -62,6 +62,17 @@ Four reference topics, loaded as needed: that produced it. On macOS this balances the internal `CFRetain`; on Windows/Linux the call is a no-op but safe to issue. +- **Action policy.** `ad_execute_action` uses the headless policy by + default, matching CLI ref commands: no focus stealing and no cursor + movement. Use `ad_execute_action_with_policy(..., + AD_POLICY_KIND_FOCUS_FALLBACK, ...)` only when focus-changing behavior is + intended, and `AD_POLICY_KIND_PHYSICAL` only for explicit physical/headed + input semantics. + +- **Text input privacy.** On macOS, explicit focus/physical policy can use the + clipboard briefly for non-ASCII text insertion. Keep the default headless + policy or set values directly for sensitive text when the target supports it. + - **Enum discriminants.** Every `#[repr(i32)]` enum field is validated at the C boundary — invalid discriminants return `AD_RESULT_ERR_INVALID_ARGS` instead of undefined behavior. diff --git a/skills/agent-desktop-ffi/references/error-handling.md b/skills/agent-desktop-ffi/references/error-handling.md index 777db03..0ffaecf 100644 --- a/skills/agent-desktop-ffi/references/error-handling.md +++ b/skills/agent-desktop-ffi/references/error-handling.md @@ -41,11 +41,18 @@ ad_check_permissions(adapter); // success printf("%s\n", msg); // still valid ``` +`ad_check_permissions` does not treat `Unknown` as success. Stub adapters that +cannot answer permission probes return `AD_RESULT_ERR_PLATFORM_NOT_SUPPORTED`. +The macOS adapter reports `AD_RESULT_ERR_INTERNAL` only if the platform probe +itself is ambiguous; read `ad_last_error_*` for the diagnostic. + Failure-path calls rotate: if a subsequent call fails, the prior pointer may dangle. Read it before the next potentially-failing call. ## Error codes +Numeric values are ABI-stable. New codes are appended; existing values are not renumbered. + | Name | i32 | Meaning | |--------------------------------------|-------|---------------------------------------| | `AD_RESULT_OK` | 0 | Success | @@ -60,7 +67,9 @@ pointer may dangle. Read it before the next potentially-failing call. | `AD_RESULT_ERR_TIMEOUT` | -9 | Wait exceeded deadline | | `AD_RESULT_ERR_INVALID_ARGS` | -10 | Null pointer, bad enum, invalid UTF-8 | | `AD_RESULT_ERR_NOTIFICATION_NOT_FOUND`| -11 | Notification index out of range | -| `AD_RESULT_ERR_INTERNAL` | -12 | Rust panic caught at FFI boundary | +| `AD_RESULT_ERR_INTERNAL` | -12 | Internal failure or ambiguous platform probe | +| `AD_RESULT_ERR_SNAPSHOT_NOT_FOUND` | -13 | Requested snapshot ref store is missing | +| `AD_RESULT_ERR_POLICY_DENIED` | -14 | Current action policy blocks fallback | ## Enum validation diff --git a/skills/agent-desktop/SKILL.md b/skills/agent-desktop/SKILL.md index d555790..22be015 100644 --- a/skills/agent-desktop/SKILL.md +++ b/skills/agent-desktop/SKILL.md @@ -1,6 +1,6 @@ --- name: agent-desktop -version: 0.1.8 +version: 0.1.14 tags: desktop-automation, accessibility, ai-agent, gui-automation, cli requirements: - agent-desktop @@ -15,7 +15,8 @@ description: > Triggers on: "click button", "fill form", "open app", "read UI", "automate desktop", "accessibility tree", "snapshot app", "type into field", "navigate menu", "toggle checkbox", "take screenshot", "desktop automation", "agent-desktop", or any desktop GUI interaction task. - Supports macOS (Phase 1), with Windows and Linux planned. + Supports the macOS Phase 1 adapter, with Windows and Linux planned against + the same core contracts. --- # agent-desktop @@ -32,7 +33,7 @@ npm install -g agent-desktop bun install -g --trust agent-desktop ``` -Requires macOS 12+ with Accessibility permission granted to your terminal. +Requires macOS 12+ with Accessibility permission granted to your terminal. Screen Recording permission is also required for screenshots. ## Reference Files @@ -55,13 +56,14 @@ Use **progressive skeleton traversal** as the default approach. It reduces token Parse the overview. Identify the region containing your target. Regions show children_count (e.g., "Sidebar" with children_count: 42). Named containers at truncation boundary have refs for drill-down. + Keep the returned snapshot_id. -2. DRILL → agent-desktop snapshot --root @e3 -i --compact +2. DRILL → agent-desktop snapshot --root @e3 --snapshot -i --compact Expand the target region. Now you see its interactive elements. -3. ACT → agent-desktop click @e12 (or type, select, toggle...) +3. ACT → agent-desktop click @e12 --snapshot (or type, select, toggle...) -4. VERIFY → agent-desktop snapshot --root @e3 -i --compact +4. VERIFY → agent-desktop snapshot --root @e3 --snapshot -i --compact Re-drill the same region to confirm the state change. Scoped invalidation: only @e3's subtree refs are replaced. @@ -85,6 +87,8 @@ Use **progressive skeleton traversal** as the default approach. It reduces token - In skeleton mode, named/described containers at truncation boundary also get refs (drill-down targets with empty `available_actions`) - Static text, groups, containers remain in tree for context but have no ref - Refs are deterministic within a snapshot but NOT stable across snapshots if UI changed +- Every snapshot returns `snapshot_id`; ref-consuming commands accept `--snapshot ` +- `last_refmap.json` is only a latest-snapshot inspection artifact. The command path uses snapshot-scoped storage. - After any action that changes UI, re-drill the affected region or re-snapshot - **Scoped invalidation:** re-drilling `--root @e3` only replaces refs from @e3's previous drill — refs from other regions and the skeleton itself are preserved @@ -92,8 +96,8 @@ Use **progressive skeleton traversal** as the default approach. It reduces token Every command returns a JSON envelope on stdout: -**Success:** `{ "version": "1.0", "ok": true, "command": "snapshot", "data": { ... } }` -**Error:** `{ "version": "1.0", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "...", "suggestion": "..." } }` +**Success:** `{ "version": "2.0", "ok": true, "command": "snapshot", "data": { ... } }` +**Error:** `{ "version": "2.0", "ok": false, "command": "click", "error": { "code": "STALE_REF", "message": "...", "suggestion": "..." } }` Exit codes: `0` success, `1` structured error, `2` argument error. @@ -101,15 +105,19 @@ Exit codes: `0` success, `1` structured error, `2` argument error. | Code | Meaning | Recovery | |------|---------|----------| -| `PERM_DENIED` | Accessibility permission not granted | Grant in System Settings > Privacy > Accessibility | -| `ELEMENT_NOT_FOUND` | Ref not in current refmap | Re-run snapshot, use fresh ref | +| `PERM_DENIED` | Accessibility or Screen Recording permission not granted | Grant the named permission in System Settings | +| `ELEMENT_NOT_FOUND` | Ref cannot be resolved against the live UI | Re-run snapshot, use fresh ref | | `APP_NOT_FOUND` | App not running | Launch it first | -| `ACTION_FAILED` | AX action rejected | Try alternative approach or coordinate-based click | +| `ACTION_FAILED` | AX action rejected | Try an explicit alternative command | | `ACTION_NOT_SUPPORTED` | Element can't do this | Use different command | | `STALE_REF` | Ref from old snapshot | Re-run snapshot | +| `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired | Run `snapshot` again and use the returned ID | +| `POLICY_DENIED` | A physical/headed path was blocked | Use an explicit mouse/focus/keyboard command if physical interaction is intended | | `WINDOW_NOT_FOUND` | No matching window | Check app name, use list-windows | +| `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform | Use a supported platform adapter | | `TIMEOUT` | Wait condition not met | Increase --timeout | | `INVALID_ARGS` | Bad arguments | Check command syntax | +| `NOTIFICATION_NOT_FOUND` | Notification index no longer exists | Re-run list-notifications | ## Command Quick Reference (54 commands) @@ -121,22 +129,22 @@ agent-desktop snapshot --app "App" -i # Full tree (simple agent-desktop snapshot --app "App" --surface menu -i # Surface snapshot agent-desktop screenshot --app "App" out.png # PNG screenshot agent-desktop find --app "App" --role button # Search elements -agent-desktop get @e1 --property text # Read element property -agent-desktop is @e1 --property enabled # Check element state +agent-desktop get @e1 --snapshot s... --property text # Read element property +agent-desktop is @e1 --snapshot s... --property enabled # Check element state agent-desktop list-surfaces --app "App" # Available surfaces ``` ### Interaction ``` -agent-desktop click @e5 # Click element -agent-desktop double-click @e3 # Double-click -agent-desktop triple-click @e2 # Triple-click (select line) -agent-desktop right-click @e5 # Right-click (context menu) -agent-desktop type @e2 "hello" # Type text into element +agent-desktop click @e5 --snapshot s... # AX-first click, no cursor move by default +agent-desktop double-click @e3 # AXOpen; physical double-click uses mouse-click --count 2 +agent-desktop triple-click @e2 # Physical triple-click uses mouse-click --count 3 +agent-desktop right-click @e5 # Right-click; menu returned when verified +agent-desktop type @e2 --snapshot s... "hello" # Headless AX text insertion when supported agent-desktop set-value @e2 "new value" # Set value directly agent-desktop clear @e2 # Clear element value agent-desktop focus @e2 # Set keyboard focus -agent-desktop select @e4 "Option B" # Select dropdown option +agent-desktop select @e4 "Option B" # Select dropdown/list option agent-desktop toggle @e6 # Toggle checkbox/switch agent-desktop check @e6 # Idempotent check agent-desktop uncheck @e6 # Idempotent uncheck @@ -152,7 +160,7 @@ agent-desktop press cmd+c # Key combo agent-desktop press return --app "App" # Targeted key press agent-desktop key-down shift # Hold key agent-desktop key-up shift # Release key -agent-desktop hover @e5 # Cursor to element +agent-desktop hover @e5 # Explicit cursor movement agent-desktop hover --xy 500,300 # Cursor to coordinates agent-desktop drag --from @e1 --to @e5 # Drag between elements agent-desktop mouse-click --xy 500,300 # Click at coordinates @@ -197,10 +205,10 @@ agent-desktop clipboard-clear # Clear clipboard ### Wait ``` agent-desktop wait 1000 # Pause 1 second -agent-desktop wait --element @e5 --timeout 5000 # Wait for element +agent-desktop wait --element @e5 --snapshot s... --timeout 5000 # Wait for element agent-desktop wait --window "Title" # Wait for window agent-desktop wait --text "Done" --app "App" # Wait for text -agent-desktop wait --menu --app "App" # Wait for context menu +agent-desktop wait --menu --app "App" # Wait for menu surface agent-desktop wait --menu-closed --app "App" # Wait for menu dismissal agent-desktop wait --notification --app "App" # Wait for new notification ``` @@ -211,7 +219,7 @@ agent-desktop status # Health check agent-desktop permissions # Check permission agent-desktop permissions --request # Trigger permission dialog agent-desktop version --json # Version info -agent-desktop batch '[...]' --stop-on-error # Batch commands +agent-desktop batch '[...]' --stop-on-error # Batch uses the same typed command path as CLI agent-desktop skills # List bundled skill docs agent-desktop skills get desktop --full # Load this skill + all references ``` @@ -220,11 +228,12 @@ agent-desktop skills get desktop --full # Load this skill + all referenc 1. **Skeleton first, drill second.** Start with `--skeleton -i --compact` for dense apps. Drill into regions with `--root @ref`. Full snapshot only for simple apps. 2. **Use `-i --compact` flags.** Filters to interactive elements and collapses empty wrappers, minimizing tokens. -3. **Refs are ephemeral.** Re-drill the affected region after any UI-changing action. Scoped invalidation keeps other refs intact. +3. **Refs are snapshot-scoped.** Keep `snapshot_id` for deterministic multi-step use; re-drill the affected region after any UI-changing action. Scoped invalidation keeps other refs intact. 4. **Prefer refs over coordinates.** `click @e5` > `mouse-click --xy 500,300`. 5. **Use `wait` for async UI.** After launch/dialog triggers, wait for expected state. -6. **Check permissions first.** Run `permissions` on first use. +6. **Check permissions first.** Run `permissions` on first use; screenshots also need Screen Recording. 7. **Handle errors.** Parse `error.code` and follow `error.suggestion`. 8. **Use `find` for targeted searches.** Faster than any snapshot when you know role/name. 9. **Use surfaces for overlays.** `snapshot --surface menu` for menus, `--surface sheet` for dialogs. Never `--skeleton` for surfaces — they're already focused. 10. **Batch for performance.** Multiple commands in one invocation. +11. **Headless by default.** Ref actions use semantic AX paths and block silent focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. Use explicit `focus`, `press`, `hover`, `drag`, or `mouse-*` commands only when physical/headed interaction is intended. diff --git a/skills/agent-desktop/references/commands-interaction.md b/skills/agent-desktop/references/commands-interaction.md index 2088f7b..3673ec7 100644 --- a/skills/agent-desktop/references/commands-interaction.md +++ b/skills/agent-desktop/references/commands-interaction.md @@ -2,33 +2,38 @@ Commands for modifying UI state — clicking, typing, selecting, scrolling, and input synthesis. +Ref-based actions are headless by default. They try semantic accessibility operations and do not silently steal focus, move the cursor, synthesize keyboard input, or use the pasteboard. Physical/headed interaction is reserved for explicit `focus`, `press`, `hover`, `drag`, and `mouse-*` commands or an explicit FFI policy. The `type` command has an explicit focus-fallback tier for callers that opt into focus changes while still forbidding cursor movement; the default CLI path remains AX-value-first and headless. + +All ref-based interaction commands accept `--snapshot `. Omit it for the latest saved snapshot, or pass the `snapshot_id` returned by `snapshot` to keep scripts pinned to the exact ref map they observed. + ## Click Actions -All click commands use a smart activation chain (AX-first) that tries accessibility actions before falling back to coordinate-based clicks. +Click commands use semantic AX activation first. In the default headless policy, coordinate click fallback is blocked; use `mouse-click` only when physical cursor movement is intended. ### click ```bash agent-desktop click @e5 +agent-desktop click @e5 --snapshot s... ``` -Primary activation. Tries AXPress > AXConfirm > AXOpen > AXPick > child activation > focus+activate > coordinate click. +Primary activation. Tries verified AXPress > AXConfirm > AXOpen > AXPick > child activation > selection/value relays > custom actions > ancestor activation. Focus-stealing and coordinate fallback steps are not used by the default ref command path. ### double-click ```bash agent-desktop double-click @e3 ``` -Tries AXOpen first, then two smart activations with 50ms gap, then CGEvent double-click. +Tries AXOpen. Physical double-click fallback is blocked by default policy; use `mouse-click --xy X,Y --count 2` when a headed physical double-click is intended. ### triple-click ```bash agent-desktop triple-click @e2 ``` -Three smart activations with 30ms gaps, then CGEvent triple-click. Useful for select-all in text fields. +Physical triple-click requires cursor/focus side effects and is blocked by default policy; use `mouse-click --xy X,Y --count 3` when a headed physical triple-click is intended. ### right-click ```bash agent-desktop right-click @e5 ``` -Opens context menu. Tries AXShowMenu > focus+AXShowMenu > parent/child AXShowMenu > coordinate right-click. Use `wait --menu` after to capture the menu, then `snapshot --surface menu` to read it. +Performs a semantic right-click/context-menu action and includes the menu tree when a menu surface can be verified. If the right-click action succeeds but menu probing fails, the command still returns the action result with `menu_probe.ok: false` so callers do not retry and double-open context menus. Combo boxes and menu buttons expose menu-opening actions for their primary dropdown; use `select` for those controls, not `right-click`. Focus-stealing and coordinate right-click fallback are blocked by default policy. ## Text Input @@ -37,7 +42,9 @@ Opens context menu. Tries AXShowMenu > focus+AXShowMenu > parent/child AXShowMen agent-desktop type @e2 "hello@example.com" agent-desktop type @e2 "multi line\ntext" ``` -Focuses the element then types each character via keyboard synthesis. Handles special characters. +In the default headless policy, inserts text by mutating the element's AX value when the target has a settable text value. If a target cannot be updated headlessly, the command returns a structured error instead of stealing focus. Physical keyboard synthesis and pasteboard-based insertion are reserved for explicit policy paths. + +When an explicit focus/physical policy is used for non-ASCII text on macOS, the adapter may briefly place the text on the clipboard to paste it. Do not use that path for secrets; prefer the default headless value path or `set-value` when the target supports it. ### set-value ```bash @@ -56,6 +63,7 @@ Clears the element's value to an empty string. Equivalent to `set-value @e2 ""`. agent-desktop focus @e2 ``` Sets keyboard focus on the element without clicking it. +This is an explicit focus-changing command. It uses accessibility focus and does not move the cursor. ## Selection & Toggle @@ -63,7 +71,7 @@ Sets keyboard focus on the element without clicking it. ```bash agent-desktop select @e4 "Option B" ``` -Selects an option in a list, dropdown, or combobox by its display text. +Selects an option in a list, dropdown, or combobox by display text. For menu-backed controls it opens the AX menu, presses the matching menu item, and verifies `AXValue` when the control exposes it. It returns a structured error when the matching item is missing or the exposed value does not change. ### toggle ```bash @@ -112,6 +120,8 @@ agent-desktop scroll @e1 --direction right --amount 2 | `--direction` | down | `up`, `down`, `left`, `right` | | `--amount` | 3 | Number of scroll units | +Uses AX scroll actions, scroll bars, and state-setting paths. If those are unavailable, the command returns a structured error instead of stealing focus or sending wheel events. + ### scroll-to ```bash agent-desktop scroll-to @e8 @@ -159,6 +169,7 @@ agent-desktop hover --xy 500,300 agent-desktop hover @e5 --duration 2000 ``` Moves cursor to element center or absolute coordinates. Optional `--duration` holds position for N ms. +This is an explicit cursor-moving command. ### drag ```bash @@ -213,11 +224,11 @@ Low-level press/release for custom drag or hold interactions. | Goal | Preferred | Alternative | |------|-----------|-------------| | Click a button | `click @ref` | `mouse-click --xy` if AX fails | -| Fill a text field | `type @ref "text"` | `set-value @ref "text"` for direct set | -| Clear then type | `clear @ref` then `type @ref "new"` | `triple-click @ref` then `type @ref "new"` | +| Fill a text field | `clear @ref` then `type @ref "text"` | `set-value @ref "text"` for direct replacement | +| Clear then type | `clear @ref` then `type @ref "new"` | `mouse-click --xy X,Y --count 3` only when physical selection is intended | | Toggle a checkbox | `check @ref` / `uncheck @ref` | `toggle @ref` if you don't know current state | -| Open context menu | `right-click @ref` then `wait --menu` | `mouse-click --xy --button right` | -| Select dropdown option | `select @ref "Option"` | `click @ref` then `find` the option | +| Open context menu | `right-click @ref` | `mouse-click --xy --button right` when physical interaction is intended | +| Select dropdown option | `select @ref "Option"` | `snapshot --surface menu` after an explicitly opened menu | | Navigate a form | `press tab` between fields | `focus @ref` to jump directly | | Copy text | `press cmd+c --app "App"` | `clipboard-set` to set directly | | Scroll to find elements | `scroll @ref --direction down` | `scroll-to @ref` if you have the ref | diff --git a/skills/agent-desktop/references/commands-observation.md b/skills/agent-desktop/references/commands-observation.md index cb06f1e..6e59c41 100644 --- a/skills/agent-desktop/references/commands-observation.md +++ b/skills/agent-desktop/references/commands-observation.md @@ -12,6 +12,8 @@ agent-desktop snapshot --app "Finder" --max-depth 5 --include-bounds agent-desktop snapshot --app "App" --surface menu agent-desktop snapshot --app "App" --window-id "w-1234" agent-desktop snapshot --app "App" -i --compact +agent-desktop snapshot --app "App" --skeleton -i +agent-desktop snapshot --root @e12 --snapshot s8f3k2p9 -i ``` | Flag | Default | Description | @@ -22,20 +24,22 @@ agent-desktop snapshot --app "App" -i --compact | `--max-depth` | 10 | Maximum tree traversal depth | | `--include-bounds` | false | Include `{x, y, width, height}` for each element | | `--compact` | false | Omit empty structural nodes | -| `--skeleton` | false | Shallow overview: clamps depth to min(max_depth, 3), adds `children_count` on truncated containers | -| `--root ` | | Start traversal from this ref instead of window root. Cannot be combined with `--surface` | | `--surface` | window | Target surface: `window`, `focused`, `menu`, `menubar`, `sheet`, `popover`, `alert` | +| `--skeleton` | false | Clamp traversal to depth 3 and add `children_count` to truncated containers | +| `--root ` | | Drill down from a ref discovered in a previous snapshot. Cannot be combined with `--surface` | +| `--snapshot ` | latest | Snapshot ID to use when resolving `--root` | **Output structure:** ```json { - "version": "1.0", + "version": "2.0", "ok": true, "command": "snapshot", "data": { "app": "System Settings", "window": { "id": "w-4521", "title": "General" }, "ref_count": 14, + "snapshot_id": "s8f3k2p9", "tree": { "role": "window", "name": "General", @@ -74,6 +78,7 @@ agent-desktop snapshot --app "App" -i --compact - Starts tree traversal from the given ref instead of the window root - Merges new refs into the existing refmap with scoped invalidation: only refs from the previous drill of the same root are replaced, leaving all other refs intact - Cannot be combined with `--surface` +- Use `--snapshot ` when drilling from a specific snapshot rather than the latest snapshot pointer **Progressive drill-down workflow:** ```bash @@ -81,10 +86,10 @@ agent-desktop snapshot --app "App" -i --compact agent-desktop snapshot --skeleton --app Slack -i # Step 2: Drill into a discovered region -agent-desktop snapshot --root @e3 -i +agent-desktop snapshot --root @e3 --snapshot s8f3k2p9 -i # Step 3: Re-drill same region (scoped invalidation replaces @e3's refs) -agent-desktop snapshot --root @e3 -i +agent-desktop snapshot --root @e3 --snapshot s8f3k2p9 -i ``` **Tips:** @@ -95,6 +100,7 @@ agent-desktop snapshot --root @e3 -i - Combine `--max-depth 5` to limit deep trees (e.g., Xcode) - Use `--skeleton` first to get a high-level map, then `--root` to drill into specific regions - Combine `--skeleton` with `-i` and `--compact` for the most token-efficient initial overview +- Keep `snapshot_id` when commands must resolve against a specific snapshot instead of the latest snapshot pointer ## find @@ -106,6 +112,7 @@ agent-desktop find --app "TextEdit" --role textfield agent-desktop find --app "Safari" --text "Sign In" --first agent-desktop find --app "App" --role checkbox --count agent-desktop find --app "App" --role button --nth 2 +agent-desktop find --app "App" --role button --limit 20 ``` | Flag | Description | @@ -119,6 +126,7 @@ agent-desktop find --app "App" --role button --nth 2 | `--last` | Return last match only | | `--nth N` | Return Nth match (0-indexed) | | `--count` | Return match count only | +| `--limit N` | Return at most N matches; defaults to 50 for match lists, use 0 for all | **Output (matches):** ```json @@ -138,6 +146,7 @@ Read a specific property from an element. ```bash agent-desktop get @e1 --property text +agent-desktop get @e1 --snapshot s8f3k2p9 --property text agent-desktop get @e2 --property value agent-desktop get @e3 --property bounds agent-desktop get @e4 --property role @@ -160,6 +169,7 @@ Check a boolean state on an element. ```bash agent-desktop is @e1 --property visible +agent-desktop is @e1 --snapshot s8f3k2p9 --property visible agent-desktop is @e2 --property enabled agent-desktop is @e3 --property checked agent-desktop is @e4 --property focused @@ -197,6 +207,8 @@ agent-desktop screenshot --window-id "w-1234" capture.png When no output path is given, the screenshot is returned as a base64-encoded string in the JSON `data` field. +Screenshots require Screen Recording permission. Permission denial is reported as `PERM_DENIED`, not `INTERNAL`. + ## list-surfaces List available accessibility surfaces for an application. diff --git a/skills/agent-desktop/references/commands-system.md b/skills/agent-desktop/references/commands-system.md index 0083cde..5b50fb6 100644 --- a/skills/agent-desktop/references/commands-system.md +++ b/skills/agent-desktop/references/commands-system.md @@ -25,8 +25,9 @@ Quits an application gracefully. Use `--force` to kill the process. ### list-apps ```bash agent-desktop list-apps +agent-desktop list-apps --app "Text" ``` -Lists all running GUI applications. Returns array of `{ name, pid, bundle_id }`. +Lists running GUI applications, optionally filtered by a case-insensitive name substring. Returns array of `{ name, pid, bundle_id }`. ## Window Management @@ -35,7 +36,7 @@ Lists all running GUI applications. Returns array of `{ name, pid, bundle_id }`. agent-desktop list-windows agent-desktop list-windows --app "Finder" ``` -Lists all visible windows, optionally filtered by app. Returns array of `{ id, title, app_name, pid, bounds }`. +Lists all visible windows, optionally filtered by app. Returns array of `{ id, title, app_name, pid, bounds, is_focused }`. Focus is detected through the platform's frontmost/focused-window APIs, not window stacking order. ### focus-window ```bash @@ -43,7 +44,7 @@ agent-desktop focus-window --app "Finder" agent-desktop focus-window --title "Documents" agent-desktop focus-window --window-id "w-4521" ``` -Brings a window to the front. At least one identifier required. +Brings a window to the front and confirms the OS reports that same window as focused. At least one identifier is required. If focus does not settle before the deadline, the command returns `ACTION_FAILED` instead of fabricating a focused result. ### resize-window ```bash @@ -167,7 +168,7 @@ Pauses for N milliseconds. Use between actions that need time to settle. ### wait (element) ```bash -agent-desktop wait --element @e5 --timeout 5000 --app "App" +agent-desktop wait --element @e5 --snapshot s... --timeout 5000 --app "App" ``` Blocks until the element ref appears in the accessibility tree. Useful after triggering UI changes. @@ -187,22 +188,24 @@ Blocks until the specified text appears anywhere in the app's accessibility tree ```bash agent-desktop wait --menu --app "Finder" --timeout 3000 ``` -Blocks until a context menu is detected as open. +Blocks until a menu surface is detected as open. ### wait (menu-closed) ```bash agent-desktop wait --menu-closed --app "Finder" --timeout 3000 ``` -Blocks until the context menu is dismissed. +Blocks until the menu surface is dismissed. | Flag | Default | Description | |------|---------|-------------| | (positional) | | Milliseconds to pause | | `--element` | | Ref to wait for | +| `--snapshot` | latest | Snapshot ID for `--element` waits | | `--window` | | Window title to wait for | -| `--text` | | Text to wait for | -| `--menu` | false | Wait for context menu to open | -| `--menu-closed` | false | Wait for context menu to close | +| `--text` | | Text to wait for; with `--notification`, filters notification title/body | +| `--menu` | false | Wait for menu surface to open | +| `--menu-closed` | false | Wait for menu surface to close | +| `--notification` | false | Wait for a new notification | | `--timeout` | 30000 | Timeout in ms (for element/window/text/menu waits) | | `--app` | | Scope the wait to a specific application | @@ -215,6 +218,8 @@ agent-desktop batch '[...]' --stop-on-error ``` Execute multiple commands in sequence from a JSON array. Each entry has `command` (string) and `args` (object). +Batch uses the same typed `Commands` enum, command policy preflight, permission report, and dispatch path as the CLI. Unknown fields are rejected instead of being silently ignored. Nested `batch` is rejected. + | Flag | Default | Description | |------|---------|-------------| | `--stop-on-error` | false | Halt on first failed command | @@ -222,17 +227,31 @@ Execute multiple commands in sequence from a JSON array. Each entry has `command **Batch format:** ```json [ - { "command": "click", "args": { "ref_id": "@e1" } }, + { "command": "click", "args": { "ref_id": "@e1", "snapshot": "s8f3k2p9" } }, { "command": "wait", "args": { "ms": 500 } }, - { "command": "type", "args": { "ref_id": "@e2", "text": "hello" } } + { "command": "type", "args": { "ref_id": "@e2", "snapshot": "s8f3k2p9", "text": "hello" } } ] ``` +**Per-entry failure shape:** +```json +{ + "version": "2.0", + "ok": false, + "command": "click", + "error": { + "code": "STALE_REF", + "message": "Ref '@e1' is stale", + "suggestion": "Run snapshot again and retry with the new ref" + } +} +``` + **Progressive snapshot in batch** — use `skeleton` and `root` fields inside `snapshot` args: ```json [ { "command": "snapshot", "args": { "app": "Slack", "skeleton": true, "interactive_only": true } }, - { "command": "snapshot", "args": { "app": "Slack", "root": "@e3", "interactive_only": true } } + { "command": "snapshot", "args": { "app": "Slack", "root": "@e3", "snapshot": "s8f3k2p9", "interactive_only": true } } ] ``` @@ -244,14 +263,16 @@ Execute multiple commands in sequence from a JSON array. Each entry has `command ```bash agent-desktop status ``` -Returns adapter health, platform info, and permission state. +Returns adapter health, platform info, permission report, and latest snapshot metadata (`snapshot_id`, `ref_count`) when available. ### permissions ```bash agent-desktop permissions agent-desktop permissions --request ``` -Checks accessibility permission status. Use `--request` to trigger the macOS system dialog. +Checks the cached per-process permission report: `accessibility`, `screen_recording`, and `automation`, each as `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording, and `not_required` for Automation because shipped commands use Accessibility, Screen Recording for screenshots, and explicit keyboard/mouse input rather than Apple Events. Use `--request` to invoke the platform request path. + +`status`, `permissions`, command preflight, and `batch` share one permission probe per process. `permissions --request` is the only path that intentionally asks the platform to prompt again. ### version ```bash diff --git a/skills/agent-desktop/references/macos.md b/skills/agent-desktop/references/macos.md index 19955bd..951ce5d 100644 --- a/skills/agent-desktop/references/macos.md +++ b/skills/agent-desktop/references/macos.md @@ -4,7 +4,7 @@ macOS-specific details for agent-desktop. Covers permissions, accessibility API ## Prerequisites -### Accessibility Permission (TCC) +### Permission Report (TCC) macOS requires explicit Accessibility permission for any process that reads or controls UI elements across applications. @@ -16,13 +16,23 @@ agent-desktop permissions agent-desktop permissions --request ``` +The report contains: + +| Field | Meaning | +|-------|---------| +| `accessibility` | Required for accessibility tree reads and most commands | +| `screen_recording` | Required for screenshots | +| `automation` | Reserved for future Apple Event automation paths; `{ "state": "not_required" }` for the current macOS command set | + +Each field is an object: `{ "state": "granted" }`, `{ "state": "denied", "suggestion": "..." }`, `{ "state": "not_required" }`, or `{ "state": "unknown" }`. The current macOS adapter reports concrete `granted` or `denied` states for Accessibility and Screen Recording, and `not_required` for Automation. + **To grant manually:** 1. Open System Settings > Privacy & Security > Accessibility 2. Click the lock to make changes -3. Add your terminal application (Terminal.app, iTerm2, Warp, VS Code, etc.) +3. Add the app that launches agent-desktop (Terminal.app, iTerm2, Warp, VS Code, Codex, etc.) 4. Toggle it ON -**Important:** The permission is granted to the **terminal application**, not to agent-desktop itself. If you run agent-desktop from a different terminal, you need to grant that terminal too. +**Important:** The permission is granted to the launching app. If macOS lists the built `agent-desktop` binary separately, grant that binary too. If you run agent-desktop from a different launcher, grant that launcher as well. After granting, restart the terminal for the permission to take effect. @@ -56,12 +66,25 @@ When you run `click @ref`, agent-desktop doesn't just do a simple click. It runs 8. **AXSelected** — set selected attribute 9. **Select via parent** — set parent's selected rows (for tables/lists) 10. **Custom actions** — AXPerformCustomAction -11. **Focus + activate** — set focus then press/confirm -12. **Keyboard activate** — focus + synthesize space key -13. **Parent activation** — try pressing ancestor elements -14. **Coordinate click** — final fallback: CGEvent click at element bounds center +11. **Ancestor activation** — try pressing ancestor elements +12. **Explicit physical path** — coordinate click only when the caller selected a policy that allows focus stealing and cursor movement -For `right-click`, the chain tries AXShowMenu first, then various focus/select combinations before falling back to a coordinate-based right-click. +For `right-click`, AXShowMenu and related semantic menu paths include a `menu` tree only after a real menu surface appears. If the action succeeds but the menu probe cannot verify a surface, the command still returns success with `menu_probe.ok: false`; callers should inspect that field instead of retrying blindly. Combo boxes and menu buttons use the same AX menu mechanism for their primary dropdown; use `select` for those controls. Coordinate right-click is blocked by the default headless policy. + +Menu verification intentionally requires a closed-to-open transition. If a menu is already open for the target app, `right-click` and `select` refuse to treat that existing menu as proof of success; dismiss the old menu and retry. This avoids acting on a stale sibling menu opened by a prior command. + +The default activation-chain deadline is 10 seconds. Set `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` to a positive millisecond value when diagnosing unusually slow AX targets; values are capped at 300000 ms. Menu verification waits use `AGENT_DESKTOP_MENU_TIMEOUT_MS` with a default of 750 ms and a 10000 ms cap. Toggle verification uses `AGENT_DESKTOP_TOGGLE_TIMEOUT_MS` with a default of 600 ms and a 10000 ms cap; changed toggle values must remain stable for `AGENT_DESKTOP_TOGGLE_STABLE_MS`, default 200 ms and cap 2000 ms. + +### Headless Interaction Policy + +Ref commands use `ActionRequest { action, policy }`. The default policy forbids focus stealing, cursor movement, keyboard synthesis, and pasteboard insertion. macOS actions split semantic AX steps from explicit physical/headed paths: + +- `click`, `right-click`, `scroll`, `set-value`, `clear`, `select`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, and `scroll-to` try AX-first semantics and fail clearly when the headless path is unavailable. +- `type` mutates a settable AX text value in headless mode. It does not call `ensure_app_focused` or use the pasteboard in the default CLI path. +- `focus`, `press`, `hover`, `drag`, and `mouse-*` are explicit physical/focus/cursor commands. +- FFI callers can opt into focus-only or physical policy explicitly; CLI ref commands do not do that implicitly. +- Explicit focus/physical policy can use the clipboard briefly for non-ASCII text insertion. Keep the default headless path or use `set-value` for sensitive text when possible. +- If a command would need a forbidden physical path, it returns a structured error with a recovery hint. ### Surfaces @@ -71,7 +94,7 @@ macOS apps can have multiple accessibility surfaces: |---------|-------------|-------------| | `window` | Main application window (default) | General UI interaction | | `focused` | Currently focused element's context | Inspecting active element | -| `menu` | Open dropdown or context menu | After click/right-click on menu triggers | +| `menu` | Open dropdown or context menu | After `select`, verified `right-click`, or explicit menu trigger | | `menubar` | Application menu bar | Navigating File/Edit/View menus | | `sheet` | Modal sheet (Save dialog, etc.) | After triggering sheet dialogs | | `popover` | Popover/popup content | Inspecting tooltips, popovers | @@ -111,7 +134,7 @@ agent-desktop interacts with macOS Notification Center via the accessibility API ### Dismiss Strategy -Headless-first approach (no cursor movement unless needed): +Headless approach (no cursor movement from the default ref command path): 1. **AXDismiss** / **AXRemoveFromParent** — native accessibility actions 2. **Close button** — find and press AXButton named "close", "clear", or "dismiss" @@ -160,14 +183,22 @@ Some apps don't expose full accessibility trees: - Increase `--max-depth` to explore deeper - Use `screenshot` as a visual fallback -### STALE_REF +### STALE_REF / SNAPSHOT_NOT_FOUND ``` "code": "STALE_REF", -"message": "RefMap is from a previous snapshot" +"message": "Element not found: role=..., name=..." ``` -The UI changed between your snapshot and action. Run `snapshot` again and use the new refs. +The UI changed between your snapshot and action, or the element could not be re-identified. Run `snapshot` again and use the new refs. If the requested `snapshot_id` does not exist, the command returns `SNAPSHOT_NOT_FOUND`. + +### POLICY_DENIED + +``` +"code": "POLICY_DENIED" +``` + +The semantic AX path could not complete and a physical/headed path was blocked by policy. Use an explicit physical command only when that is intended, for example `focus`, `press`, `hover`, `drag`, or `mouse-click`. ### ACTION_FAILED @@ -182,8 +213,8 @@ The accessibility action was rejected. This can happen when: **Try:** 1. Check `is @ref --property enabled` first -2. Try coordinate-based click: get bounds with `get @ref --property bounds`, then `mouse-click --xy x,y` -3. Use keyboard: `focus @ref` then `press return` +2. If physical interaction is intended, get bounds with `get @ref --property bounds`, then use `mouse-click --xy x,y` +3. Use keyboard explicitly: `focus @ref` then `press return` ### APP_NOT_FOUND @@ -200,15 +231,17 @@ Large apps (Xcode, Safari with many tabs) can have deep trees. - Use `-i` to filter to interactive elements only - Use `--max-depth 5` to limit depth - Use `--compact` to remove empty structural nodes +- Use `--skeleton` for dense apps, then `snapshot --root @ref --snapshot ` to drill down - Target a specific window with `--window-id` - Use `find` instead of full snapshot when you know what you're looking for ### Context Menu Doesn't Appear -After `right-click @ref`, if no menu appears: +After `right-click @ref`, inspect `menu` first. If it is absent and `menu_probe.ok` is `false`: 1. The element may not support context menus -2. The app may need to be focused first: `focus-window --app "App"` before right-clicking -3. Try `mouse-click --xy x,y --button right` with coordinates from `get @ref --property bounds` +2. If the target is a combo box or menu button, use `select @ref "Option"` instead +3. Run `list-surfaces --app "App"` to confirm whether a menu surface exists +4. If physical interaction is intended, try `mouse-click --xy x,y --button right` with coordinates from `get @ref --property bounds` ## macOS-Specific Behavior diff --git a/skills/agent-desktop/references/workflows.md b/skills/agent-desktop/references/workflows.md index c50c68a..e943981 100644 --- a/skills/agent-desktop/references/workflows.md +++ b/skills/agent-desktop/references/workflows.md @@ -13,6 +13,8 @@ agent-desktop permissions --request # Then: System Settings > Privacy & Security > Accessibility > enable your terminal ``` +For screenshots, also grant Screen Recording. `permissions` reports `accessibility`, `screen_recording`, and `automation` separately. + ## Pattern: Progressive Skeleton Traversal (Default for Dense Apps) The recommended approach for Electron apps (Slack, VS Code, Discord) and any app with 50+ interactive elements. Reduces token consumption 78-96%. @@ -20,6 +22,7 @@ The recommended approach for Electron apps (Slack, VS Code, Discord) and any app ```bash # 1. Get skeleton overview — shallow 3-level map with children_count hints agent-desktop snapshot --skeleton --app "Slack" -i --compact +# Keep snapshot_id = s8f... # Output shows regions like: # @e1 = group "Workspaces" (children_count: 4) # @e2 = group "Channels" (children_count: 42) @@ -27,19 +30,19 @@ agent-desktop snapshot --skeleton --app "Slack" -i --compact # @e4 = button "New Message" ← interactive elements at top levels still get refs # 2. Identify the region you need and drill into it -agent-desktop snapshot --root @e2 -i --compact +agent-desktop snapshot --root @e2 --snapshot s8f... -i --compact # Now you see all 42 children inside "Channels" with full refs # 3. Act on an element found in the drill-down -agent-desktop click @e18 # Click "general" channel +agent-desktop click @e18 --snapshot s8f... # Click "general" channel # 4. Re-drill the same or a different region to verify / continue -agent-desktop snapshot --root @e3 -i --compact +agent-desktop snapshot --root @e3 --snapshot s8f... -i --compact # Scoped invalidation: only @e3's previous refs are replaced # @e2's drill-down refs and the skeleton refs are preserved # 5. Drill into another region as needed — refs accumulate -agent-desktop snapshot --root @e1 -i --compact +agent-desktop snapshot --root @e1 --snapshot s8f... -i --compact # Now you have refs from skeleton + @e2 drill + @e3 drill + @e1 drill ``` @@ -55,21 +58,22 @@ agent-desktop snapshot --root @e1 -i --compact ```bash # For simple apps, full snapshot is fine agent-desktop snapshot --app "System Settings" -i +# Keep snapshot_id = s8f... # For dense apps, use skeleton first to find the form region, then drill # agent-desktop snapshot --skeleton --app "System Settings" -i --compact -# agent-desktop snapshot --root @e5 -i --compact +# agent-desktop snapshot --root @e5 --snapshot s8f... -i --compact # Found: @e3 = "Computer Name" textfield, @e5 = "Local Hostname" textfield # Clear and fill each field -agent-desktop clear @e3 -agent-desktop type @e3 "My MacBook Pro" -agent-desktop clear @e5 -agent-desktop type @e5 "my-macbook-pro" +agent-desktop clear @e3 --snapshot s8f... +agent-desktop type @e3 --snapshot s8f... "My MacBook Pro" +agent-desktop clear @e5 --snapshot s8f... +agent-desktop type @e5 --snapshot s8f... "my-macbook-pro" # Click the save/apply button -agent-desktop click @e8 +agent-desktop click @e8 --snapshot s8f... # Verify success — re-snapshot or re-drill agent-desktop snapshot --app "System Settings" -i @@ -82,12 +86,12 @@ agent-desktop snapshot --app "System Settings" -i agent-desktop snapshot --app "TextEdit" --surface menubar -i # Found: @e1 = "File" menuitem -agent-desktop click @e1 +agent-desktop click @e1 --snapshot s8f... agent-desktop wait --menu --app "TextEdit" agent-desktop snapshot --app "TextEdit" --surface menu -i # Found: @e5 = "Save As..." menuitem -agent-desktop click @e5 +agent-desktop click @e5 --snapshot s9a... # 2. Wait for the dialog, then snapshot the SHEET surface (not the full window) agent-desktop wait --window "Save" @@ -97,19 +101,16 @@ agent-desktop snapshot --app "TextEdit" --surface sheet -i ## Pattern: Right-Click Context Menu ```bash -# 1. Right-click the target element +# 1. Right-click the target element. Success means a menu surface was verified. agent-desktop right-click @e3 -# 2. Wait for context menu to appear -agent-desktop wait --menu --app "Finder" --timeout 3000 - -# 3. Snapshot the menu surface +# 2. Use the returned menu tree, or snapshot the menu surface if you need a fresh read. agent-desktop snapshot --app "Finder" --surface menu -i -# 4. Click the desired menu item -agent-desktop click @e7 +# 3. Click the desired menu item +agent-desktop click @e7 --snapshot s8f... -# 5. Wait for menu to close +# 4. Wait for menu to close agent-desktop wait --menu-closed --app "Finder" --timeout 2000 ``` @@ -124,10 +125,10 @@ agent-desktop snapshot --app "TextEdit" --surface sheet -i # For alerts: --surface alert | For popovers: --surface popover # Fill dialog fields -agent-desktop type @e2 "my-document.txt" +agent-desktop type @e2 --snapshot s8f... "my-document.txt" # Click OK/Save -agent-desktop click @e5 +agent-desktop click @e5 --snapshot s8f... # After dialog closes, snapshot the window again for fresh refs agent-desktop snapshot --app "TextEdit" -i @@ -143,24 +144,24 @@ agent-desktop snapshot --skeleton --app "App" -i --compact # Found: @e2 = group "Content" (children_count: 200) # 2. Drill into the region to get a scroll area ref -agent-desktop snapshot --root @e2 -i --compact +agent-desktop snapshot --root @e2 --snapshot s8f... -i --compact # Found: @e8 = scroll area # 3. Scroll and search in a loop -agent-desktop scroll @e8 --direction down --amount 5 +agent-desktop scroll @e8 --snapshot s8f... --direction down --amount 5 agent-desktop find --app "App" --name "Target Item" # If no matches, scroll again -agent-desktop scroll @e8 --direction down --amount 5 +agent-desktop scroll @e8 --snapshot s8f... --direction down --amount 5 agent-desktop find --app "App" --name "Target Item" # Found: @e14 = "Target Item" -agent-desktop click @e14 +agent-desktop click @e14 --snapshot s9a... ``` ## Pattern: Tab Through Fields ```bash # For sequential form filling without needing refs for each field: -agent-desktop click @e1 # Focus first field +agent-desktop focus @e1 # Explicit focus change agent-desktop type @e1 "value1" agent-desktop press tab # Now in next field — type directly since focus moved @@ -175,7 +176,7 @@ agent-desktop type @e3 "value3" # Or snapshot again to get new refs agent-desktop get @e5 --property value # Option B: Copy via keyboard -agent-desktop click @e5 +agent-desktop focus @e5 agent-desktop press cmd+a agent-desktop press cmd+c agent-desktop clipboard-get @@ -198,13 +199,13 @@ agent-desktop drag --from @e3 --to-xy 500,400 --duration 500 ```bash # After triggering a long operation: -agent-desktop click @e5 # "Download" button +agent-desktop click @e5 --snapshot s8f... # "Download" button # Wait for completion text agent-desktop wait --text "Download complete" --app "App" --timeout 30000 # Or wait for a specific element to appear -agent-desktop wait --element @e10 --timeout 10000 +agent-desktop wait --element @e10 --snapshot --timeout 10000 ``` ## Pattern: Launch, Automate, Close @@ -218,7 +219,7 @@ agent-desktop snapshot --app "Calculator" -i # Dense app → skeleton first # agent-desktop launch "Slack" # agent-desktop snapshot --skeleton --app "Slack" -i --compact -# agent-desktop snapshot --root @e2 -i --compact +# agent-desktop snapshot --root @e2 --snapshot s8f... -i --compact # ... perform automation ... @@ -257,9 +258,9 @@ agent-desktop uncheck @e6 # No-op if already unchecked ```bash # Run multiple commands atomically agent-desktop batch '[ - {"command":"click","args":{"ref_id":"@e1"}}, + {"command":"click","args":{"ref_id":"@e1","snapshot":"s8f..."}}, {"command":"wait","args":{"ms":200}}, - {"command":"type","args":{"ref_id":"@e2","text":"hello"}}, + {"command":"type","args":{"ref_id":"@e2","snapshot":"s8f...","text":"hello"}}, {"command":"press","args":{"combo":"return"}} ]' --stop-on-error ``` @@ -275,3 +276,4 @@ agent-desktop batch '[ 7. **Assuming UI stability.** Re-drill the affected region after every action that could change the UI. 8. **Snapshotting the full window when an overlay is open.** Use `--surface sheet/alert/popover/menu` instead. Never `--skeleton` for surfaces — they're already focused. 9. **Re-snapshotting everything after one action.** Use scoped re-drill (`--root @ref`) to refresh only the affected region. Other refs stay valid. +10. **Relying on implicit focus or cursor movement.** Non-mouse ref commands use semantic paths and block silent physical/headed paths. diff --git a/src/batch.rs b/src/batch.rs new file mode 100644 index 0000000..daa4bc4 --- /dev/null +++ b/src/batch.rs @@ -0,0 +1,309 @@ +use agent_desktop_core::{ + adapter::PlatformAdapter, + commands::batch::BatchCommand, + error::AppError, + output::{ErrorPayload, ENVELOPE_VERSION}, + PermissionReport, +}; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use serde_json::{json, Map, Value}; + +use crate::{ + cli::Commands, + cli_args_skills::{SkillsAction, SkillsArgs, SkillsGetArgs}, + cli_args_system::BatchArgs, +}; + +pub(crate) fn execute( + args: BatchArgs, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, +) -> Result { + let commands = agent_desktop_core::commands::batch::parse_commands(&args.commands_json)?; + let mut results = Vec::new(); + + for item in commands { + let command = item.command.clone(); + let result = parse_command(item).and_then(|typed| { + crate::command_policy::preflight(&typed, permission_report)?; + crate::dispatch::dispatch(typed, adapter, permission_report) + }); + let ok = result.is_ok(); + results.push(batch_entry(&command, result)); + if !ok && args.stop_on_error { + break; + } + } + + Ok(json!({ "results": results })) +} + +pub(crate) fn parse_command(item: BatchCommand) -> Result { + let command = item.command.as_str(); + match command { + "snapshot" => decode(command, item.args).map(Commands::Snapshot), + "find" => decode(command, item.args).map(Commands::Find), + "screenshot" => decode(command, item.args).map(Commands::Screenshot), + "get" => decode(command, item.args).map(Commands::Get), + "is" => decode(command, item.args).map(Commands::Is), + "click" => decode(command, item.args).map(Commands::Click), + "double-click" => decode(command, item.args).map(Commands::DoubleClick), + "triple-click" => decode(command, item.args).map(Commands::TripleClick), + "right-click" => decode(command, item.args).map(Commands::RightClick), + "type" => decode(command, item.args).map(Commands::Type), + "set-value" => decode(command, item.args).map(Commands::SetValue), + "clear" => decode(command, item.args).map(Commands::Clear), + "focus" => decode(command, item.args).map(Commands::Focus), + "select" => decode(command, item.args).map(Commands::Select), + "toggle" => decode(command, item.args).map(Commands::Toggle), + "check" => decode(command, item.args).map(Commands::Check), + "uncheck" => decode(command, item.args).map(Commands::Uncheck), + "expand" => decode(command, item.args).map(Commands::Expand), + "collapse" => decode(command, item.args).map(Commands::Collapse), + "scroll" => decode(command, item.args).map(Commands::Scroll), + "scroll-to" => decode(command, item.args).map(Commands::ScrollTo), + "press" => decode(command, item.args).map(Commands::Press), + "key-down" => decode(command, item.args).map(Commands::KeyDown), + "key-up" => decode(command, item.args).map(Commands::KeyUp), + "hover" => decode(command, item.args).map(Commands::Hover), + "drag" => decode(command, item.args).map(Commands::Drag), + "mouse-move" => decode(command, item.args).map(Commands::MouseMove), + "mouse-click" => decode(command, item.args).map(Commands::MouseClick), + "mouse-down" => decode(command, item.args).map(Commands::MouseDown), + "mouse-up" => decode(command, item.args).map(Commands::MouseUp), + "launch" => decode(command, item.args).map(Commands::Launch), + "close-app" => decode(command, item.args).map(Commands::CloseApp), + "list-windows" => decode(command, item.args).map(Commands::ListWindows), + "list-apps" => decode(command, item.args).map(Commands::ListApps), + "focus-window" => decode(command, item.args).map(Commands::FocusWindow), + "resize-window" => decode(command, item.args).map(Commands::ResizeWindow), + "move-window" => decode(command, item.args).map(Commands::MoveWindow), + "minimize" => decode(command, item.args).map(Commands::Minimize), + "maximize" => decode(command, item.args).map(Commands::Maximize), + "restore" => decode(command, item.args).map(Commands::Restore), + "list-surfaces" => decode(command, item.args).map(Commands::ListSurfaces), + "list-notifications" => decode(command, item.args).map(Commands::ListNotifications), + "dismiss-notification" => decode(command, item.args).map(Commands::DismissNotification), + "dismiss-all-notifications" => { + decode(command, item.args).map(Commands::DismissAllNotifications) + } + "notification-action" => decode(command, item.args).map(Commands::NotificationAction), + "clipboard-get" => no_args(command, item.args).map(|()| Commands::ClipboardGet), + "clipboard-set" => decode(command, item.args).map(Commands::ClipboardSet), + "clipboard-clear" => no_args(command, item.args).map(|()| Commands::ClipboardClear), + "wait" => decode(command, item.args).map(Commands::Wait), + "status" => no_args(command, item.args).map(|()| Commands::Status), + "permissions" => decode(command, item.args).map(Commands::Permissions), + "version" => decode(command, item.args).map(Commands::Version), + "skills" => parse_skills(item.args).map(Commands::Skills), + "batch" => Err(AppError::invalid_input_with_suggestion( + "Batch commands cannot be nested", + "Flatten nested batches into one top-level batch array", + )), + other => Err(AppError::invalid_input(format!( + "Unknown batch command '{other}'" + ))), + } +} + +fn batch_entry(command: &str, result: Result) -> Value { + match result { + Ok(data) => { + json!({ "version": ENVELOPE_VERSION, "ok": true, "command": command, "data": data }) + } + Err(err) => { + json!({ "version": ENVELOPE_VERSION, "ok": false, "command": command, "error": error_payload(err) }) + } + } +} + +fn error_payload(err: AppError) -> ErrorPayload { + let mut payload = ErrorPayload::new(err.code(), err.to_string()); + if let Some(suggestion) = err.suggestion() { + payload = payload.with_suggestion(suggestion); + } + if let AppError::Adapter(adapter_error) = err { + payload.platform_detail = adapter_error.platform_detail; + } + payload +} + +fn decode(command: &str, args: Value) -> Result +where + T: DeserializeOwned, +{ + serde_json::from_value(args_or_empty(args)).map_err(|e| { + AppError::invalid_input_with_suggestion( + format!("Invalid batch args for '{command}': {e}"), + "Use the same argument names and value types as the matching CLI command", + ) + }) +} + +fn no_args(command: &str, args: Value) -> Result<(), AppError> { + match args_or_empty(args) { + Value::Object(map) if map.is_empty() => Ok(()), + _ => Err(AppError::invalid_input_with_suggestion( + format!("Batch command '{command}' does not accept args"), + "Use an empty object or omit args for this command", + )), + } +} + +fn args_or_empty(args: Value) -> Value { + match args { + Value::Null => Value::Object(Map::new()), + other => other, + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct BatchSkillsArgs { + action: Option, + name: Option, + reference: Option, + #[serde(default)] + full: bool, +} + +fn parse_skills(args: Value) -> Result { + let args: BatchSkillsArgs = decode("skills", args)?; + let action = match args.action.as_deref() { + None if args.name.is_none() => Some(SkillsAction::List), + None | Some("get") => Some(SkillsAction::Get(SkillsGetArgs { + name: args + .name + .ok_or_else(|| AppError::invalid_input("Batch skills get requires 'name'"))?, + reference: args.reference, + full: args.full, + })), + Some("list") => Some(SkillsAction::List), + Some("path") => Some(SkillsAction::Path), + Some(other) => { + return Err(AppError::invalid_input(format!( + "Unknown skills action '{other}'" + ))) + } + }; + Ok(SkillsArgs { action }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cli_args::Surface; + use agent_desktop_core::{adapter::PlatformAdapter, PermissionReport}; + use clap::CommandFactory; + + struct NoopAdapter; + impl PlatformAdapter for NoopAdapter {} + + fn item(command: &str, args: Value) -> BatchCommand { + BatchCommand { + command: command.to_string(), + args, + } + } + + #[test] + fn parses_ref_command_into_cli_enum() { + let command = parse_command(item("click", serde_json::json!({ "ref_id": "@e1" }))) + .expect("click parses"); + + match command { + Commands::Click(args) => { + assert_eq!(args.ref_id, "@e1"); + assert_eq!(args.snapshot_id, None); + } + other => panic!("unexpected command: {other:?}"), + } + } + + #[test] + fn applies_cli_defaults_during_batch_decode() { + let command = + parse_command(item("snapshot", serde_json::json!({}))).expect("snapshot parses"); + + match command { + Commands::Snapshot(args) => { + assert_eq!(args.max_depth, 10); + assert!(!args.interactive_only); + assert!(matches!(args.surface, Surface::Window)); + } + other => panic!("unexpected command: {other:?}"), + } + } + + #[test] + fn rejects_unknown_batch_args() { + let err = parse_command(item( + "click", + serde_json::json!({ "ref_id": "@e1", "x": 1 }), + )) + .expect_err("unknown field is rejected"); + + assert_eq!(err.code(), "INVALID_ARGS"); + } + + #[test] + fn stop_on_error_halts_after_first_failure() { + let args = BatchArgs { + commands_json: serde_json::json!([ + {"command": "missing", "args": {}}, + {"command": "version", "args": {"json": true}} + ]) + .to_string(), + stop_on_error: true, + }; + + let value = execute(args, &NoopAdapter, &PermissionReport::default()).unwrap(); + let results = value["results"].as_array().unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0]["ok"], false); + assert_eq!(results[0]["version"], ENVELOPE_VERSION); + assert_eq!(results[0]["command"], "missing"); + assert_eq!(results[0]["error"]["code"], "INVALID_ARGS"); + assert!(results[0]["error"]["message"] + .as_str() + .unwrap() + .contains("Unknown batch command")); + } + + #[test] + fn no_args_rejection_has_suggestion() { + let err = parse_command(item("status", serde_json::json!({"x": 1}))) + .expect_err("status rejects args"); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(err.suggestion().is_some()); + } + + #[test] + fn nested_batch_rejection_has_suggestion() { + let err = parse_command(item("batch", serde_json::json!({}))).expect_err("batch rejected"); + + assert_eq!(err.code(), "INVALID_ARGS"); + assert!(err.suggestion().is_some()); + } + + #[test] + fn every_cli_subcommand_is_known_to_batch_parser() { + for subcommand in crate::cli::Cli::command().get_subcommands() { + let name = subcommand.get_name(); + let result = parse_command(item(name, serde_json::json!({}))); + if name == "batch" { + assert!(result.unwrap_err().to_string().contains("nested")); + continue; + } + if let Err(err) = result { + assert!( + !err.to_string().contains("Unknown batch command"), + "{name} is missing from batch parser" + ); + } + } + } +} diff --git a/src/batch_dispatch.rs b/src/batch_dispatch.rs deleted file mode 100644 index 7858554..0000000 --- a/src/batch_dispatch.rs +++ /dev/null @@ -1,348 +0,0 @@ -use agent_desktop_core::{ - commands::{ - check, clear, click, collapse, double_click, drag, expand, find, focus, get, helpers, - hover, is_check, key_down, key_up, mouse_click, mouse_down, mouse_move, mouse_up, press, - right_click, screenshot, scroll, scroll_to, select, set_value, snapshot, toggle, - triple_click, type_text, uncheck, - }, - error::AppError, -}; -use serde_json::Value; - -use crate::dispatch::{parse_direction, parse_mouse_button, parse_xy}; - -pub fn dispatch_batch_command( - command: &str, - args: Value, - adapter: &dyn agent_desktop_core::adapter::PlatformAdapter, -) -> Result { - match command { - "snapshot" => snapshot::execute( - snapshot::SnapshotArgs { - app: str_field(&args, "app"), - window_id: str_field(&args, "window_id"), - max_depth: args - .get("max_depth") - .and_then(|v| v.as_u64()) - .map(|v| v as u8) - .unwrap_or(10), - include_bounds: args - .get("include_bounds") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - interactive_only: args - .get("interactive_only") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - compact: args - .get("compact") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - surface: parse_batch_surface(args.get("surface").and_then(|v| v.as_str())), - skeleton: args - .get("skeleton") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - root_ref: str_field(&args, "root"), - }, - adapter, - ), - - "find" => find::execute( - find::FindArgs { - app: str_field(&args, "app"), - role: str_field(&args, "role"), - name: str_field(&args, "name"), - value: str_field(&args, "value"), - text: str_field(&args, "text"), - count: args.get("count").and_then(|v| v.as_bool()).unwrap_or(false), - first: args.get("first").and_then(|v| v.as_bool()).unwrap_or(false), - last: args.get("last").and_then(|v| v.as_bool()).unwrap_or(false), - nth: args.get("nth").and_then(|v| v.as_u64()).map(|v| v as usize), - }, - adapter, - ), - - "screenshot" => screenshot::execute( - screenshot::ScreenshotArgs { - app: str_field(&args, "app"), - window_id: str_field(&args, "window_id"), - output_path: str_field(&args, "output_path").map(std::path::PathBuf::from), - }, - adapter, - ), - - "get" => get::execute( - get::GetArgs { - ref_id: req_str(&args, "ref_id")?, - property: crate::dispatch::parse_get_property( - args.get("property") - .and_then(|v| v.as_str()) - .unwrap_or("text"), - )?, - }, - adapter, - ), - - "is" => is_check::execute( - is_check::IsArgs { - ref_id: req_str(&args, "ref_id")?, - property: crate::dispatch::parse_is_property( - args.get("property") - .and_then(|v| v.as_str()) - .unwrap_or("visible"), - )?, - }, - adapter, - ), - - "click" => click::execute( - click::ClickArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "double-click" => double_click::execute( - double_click::DoubleClickArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "triple-click" => triple_click::execute( - triple_click::TripleClickArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "right-click" => right_click::execute( - right_click::RightClickArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "focus" => focus::execute( - helpers::RefArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "toggle" => toggle::execute( - helpers::RefArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "check" => check::execute( - check::CheckArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "uncheck" => uncheck::execute( - uncheck::UncheckArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "expand" => expand::execute( - helpers::RefArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "collapse" => collapse::execute( - helpers::RefArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "clear" => clear::execute( - clear::ClearArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - "scroll-to" => scroll_to::execute( - scroll_to::ScrollToArgs { - ref_id: req_str(&args, "ref_id")?, - }, - adapter, - ), - - "type" => type_text::execute( - type_text::TypeArgs { - ref_id: req_str(&args, "ref_id")?, - text: req_str(&args, "text")?, - }, - adapter, - ), - - "set-value" => set_value::execute( - set_value::SetValueArgs { - ref_id: req_str(&args, "ref_id")?, - value: req_str(&args, "value")?, - }, - adapter, - ), - - "select" => select::execute( - select::SelectArgs { - ref_id: req_str(&args, "ref_id")?, - value: req_str(&args, "value")?, - }, - adapter, - ), - - "scroll" => scroll::execute( - scroll::ScrollArgs { - ref_id: req_str(&args, "ref_id")?, - direction: parse_direction( - args.get("direction") - .and_then(|v| v.as_str()) - .unwrap_or("down"), - )?, - amount: args - .get("amount") - .and_then(|v| v.as_u64()) - .map(|v| v as u32) - .unwrap_or(3), - }, - adapter, - ), - - "press" => press::execute( - press::PressArgs { - combo: req_str(&args, "combo")?, - app: str_field(&args, "app"), - }, - adapter, - ), - - "key-down" => key_down::execute( - key_down::KeyDownArgs { - combo: req_str(&args, "combo")?, - }, - adapter, - ), - - "key-up" => key_up::execute( - key_up::KeyUpArgs { - combo: req_str(&args, "combo")?, - }, - adapter, - ), - - "hover" => { - let xy = str_field(&args, "xy").map(|s| parse_xy(&s)).transpose()?; - hover::execute( - hover::HoverArgs { - ref_id: str_field(&args, "ref_id"), - xy, - duration_ms: args.get("duration_ms").and_then(|v| v.as_u64()), - }, - adapter, - ) - } - - "drag" => { - let from_xy = str_field(&args, "from_xy") - .map(|s| parse_xy(&s)) - .transpose()?; - let to_xy = str_field(&args, "to_xy") - .map(|s| parse_xy(&s)) - .transpose()?; - drag::execute( - drag::DragArgs { - from_ref: str_field(&args, "from"), - from_xy, - to_ref: str_field(&args, "to"), - to_xy, - duration_ms: args.get("duration_ms").and_then(|v| v.as_u64()), - }, - adapter, - ) - } - - "mouse-move" => { - let xy = req_str(&args, "xy")?; - let (x, y) = parse_xy(&xy)?; - mouse_move::execute(mouse_move::MouseMoveArgs { x, y }, adapter) - } - - "mouse-click" => { - let xy = req_str(&args, "xy")?; - let (x, y) = parse_xy(&xy)?; - let button = args - .get("button") - .and_then(|v| v.as_str()) - .unwrap_or("left"); - mouse_click::execute( - mouse_click::MouseClickArgs { - x, - y, - button: parse_mouse_button(button)?, - count: args.get("count").and_then(|v| v.as_u64()).unwrap_or(1) as u32, - }, - adapter, - ) - } - - "mouse-down" => { - let xy = req_str(&args, "xy")?; - let (x, y) = parse_xy(&xy)?; - let button = args - .get("button") - .and_then(|v| v.as_str()) - .unwrap_or("left"); - mouse_down::execute( - mouse_down::MouseDownArgs { - x, - y, - button: parse_mouse_button(button)?, - }, - adapter, - ) - } - - "mouse-up" => { - let xy = req_str(&args, "xy")?; - let (x, y) = parse_xy(&xy)?; - let button = args - .get("button") - .and_then(|v| v.as_str()) - .unwrap_or("left"); - mouse_up::execute( - mouse_up::MouseUpArgs { - x, - y, - button: parse_mouse_button(button)?, - }, - adapter, - ) - } - - other => crate::batch_dispatch_ext::dispatch(other, args, adapter), - } -} - -pub(crate) fn str_field(v: &Value, key: &str) -> Option { - v.get(key).and_then(|v| v.as_str()).map(String::from) -} - -pub(crate) fn req_str(v: &Value, key: &str) -> Result { - str_field(v, key) - .ok_or_else(|| AppError::invalid_input(format!("Batch: missing required field '{key}'"))) -} - -fn parse_batch_surface(s: Option<&str>) -> agent_desktop_core::adapter::SnapshotSurface { - use agent_desktop_core::adapter::SnapshotSurface; - match s { - Some("menu") => SnapshotSurface::Menu, - Some("menubar") => SnapshotSurface::Menubar, - Some("sheet") => SnapshotSurface::Sheet, - Some("popover") => SnapshotSurface::Popover, - Some("alert") => SnapshotSurface::Alert, - Some("focused") => SnapshotSurface::Focused, - _ => SnapshotSurface::Window, - } -} diff --git a/src/batch_dispatch_ext.rs b/src/batch_dispatch_ext.rs deleted file mode 100644 index a609c75..0000000 --- a/src/batch_dispatch_ext.rs +++ /dev/null @@ -1,208 +0,0 @@ -use agent_desktop_core::{ - commands::{ - clipboard_clear, clipboard_get, clipboard_set, close_app, dismiss_all_notifications, - dismiss_notification, focus_window, launch, list_apps, list_notifications, list_surfaces, - list_windows, maximize, minimize, move_window, notification_action, permissions, - resize_window, restore, status, version, wait, - }, - error::AppError, -}; -use serde_json::Value; - -use crate::batch_dispatch::{req_str, str_field}; - -pub fn dispatch( - command: &str, - args: Value, - adapter: &dyn agent_desktop_core::adapter::PlatformAdapter, -) -> Result { - match command { - "launch" => launch::execute( - launch::LaunchArgs { - app: req_str(&args, "app")?, - timeout_ms: args - .get("timeout") - .and_then(|v| v.as_u64()) - .unwrap_or(30000), - }, - adapter, - ), - - "close-app" => close_app::execute( - close_app::CloseAppArgs { - app: req_str(&args, "app")?, - force: args.get("force").and_then(|v| v.as_bool()).unwrap_or(false), - }, - adapter, - ), - - "list-windows" => list_windows::execute( - list_windows::ListWindowsArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "list-apps" => list_apps::execute(adapter), - - "focus-window" => focus_window::execute( - focus_window::FocusWindowArgs { - window_id: str_field(&args, "window_id"), - app: str_field(&args, "app"), - title: str_field(&args, "title"), - }, - adapter, - ), - - "resize-window" => resize_window::execute( - resize_window::ResizeWindowArgs { - app: str_field(&args, "app"), - width: args.get("width").and_then(|v| v.as_f64()).unwrap_or(800.0), - height: args.get("height").and_then(|v| v.as_f64()).unwrap_or(600.0), - }, - adapter, - ), - - "move-window" => move_window::execute( - move_window::MoveWindowArgs { - app: str_field(&args, "app"), - x: args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0), - y: args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0), - }, - adapter, - ), - - "minimize" => minimize::execute( - minimize::MinimizeArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "maximize" => maximize::execute( - maximize::MaximizeArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "restore" => restore::execute( - restore::RestoreArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "list-notifications" => list_notifications::execute( - list_notifications::ListNotificationsArgs { - app: str_field(&args, "app"), - text: str_field(&args, "text"), - limit: args - .get("limit") - .and_then(|v| v.as_u64()) - .map(|v| v as usize), - }, - adapter, - ), - - "dismiss-notification" => { - let index = args - .get("index") - .and_then(|v| v.as_u64()) - .map(|v| v as usize) - .ok_or_else(|| AppError::invalid_input("Batch: missing required field 'index'"))?; - if index == 0 { - return Err(AppError::invalid_input("Index must be >= 1")); - } - dismiss_notification::execute( - dismiss_notification::DismissNotificationArgs { - index, - app: str_field(&args, "app"), - }, - adapter, - ) - } - - "dismiss-all-notifications" => dismiss_all_notifications::execute( - dismiss_all_notifications::DismissAllNotificationsArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "notification-action" => { - let index = args - .get("index") - .and_then(|v| v.as_u64()) - .map(|v| v as usize) - .ok_or_else(|| AppError::invalid_input("Batch: missing required field 'index'"))?; - if index == 0 { - return Err(AppError::invalid_input("Index must be >= 1")); - } - notification_action::execute( - notification_action::NotificationActionArgs { - index, - action: req_str(&args, "action")?, - expected_app: str_field(&args, "expected_app"), - expected_title: str_field(&args, "expected_title"), - }, - adapter, - ) - } - - "clipboard-get" => clipboard_get::execute(adapter), - "clipboard-set" => clipboard_set::execute(req_str(&args, "text")?, adapter), - "clipboard-clear" => clipboard_clear::execute(adapter), - - "wait" => wait::execute( - wait::WaitArgs { - ms: args.get("ms").and_then(|v| v.as_u64()), - element: str_field(&args, "element"), - window: str_field(&args, "window"), - text: str_field(&args, "text"), - timeout_ms: args - .get("timeout_ms") - .and_then(|v| v.as_u64()) - .unwrap_or(30000), - menu: args.get("menu").and_then(|v| v.as_bool()).unwrap_or(false), - menu_closed: args - .get("menu_closed") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - notification: args - .get("notification") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - app: str_field(&args, "app"), - }, - adapter, - ), - - "list-surfaces" => list_surfaces::execute( - list_surfaces::ListSurfacesArgs { - app: str_field(&args, "app"), - }, - adapter, - ), - - "status" => status::execute(adapter), - - "permissions" => permissions::execute( - permissions::PermissionsArgs { - request: args - .get("request") - .and_then(|v| v.as_bool()) - .unwrap_or(false), - }, - adapter, - ), - - "version" => version::execute(version::VersionArgs { - json: args.get("json").and_then(|v| v.as_bool()).unwrap_or(false), - }), - - other => Err(AppError::invalid_input(format!( - "Unknown batch command '{other}'" - ))), - } -} diff --git a/src/cli.rs b/src/cli.rs index 94402ab..c71293a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,8 +1,22 @@ use clap::{Parser, Subcommand}; -pub use crate::cli_args::*; -pub use crate::cli_args_notifications::*; -pub use crate::cli_args_skills::*; +use crate::cli_args::{ + FindArgs, GetArgs, IsArgs, ListSurfacesArgs, RefArgs, ScreenshotArgs, SnapshotArgs, +}; +use crate::cli_args_actions::{ + DragCliArgs, HoverArgs, KeyComboArgs, MouseClickArgs, MouseMoveArgs, MousePointArgs, PressArgs, + ScrollArgs, SelectArgs, SetValueArgs, TypeArgs, +}; +use crate::cli_args_notifications::{ + DismissAllNotificationsCliArgs, DismissNotificationCliArgs, ListNotificationsCliArgs, + NotificationActionCliArgs, +}; +use crate::cli_args_skills::SkillsArgs; +use crate::cli_args_system::{ + AppRefArgs, BatchArgs, ClipboardSetArgs, CloseAppArgs, FocusWindowArgs, LaunchArgs, + ListAppsArgs, ListWindowsArgs, MoveWindowCliArgs, PermissionsArgs, ResizeWindowCliArgs, + VersionArgs, WaitArgs, +}; const BEFORE_HELP: &str = include_str!("help_before.txt"); const AFTER_HELP: &str = include_str!("help_after.txt"); @@ -42,13 +56,15 @@ pub enum Commands { Is(IsArgs), #[command(about = "Click element via accessibility press action")] Click(RefArgs), - #[command(about = "Double-click element")] + #[command(about = "Open element via AXOpen; physical double-click uses mouse-click")] DoubleClick(RefArgs), - #[command(about = "Triple-click element to select line or paragraph")] + #[command( + about = "Triple-click element; returns POLICY_DENIED when physical input is disabled" + )] TripleClick(RefArgs), - #[command(about = "Right-click element to open context menu")] + #[command(about = "Right-click and include menu/menu_probe details when available")] RightClick(RefArgs), - #[command(about = "Focus element and type text")] + #[command(about = "Insert text into a text target")] Type(TypeArgs), #[command(about = "Set element value directly via accessibility attribute")] SetValue(SetValueArgs), @@ -96,9 +112,9 @@ pub enum Commands { CloseApp(CloseAppArgs), #[command(about = "List all visible windows (--app to filter by application)")] ListWindows(ListWindowsArgs), - #[command(about = "List all running GUI applications")] - ListApps, - #[command(about = "Bring a window to front (--app, --title, or --window-id)")] + #[command(about = "List all running GUI applications (--app to filter)")] + ListApps(ListAppsArgs), + #[command(about = "Bring a window to front and confirm OS focus")] FocusWindow(FocusWindowArgs), #[command(about = "Resize application window")] ResizeWindow(ResizeWindowCliArgs), @@ -130,7 +146,9 @@ pub enum Commands { Wait(WaitArgs), #[command(about = "Show adapter health, platform info, and permission state")] Status, - #[command(about = "Check accessibility permission status (--request to prompt system dialog)")] + #[command( + about = "Check nested permission states: accessibility/screen_recording/automation each return {state,...}" + )] Permissions(PermissionsArgs), #[command(about = "Show version (--json for machine-readable output)")] Version(VersionArgs), @@ -176,7 +194,7 @@ impl Commands { Self::Launch(_) => "launch", Self::CloseApp(_) => "close-app", Self::ListWindows(_) => "list-windows", - Self::ListApps => "list-apps", + Self::ListApps(_) => "list-apps", Self::FocusWindow(_) => "focus-window", Self::ResizeWindow(_) => "resize-window", Self::MoveWindow(_) => "move-window", diff --git a/src/cli_args.rs b/src/cli_args.rs index 8ac60f0..4283b5e 100644 --- a/src/cli_args.rs +++ b/src/cli_args.rs @@ -1,6 +1,20 @@ use clap::{Parser, ValueEnum}; +use serde::Deserialize; -#[derive(ValueEnum, Clone, Debug, Default)] +fn default_max_depth() -> u8 { + 10 +} + +fn default_get_property() -> String { + "text".to_string() +} + +fn default_is_property() -> String { + "visible".to_string() +} + +#[derive(ValueEnum, Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum Surface { #[default] Window, @@ -27,7 +41,8 @@ impl Surface { } } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SnapshotArgs { #[arg(long, help = "Filter to application by name")] pub app: Option, @@ -38,28 +53,42 @@ pub struct SnapshotArgs { )] pub window_id: Option, #[arg(long, default_value = "10", help = "Maximum tree depth")] + #[serde(default = "default_max_depth")] pub max_depth: u8, #[arg(long, help = "Include element bounds (x, y, width, height)")] + #[serde(default)] pub include_bounds: bool, #[arg(long, short = 'i', help = "Include interactive elements only")] + #[serde(default)] pub interactive_only: bool, #[arg( long, help = "Collapse single-child unnamed nodes to reduce tree depth" )] + #[serde(default)] pub compact: bool, - #[arg(long, value_enum, default_value_t = Surface::Window, help = "Surface to snapshot")] + #[arg( + long, + value_enum, + default_value_t = Surface::Window, + help = "Surface to snapshot" + )] + #[serde(default)] pub surface: Surface, #[arg( long, help = "Shallow overview with children_count on truncated containers" )] + #[serde(default)] pub skeleton: bool, #[arg(long, help = "Start traversal from this ref instead of window root")] pub root: Option, + #[arg(long, help = "Snapshot ID to use when resolving --root")] + pub snapshot: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct FindArgs { #[arg(long, help = "Filter to application by name")] pub app: Option, @@ -74,17 +103,43 @@ pub struct FindArgs { pub value: Option, #[arg(long, help = "Match by text in name, value, title, or description")] pub text: Option, - #[arg(long, help = "Return match count only")] + #[arg( + long, + help = "Return match count only", + conflicts_with_all = ["first", "last", "nth", "limit"] + )] + #[serde(default)] pub count: bool, - #[arg(long, help = "Return first match only")] + #[arg( + long, + help = "Return first match only", + conflicts_with_all = ["count", "last", "nth", "limit"] + )] + #[serde(default)] pub first: bool, - #[arg(long, help = "Return last match only")] + #[arg( + long, + help = "Return last match only", + conflicts_with_all = ["count", "first", "nth", "limit"] + )] + #[serde(default)] pub last: bool, - #[arg(long, help = "Return Nth match (0-indexed)")] + #[arg( + long, + help = "Return Nth match (0-indexed)", + conflicts_with_all = ["count", "first", "last", "limit"] + )] pub nth: Option, + #[arg( + long, + help = "Return at most N matches; defaults to 50 when omitted, use 0 for all", + conflicts_with_all = ["count", "first", "last", "nth"] + )] + pub limit: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ScreenshotArgs { #[arg(long, help = "Filter to application by name")] pub app: Option, @@ -98,273 +153,54 @@ pub struct ScreenshotArgs { pub output_path: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct GetArgs { #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, #[arg( long, default_value = "text", help = "Property: text, value, title, bounds, role, states" )] + #[serde(default = "default_get_property")] pub property: String, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct IsArgs { #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, #[arg( long, default_value = "visible", help = "State: visible, enabled, checked, focused, expanded" )] + #[serde(default = "default_is_property")] pub property: String, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct RefArgs { #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] pub ref_id: String, -} - -#[derive(Parser, Debug)] -pub struct TypeArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] - pub ref_id: String, - #[arg(value_name = "TEXT", allow_hyphen_values = true, help = "Text to type")] - pub text: String, -} - -#[derive(Parser, Debug)] -pub struct SetValueArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] - pub ref_id: String, #[arg( - value_name = "VALUE", - allow_hyphen_values = true, - help = "Value to set" + long = "snapshot", + help = "Snapshot ID returned by snapshot; omit to use latest" )] - pub value: String, + #[serde(rename = "snapshot", alias = "snapshot_id")] + pub snapshot_id: Option, } -#[derive(Parser, Debug)] -pub struct SelectArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] - pub ref_id: String, - #[arg(value_name = "VALUE", help = "Option to select")] - pub value: String, -} - -#[derive(Parser, Debug)] -pub struct ScrollArgs { - #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] - pub ref_id: String, - #[arg( - long, - default_value = "down", - help = "Direction: up, down, left, right" - )] - pub direction: String, - #[arg(long, default_value = "3", help = "Number of scroll units")] - pub amount: u32, -} - -#[derive(Parser, Debug)] -pub struct PressArgs { - #[arg( - value_name = "COMBO", - help = "Key combo: return, escape, cmd+c, shift+tab ..." - )] - pub combo: String, - #[arg(long, help = "Target application name (focuses app before pressing)")] - pub app: Option, -} - -#[derive(Parser, Debug)] -pub struct KeyComboArgs { - #[arg( - value_name = "COMBO", - help = "Key or modifier to hold/release: shift, cmd, ctrl ..." - )] - pub combo: String, -} - -#[derive(Parser, Debug)] -pub struct HoverArgs { - #[arg(value_name = "REF", help = "Element ref to hover over")] - pub ref_id: Option, - #[arg(long, help = "Absolute coordinates as x,y")] - pub xy: Option, - #[arg(long, help = "Hold hover position for N milliseconds")] - pub duration: Option, -} - -#[derive(Parser, Debug)] -pub struct DragCliArgs { - #[arg(long, help = "Source element ref")] - pub from: Option, - #[arg(long, name = "from-xy", help = "Source coordinates as x,y")] - pub from_xy: Option, - #[arg(long, help = "Destination element ref")] - pub to: Option, - #[arg(long, name = "to-xy", help = "Destination coordinates as x,y")] - pub to_xy: Option, - #[arg(long, help = "Drag duration in milliseconds")] - pub duration: Option, -} - -#[derive(Parser, Debug)] -pub struct MouseMoveArgs { - #[arg(long, help = "Absolute coordinates as x,y")] - pub xy: String, -} - -#[derive(Parser, Debug)] -pub struct MouseClickArgs { - #[arg(long, help = "Absolute coordinates as x,y")] - pub xy: String, - #[arg( - long, - default_value = "left", - help = "Mouse button: left, right, middle" - )] - pub button: String, - #[arg(long, default_value = "1", help = "Number of clicks")] - pub count: u32, -} - -#[derive(Parser, Debug)] -pub struct MousePointArgs { - #[arg(long, help = "Absolute coordinates as x,y")] - pub xy: String, - #[arg( - long, - default_value = "left", - help = "Mouse button: left, right, middle" - )] - pub button: String, -} - -#[derive(Parser, Debug)] -pub struct LaunchArgs { - #[arg(value_name = "APP", help = "Application name or bundle ID")] - pub app: String, - #[arg( - long, - default_value = "30000", - help = "Max time in ms to wait for the window to appear" - )] - pub timeout: u64, -} - -#[derive(Parser, Debug)] -pub struct CloseAppArgs { - #[arg(value_name = "APP", help = "Application name")] - pub app: String, - #[arg(long, help = "Force-kill the process instead of quitting gracefully")] - pub force: bool, -} - -#[derive(Parser, Debug)] -pub struct ListWindowsArgs { - #[arg(long, help = "Filter to application by name")] - pub app: Option, -} - -#[derive(Parser, Debug)] -pub struct FocusWindowArgs { - #[arg(long, name = "window-id", help = "Window ID from list-windows")] - pub window_id: Option, - #[arg(long, help = "Application name")] - pub app: Option, - #[arg(long, help = "Window title (partial match accepted)")] - pub title: Option, -} - -#[derive(Parser, Debug)] -pub struct ResizeWindowCliArgs { - #[arg(long, help = "Application name")] - pub app: Option, - #[arg(long, help = "New window width in pixels")] - pub width: f64, - #[arg(long, help = "New window height in pixels")] - pub height: f64, -} - -#[derive(Parser, Debug)] -pub struct MoveWindowCliArgs { - #[arg(long, help = "Application name")] - pub app: Option, - #[arg(long, help = "New window X position")] - pub x: f64, - #[arg(long, help = "New window Y position")] - pub y: f64, -} - -#[derive(Parser, Debug)] -pub struct AppRefArgs { - #[arg(long, help = "Application name")] - pub app: Option, -} - -#[derive(Parser, Debug)] -pub struct ClipboardSetArgs { - #[arg(value_name = "TEXT", help = "Text to write to the clipboard")] - pub text: String, -} - -#[derive(Parser, Debug)] -pub struct WaitArgs { - #[arg(value_name = "MS", help = "Milliseconds to pause")] - pub ms: Option, - #[arg(long, help = "Block until this element ref appears in the tree")] - pub element: Option, - #[arg(long, help = "Block until a window with this title appears")] - pub window: Option, - #[arg( - long, - help = "Block until text appears in the app's accessibility tree" - )] - pub text: Option, - #[arg( - long, - default_value = "30000", - help = "Timeout in milliseconds for element/window/text waits" - )] - pub timeout: u64, - #[arg(long, help = "Block until a context menu is open")] - pub menu: bool, - #[arg(long, help = "Block until the context menu is dismissed")] - pub menu_closed: bool, - #[arg(long, help = "Block until a new notification arrives")] - pub notification: bool, - #[arg(long, help = "Scope element, window, or text wait to this application")] - pub app: Option, -} - -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ListSurfacesArgs { #[arg(long, help = "Filter to application by name")] pub app: Option, } - -#[derive(Parser, Debug)] -pub struct PermissionsArgs { - #[arg(long, help = "Trigger the system accessibility permission dialog")] - pub request: bool, -} - -#[derive(Parser, Debug)] -pub struct VersionArgs { - #[arg(long, help = "Output version as JSON object")] - pub json: bool, -} - -#[derive(Parser, Debug)] -pub struct BatchArgs { - #[arg(value_name = "JSON", help = "JSON array of {command, args} objects")] - pub commands_json: String, - #[arg(long, help = "Halt the batch on the first failed command")] - pub stop_on_error: bool, -} diff --git a/src/cli_args_actions.rs b/src/cli_args_actions.rs new file mode 100644 index 0000000..4dbb08b --- /dev/null +++ b/src/cli_args_actions.rs @@ -0,0 +1,164 @@ +use clap::Parser; +use serde::Deserialize; + +fn default_scroll_amount() -> u32 { + 3 +} + +fn default_mouse_button() -> String { + "left".to_string() +} + +fn default_mouse_click_count() -> u32 { + 1 +} + +fn default_scroll_direction() -> String { + "down".to_string() +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TypeArgs { + #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg(value_name = "TEXT", allow_hyphen_values = true, help = "Text to type")] + pub text: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SetValueArgs { + #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg( + value_name = "VALUE", + allow_hyphen_values = true, + help = "Value to set" + )] + pub value: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SelectArgs { + #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg(value_name = "VALUE", help = "Option to select")] + pub value: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScrollArgs { + #[arg(value_name = "REF", help = "Element ref from snapshot (@e1, @e2 ...)")] + pub ref_id: String, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg( + long, + default_value = "down", + help = "Direction: up, down, left, right" + )] + #[serde(default = "default_scroll_direction")] + pub direction: String, + #[arg(long, default_value = "3", help = "Number of scroll units")] + #[serde(default = "default_scroll_amount")] + pub amount: u32, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PressArgs { + #[arg( + value_name = "COMBO", + help = "Key combo: return, escape, cmd+c, shift+tab ..." + )] + pub combo: String, + #[arg(long, help = "Target application name (focuses app before pressing)")] + pub app: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct KeyComboArgs { + #[arg( + value_name = "COMBO", + help = "Key or modifier to hold/release: shift, cmd, ctrl ..." + )] + pub combo: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HoverArgs { + #[arg(value_name = "REF", help = "Element ref to hover over")] + pub ref_id: Option, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg(long, help = "Absolute coordinates as x,y")] + pub xy: Option, + #[arg(long, help = "Hold hover position for N milliseconds")] + pub duration: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DragCliArgs { + #[arg(long, help = "Source element ref")] + pub from: Option, + #[arg(long, name = "from-xy", help = "Source coordinates as x,y")] + pub from_xy: Option, + #[arg(long, help = "Destination element ref")] + pub to: Option, + #[arg(long, name = "to-xy", help = "Destination coordinates as x,y")] + pub to_xy: Option, + #[arg(long, help = "Snapshot ID returned by snapshot; omit to use latest")] + pub snapshot: Option, + #[arg(long, help = "Drag duration in milliseconds")] + pub duration: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MouseMoveArgs { + #[arg(long, help = "Absolute coordinates as x,y")] + pub xy: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MouseClickArgs { + #[arg(long, help = "Absolute coordinates as x,y")] + pub xy: String, + #[arg( + long, + default_value = "left", + help = "Mouse button: left, right, middle" + )] + #[serde(default = "default_mouse_button")] + pub button: String, + #[arg(long, default_value = "1", help = "Number of clicks")] + #[serde(default = "default_mouse_click_count")] + pub count: u32, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MousePointArgs { + #[arg(long, help = "Absolute coordinates as x,y")] + pub xy: String, + #[arg( + long, + default_value = "left", + help = "Mouse button: left, right, middle" + )] + #[serde(default = "default_mouse_button")] + pub button: String, +} diff --git a/src/cli_args_notifications.rs b/src/cli_args_notifications.rs index c1a3e35..384bbad 100644 --- a/src/cli_args_notifications.rs +++ b/src/cli_args_notifications.rs @@ -1,6 +1,8 @@ use clap::Parser; +use serde::Deserialize; -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ListNotificationsCliArgs { #[arg(long, help = "Filter to notifications from this app")] pub app: Option, @@ -10,7 +12,8 @@ pub struct ListNotificationsCliArgs { pub limit: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DismissNotificationCliArgs { #[arg(value_name = "INDEX", help = "1-based notification index from list-notifications", value_parser = clap::value_parser!(u64).range(1..))] @@ -19,13 +22,15 @@ pub struct DismissNotificationCliArgs { pub app: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DismissAllNotificationsCliArgs { #[arg(long, help = "Only dismiss notifications from this app")] pub app: Option, } -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct NotificationActionCliArgs { #[arg(value_name = "INDEX", help = "1-based notification index from list-notifications", value_parser = clap::value_parser!(u64).range(1..))] diff --git a/src/cli_args_system.rs b/src/cli_args_system.rs new file mode 100644 index 0000000..9664159 --- /dev/null +++ b/src/cli_args_system.rs @@ -0,0 +1,158 @@ +use clap::Parser; +use serde::Deserialize; + +fn default_launch_timeout() -> u64 { + 30000 +} + +fn default_wait_timeout() -> u64 { + 30000 +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaunchArgs { + #[arg(value_name = "APP", help = "Application name or bundle ID")] + pub app: String, + #[arg( + long, + default_value = "30000", + help = "Max time in ms to wait for the window to appear" + )] + #[serde(default = "default_launch_timeout")] + pub timeout: u64, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CloseAppArgs { + #[arg(value_name = "APP", help = "Application name")] + pub app: String, + #[arg(long, help = "Force-kill the process instead of quitting gracefully")] + #[serde(default)] + pub force: bool, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListWindowsArgs { + #[arg(long, help = "Filter running apps by case-insensitive name substring")] + pub app: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListAppsArgs { + #[arg(long, help = "Filter to application by name")] + pub app: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FocusWindowArgs { + #[arg(long, name = "window-id", help = "Window ID from list-windows")] + pub window_id: Option, + #[arg(long, help = "Application name")] + pub app: Option, + #[arg(long, help = "Window title (partial match accepted)")] + pub title: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResizeWindowCliArgs { + #[arg(long, help = "Application name")] + pub app: Option, + #[arg(long, help = "New window width in pixels")] + pub width: f64, + #[arg(long, help = "New window height in pixels")] + pub height: f64, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MoveWindowCliArgs { + #[arg(long, help = "Application name")] + pub app: Option, + #[arg(long, help = "New window X position")] + pub x: f64, + #[arg(long, help = "New window Y position")] + pub y: f64, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppRefArgs { + #[arg(long, help = "Application name")] + pub app: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClipboardSetArgs { + #[arg(value_name = "TEXT", help = "Text to write to the clipboard")] + pub text: String, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WaitArgs { + #[arg(value_name = "MS", help = "Milliseconds to pause")] + pub ms: Option, + #[arg(long, help = "Block until this element ref appears in the tree")] + pub element: Option, + #[arg( + long, + help = "Snapshot ID returned by snapshot for --element waits; omit to use latest" + )] + pub snapshot: Option, + #[arg(long, help = "Block until a window with this title appears")] + pub window: Option, + #[arg( + long, + help = "Block until text appears in the app's accessibility tree; with --notification, filter notification text" + )] + pub text: Option, + #[arg( + long, + default_value = "30000", + help = "Timeout in milliseconds for element/window/text waits" + )] + #[serde(default = "default_wait_timeout")] + pub timeout: u64, + #[arg(long, help = "Block until a menu surface is open")] + #[serde(default)] + pub menu: bool, + #[arg(long, help = "Block until the menu surface is dismissed")] + #[serde(default)] + pub menu_closed: bool, + #[arg(long, help = "Block until a new notification arrives")] + #[serde(default)] + pub notification: bool, + #[arg(long, help = "Scope element, window, or text wait to this application")] + pub app: Option, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PermissionsArgs { + #[arg(long, help = "Trigger the system accessibility permission dialog")] + #[serde(default)] + pub request: bool, +} + +#[derive(Parser, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VersionArgs { + #[arg(long, help = "Output version as JSON object")] + #[serde(default)] + pub json: bool, +} + +#[derive(Parser, Debug)] +pub struct BatchArgs { + #[arg(value_name = "JSON", help = "JSON array of {command, args} objects")] + pub commands_json: String, + #[arg(long, help = "Halt the batch on the first failed command")] + pub stop_on_error: bool, +} diff --git a/src/command_policy.rs b/src/command_policy.rs new file mode 100644 index 0000000..e29ea06 --- /dev/null +++ b/src/command_policy.rs @@ -0,0 +1,228 @@ +use agent_desktop_core::{ + error::{AdapterError, AppError, ErrorCode}, + PermissionReport, +}; + +use crate::cli::Commands; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PermissionNeed { + None, + Accessibility, + ScreenRecording, + AccessibilityAndScreenRecording, +} + +pub(crate) fn policy_for(cmd: &Commands) -> PermissionNeed { + use PermissionNeed::{Accessibility, AccessibilityAndScreenRecording, None, ScreenRecording}; + match cmd { + Commands::Version(_) | Commands::Skills(_) => None, + Commands::Status | Commands::Permissions(_) => None, + Commands::ListWindows(_) | Commands::ListApps(_) => None, + Commands::ClipboardGet | Commands::ClipboardSet(_) | Commands::ClipboardClear => None, + Commands::Batch(_) => None, + + Commands::Snapshot(_) + | Commands::Find(_) + | Commands::ListSurfaces(_) + | Commands::Wait(_) + | Commands::ListNotifications(_) => Accessibility, + + Commands::Screenshot(a) if a.app.is_some() || a.window_id.is_some() => { + AccessibilityAndScreenRecording + } + Commands::Screenshot(_) => ScreenRecording, + + Commands::Get(_) | Commands::Is(_) => Accessibility, + + Commands::Click(_) + | Commands::DoubleClick(_) + | Commands::TripleClick(_) + | Commands::RightClick(_) + | Commands::SetValue(_) + | Commands::Clear(_) + | Commands::Select(_) + | Commands::Toggle(_) + | Commands::Check(_) + | Commands::Uncheck(_) + | Commands::Expand(_) + | Commands::Collapse(_) + | Commands::Scroll(_) + | Commands::ScrollTo(_) => Accessibility, + + Commands::Type(_) => Accessibility, + Commands::Focus(_) => Accessibility, + Commands::Press(_) | Commands::KeyDown(_) | Commands::KeyUp(_) => Accessibility, + Commands::Hover(_) + | Commands::Drag(_) + | Commands::MouseMove(_) + | Commands::MouseClick(_) + | Commands::MouseDown(_) + | Commands::MouseUp(_) => Accessibility, + + Commands::Launch(_) + | Commands::CloseApp(_) + | Commands::FocusWindow(_) + | Commands::ResizeWindow(_) + | Commands::MoveWindow(_) + | Commands::Minimize(_) + | Commands::Maximize(_) + | Commands::Restore(_) + | Commands::DismissNotification(_) + | Commands::DismissAllNotifications(_) + | Commands::NotificationAction(_) => Accessibility, + } +} + +pub(crate) fn preflight(cmd: &Commands, report: &PermissionReport) -> Result<(), AppError> { + let permission = policy_for(cmd); + if requires_screen_recording(permission) && report.screen_recording_denied() { + let err = AdapterError::new( + ErrorCode::PermDenied, + "Screen Recording permission not granted", + ) + .with_suggestion( + report + .screen_recording_suggestion() + .unwrap_or("Grant Screen Recording permission and retry"), + ); + return Err(AppError::Adapter(err)); + } + Ok(()) +} + +fn requires_screen_recording(permission: PermissionNeed) -> bool { + matches!( + permission, + PermissionNeed::ScreenRecording | PermissionNeed::AccessibilityAndScreenRecording + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cli::Cli; + use crate::cli_args::ScreenshotArgs; + use agent_desktop_core::{PermissionReport, PermissionState}; + use clap::CommandFactory; + + #[test] + fn every_cli_subcommand_has_policy() { + for subcommand in Cli::command().get_subcommands() { + let name = subcommand.get_name(); + assert!( + command_name_is_covered(name), + "missing permission policy coverage for {name}" + ); + } + } + + fn command_name_is_covered(name: &str) -> bool { + matches!( + name, + "snapshot" + | "find" + | "screenshot" + | "get" + | "is" + | "click" + | "double-click" + | "triple-click" + | "right-click" + | "type" + | "set-value" + | "clear" + | "focus" + | "select" + | "toggle" + | "check" + | "uncheck" + | "expand" + | "collapse" + | "scroll" + | "scroll-to" + | "press" + | "key-down" + | "key-up" + | "hover" + | "drag" + | "mouse-move" + | "mouse-click" + | "mouse-down" + | "mouse-up" + | "launch" + | "close-app" + | "list-windows" + | "list-apps" + | "focus-window" + | "resize-window" + | "move-window" + | "minimize" + | "maximize" + | "restore" + | "list-surfaces" + | "list-notifications" + | "dismiss-notification" + | "dismiss-all-notifications" + | "notification-action" + | "clipboard-get" + | "clipboard-set" + | "clipboard-clear" + | "wait" + | "status" + | "permissions" + | "version" + | "batch" + | "skills" + ) + } + + #[test] + fn unknown_permission_does_not_mask_platform_errors() { + let report = PermissionReport::default(); + let command = Commands::Screenshot(ScreenshotArgs { + app: None, + window_id: None, + output_path: None, + }); + + assert!(preflight(&command, &report).is_ok()); + } + + #[test] + fn screen_recording_denial_is_preflighted() { + let report = PermissionReport { + accessibility: PermissionState::Granted, + screen_recording: PermissionState::Denied { + suggestion: "grant screen recording".into(), + }, + automation: PermissionState::NotRequired, + }; + let command = Commands::Screenshot(ScreenshotArgs { + app: None, + window_id: None, + output_path: None, + }); + + let err = preflight(&command, &report).expect_err("denied screen capture fails"); + + assert_eq!(err.code(), "PERM_DENIED"); + } + + #[test] + fn accessibility_denial_does_not_preflight_ax_commands() { + let report = PermissionReport { + accessibility: PermissionState::Denied { + suggestion: "grant accessibility".into(), + }, + screen_recording: PermissionState::Granted, + automation: PermissionState::NotRequired, + }; + let command = Commands::Click(crate::cli_args::RefArgs { + ref_id: "@e1".into(), + snapshot_id: None, + }); + + preflight(&command, &report).expect("AX command should execute and report adapter result"); + } +} diff --git a/src/dispatch.rs b/src/dispatch.rs index 97e39bc..2f2447f 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -1,21 +1,30 @@ use agent_desktop_core::{ - action::{Direction, MouseButton}, adapter::PlatformAdapter, commands::{ - batch, check, clear, click, clipboard_clear, clipboard_get, clipboard_set, close_app, - collapse, double_click, drag, expand, find, focus, focus_window, get, helpers, hover, - is_check, key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize, - minimize, mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press, + check, clear, click, clipboard_clear, clipboard_get, clipboard_set, close_app, collapse, + double_click, drag, expand, find, focus, focus_window, get, helpers, hover, is_check, + key_down, key_up, launch, list_apps, list_surfaces, list_windows, maximize, minimize, + mouse_click, mouse_down, mouse_move, mouse_up, move_window, permissions, press, resize_window, restore, right_click, screenshot, scroll, scroll_to, select, set_value, skills, snapshot, status, toggle, triple_click, type_text, uncheck, version, wait, }, error::AppError, + PermissionReport, }; use serde_json::Value; -use crate::cli::{Commands, SkillsAction}; +use crate::cli::Commands; +use crate::cli_args_skills::SkillsAction; +use crate::dispatch_parse::{ + parse_direction, parse_get_property, parse_is_property, parse_mouse_button, parse_xy, + parse_xy_opt, +}; -pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result { +pub fn dispatch( + cmd: Commands, + adapter: &dyn PlatformAdapter, + permission_report: &PermissionReport, +) -> Result { tracing::debug!("dispatch: {}", cmd.name()); match cmd { Commands::Snapshot(a) => snapshot::execute( @@ -29,6 +38,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result Result Result get::execute( get::GetArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, property: parse_get_property(&a.property)?, }, adapter, @@ -68,25 +80,21 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result is_check::execute( is_check::IsArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, property: parse_is_property(&a.property)?, }, adapter, ), - Commands::Click(a) => click::execute(click::ClickArgs { ref_id: a.ref_id }, adapter), - Commands::DoubleClick(a) => { - double_click::execute(double_click::DoubleClickArgs { ref_id: a.ref_id }, adapter) - } - Commands::TripleClick(a) => { - triple_click::execute(triple_click::TripleClickArgs { ref_id: a.ref_id }, adapter) - } - Commands::RightClick(a) => { - right_click::execute(right_click::RightClickArgs { ref_id: a.ref_id }, adapter) - } + Commands::Click(a) => click::execute(ref_args(a), adapter), + Commands::DoubleClick(a) => double_click::execute(ref_args(a), adapter), + Commands::TripleClick(a) => triple_click::execute(ref_args(a), adapter), + Commands::RightClick(a) => right_click::execute(ref_args(a), adapter), Commands::Type(a) => type_text::execute( type_text::TypeArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, text: a.text, }, adapter, @@ -95,25 +103,25 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result set_value::execute( set_value::SetValueArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, value: a.value, }, adapter, ), - Commands::Clear(a) => clear::execute(clear::ClearArgs { ref_id: a.ref_id }, adapter), + Commands::Clear(a) => clear::execute(ref_args(a), adapter), - Commands::Focus(a) => focus::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), - Commands::Toggle(a) => toggle::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), - Commands::Check(a) => check::execute(check::CheckArgs { ref_id: a.ref_id }, adapter), - Commands::Uncheck(a) => { - uncheck::execute(uncheck::UncheckArgs { ref_id: a.ref_id }, adapter) - } - Commands::Expand(a) => expand::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), - Commands::Collapse(a) => collapse::execute(helpers::RefArgs { ref_id: a.ref_id }, adapter), + Commands::Focus(a) => focus::execute(ref_args(a), adapter), + Commands::Toggle(a) => toggle::execute(ref_args(a), adapter), + Commands::Check(a) => check::execute(ref_args(a), adapter), + Commands::Uncheck(a) => uncheck::execute(ref_args(a), adapter), + Commands::Expand(a) => expand::execute(ref_args(a), adapter), + Commands::Collapse(a) => collapse::execute(ref_args(a), adapter), Commands::Select(a) => select::execute( select::SelectArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, value: a.value, }, adapter, @@ -122,15 +130,14 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result scroll::execute( scroll::ScrollArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, direction: parse_direction(&a.direction)?, amount: a.amount, }, adapter, ), - Commands::ScrollTo(a) => { - scroll_to::execute(scroll_to::ScrollToArgs { ref_id: a.ref_id }, adapter) - } + Commands::ScrollTo(a) => scroll_to::execute(ref_args(a), adapter), Commands::Press(a) => press::execute( press::PressArgs { @@ -149,6 +156,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result hover::execute( hover::HoverArgs { ref_id: a.ref_id, + snapshot_id: a.snapshot, xy: parse_xy_opt(a.xy.as_deref())?, duration_ms: a.duration, }, @@ -161,6 +169,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result Result list_apps::execute(adapter), + Commands::ListApps(a) => { + list_apps::execute(list_apps::ListAppsArgs { app: a.app }, adapter) + } Commands::ListSurfaces(a) => { list_surfaces::execute(list_surfaces::ListSurfacesArgs { app: a.app }, adapter) @@ -261,11 +272,11 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result minimize::execute(minimize::MinimizeArgs { app: a.app }, adapter), + Commands::Minimize(a) => minimize::execute(helpers::AppArgs { app: a.app }, adapter), - Commands::Maximize(a) => maximize::execute(maximize::MaximizeArgs { app: a.app }, adapter), + Commands::Maximize(a) => maximize::execute(helpers::AppArgs { app: a.app }, adapter), - Commands::Restore(a) => restore::execute(restore::RestoreArgs { app: a.app }, adapter), + Commands::Restore(a) => restore::execute(helpers::AppArgs { app: a.app }, adapter), Commands::ListNotifications(_) | Commands::DismissNotification(_) @@ -282,6 +293,7 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result Result status::execute(adapter), + Commands::Status => status::execute_with_report(adapter, permission_report), - Commands::Permissions(a) => { - permissions::execute(permissions::PermissionsArgs { request: a.request }, adapter) - } + Commands::Permissions(a) => permissions::execute_with_report( + permissions::PermissionsArgs { request: a.request }, + adapter, + permission_report, + ), Commands::Version(a) => version::execute(version::VersionArgs { json: a.json }), @@ -311,102 +325,13 @@ pub fn dispatch(cmd: Commands, adapter: &dyn PlatformAdapter) -> Result { - let commands = batch::parse_commands(&a.commands_json)?; - let mut results = Vec::new(); - for cmd in commands { - let result = - crate::batch_dispatch::dispatch_batch_command(&cmd.command, cmd.args, adapter); - let ok = result.is_ok(); - let entry = match result { - Ok(data) => { - serde_json::json!({ "ok": true, "command": cmd.command, "data": data }) - } - Err(e) => { - serde_json::json!({ "ok": false, "command": cmd.command, "error": e.to_string() }) - } - }; - results.push(entry); - if !ok && a.stop_on_error { - break; - } - } - Ok(serde_json::json!({ "results": results })) - } + Commands::Batch(a) => crate::batch::execute(a, adapter, permission_report), } } -pub(crate) fn parse_get_property(s: &str) -> Result { - match s { - "text" => Ok(get::GetProperty::Text), - "value" => Ok(get::GetProperty::Value), - "title" => Ok(get::GetProperty::Title), - "bounds" => Ok(get::GetProperty::Bounds), - "role" => Ok(get::GetProperty::Role), - "states" => Ok(get::GetProperty::States), - other => Err(AppError::invalid_input(format!( - "Unknown property '{other}'. Valid: text, value, title, bounds, role, states" - ))), - } -} - -pub(crate) fn parse_is_property(s: &str) -> Result { - match s { - "visible" => Ok(is_check::IsProperty::Visible), - "enabled" => Ok(is_check::IsProperty::Enabled), - "checked" => Ok(is_check::IsProperty::Checked), - "focused" => Ok(is_check::IsProperty::Focused), - "expanded" => Ok(is_check::IsProperty::Expanded), - other => Err(AppError::invalid_input(format!( - "Unknown property '{other}'. Valid: visible, enabled, checked, focused, expanded" - ))), - } -} - -pub(crate) fn parse_direction(s: &str) -> Result { - match s { - "up" => Ok(Direction::Up), - "down" => Ok(Direction::Down), - "left" => Ok(Direction::Left), - "right" => Ok(Direction::Right), - other => Err(AppError::invalid_input(format!( - "Unknown direction '{other}'. Valid: up, down, left, right" - ))), - } -} - -pub(crate) fn parse_mouse_button(s: &str) -> Result { - match s { - "left" => Ok(MouseButton::Left), - "right" => Ok(MouseButton::Right), - "middle" => Ok(MouseButton::Middle), - other => Err(AppError::invalid_input(format!( - "Unknown button '{other}'. Valid: left, right, middle" - ))), - } -} - -pub(crate) fn parse_xy(s: &str) -> Result<(f64, f64), AppError> { - let parts: Vec<&str> = s.split(',').collect(); - if parts.len() != 2 { - return Err(AppError::invalid_input(format!( - "Invalid coordinates '{s}'. Expected format: x,y (e.g., 500,300)" - ))); - } - let x: f64 = parts[0] - .trim() - .parse() - .map_err(|_| AppError::invalid_input(format!("Invalid x coordinate: '{}'", parts[0])))?; - let y: f64 = parts[1] - .trim() - .parse() - .map_err(|_| AppError::invalid_input(format!("Invalid y coordinate: '{}'", parts[1])))?; - Ok((x, y)) -} - -fn parse_xy_opt(s: Option<&str>) -> Result, AppError> { - match s { - Some(s) => parse_xy(s).map(Some), - None => Ok(None), +fn ref_args(args: crate::cli_args::RefArgs) -> helpers::RefArgs { + helpers::RefArgs { + ref_id: args.ref_id, + snapshot_id: args.snapshot_id, } } diff --git a/src/dispatch_parse.rs b/src/dispatch_parse.rs new file mode 100644 index 0000000..012479b --- /dev/null +++ b/src/dispatch_parse.rs @@ -0,0 +1,129 @@ +use agent_desktop_core::{ + action::{Direction, MouseButton}, + commands::{get, is_check}, + error::AppError, +}; + +pub(crate) fn parse_get_property(s: &str) -> Result { + match s { + "text" => Ok(get::GetProperty::Text), + "value" => Ok(get::GetProperty::Value), + "title" => Ok(get::GetProperty::Title), + "bounds" => Ok(get::GetProperty::Bounds), + "role" => Ok(get::GetProperty::Role), + "states" => Ok(get::GetProperty::States), + other => Err(AppError::invalid_input(format!( + "Unknown property '{other}'. Valid: text, value, title, bounds, role, states" + ))), + } +} + +pub(crate) fn parse_is_property(s: &str) -> Result { + match s { + "visible" => Ok(is_check::IsProperty::Visible), + "enabled" => Ok(is_check::IsProperty::Enabled), + "checked" => Ok(is_check::IsProperty::Checked), + "focused" => Ok(is_check::IsProperty::Focused), + "expanded" => Ok(is_check::IsProperty::Expanded), + other => Err(AppError::invalid_input(format!( + "Unknown property '{other}'. Valid: visible, enabled, checked, focused, expanded" + ))), + } +} + +pub(crate) fn parse_direction(s: &str) -> Result { + match s { + "up" => Ok(Direction::Up), + "down" => Ok(Direction::Down), + "left" => Ok(Direction::Left), + "right" => Ok(Direction::Right), + other => Err(AppError::invalid_input(format!( + "Unknown direction '{other}'. Valid: up, down, left, right" + ))), + } +} + +pub(crate) fn parse_mouse_button(s: &str) -> Result { + match s { + "left" => Ok(MouseButton::Left), + "right" => Ok(MouseButton::Right), + "middle" => Ok(MouseButton::Middle), + other => Err(AppError::invalid_input(format!( + "Unknown button '{other}'. Valid: left, right, middle" + ))), + } +} + +pub(crate) fn parse_xy(s: &str) -> Result<(f64, f64), AppError> { + let parts: Vec<&str> = s.split(',').collect(); + if parts.len() != 2 { + return Err(AppError::invalid_input(format!( + "Invalid coordinates '{s}'. Expected format: x,y (e.g., 500,300)" + ))); + } + let x: f64 = parts[0] + .trim() + .parse() + .map_err(|_| AppError::invalid_input(format!("Invalid x coordinate: '{}'", parts[0])))?; + let y: f64 = parts[1] + .trim() + .parse() + .map_err(|_| AppError::invalid_input(format!("Invalid y coordinate: '{}'", parts[1])))?; + Ok((x, y)) +} + +pub(crate) fn parse_xy_opt(s: Option<&str>) -> Result, AppError> { + match s { + Some(s) => parse_xy(s).map(Some), + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_direction() { + assert_eq!( + parse_direction("sideways").unwrap_err().code(), + "INVALID_ARGS" + ); + } + + #[test] + fn rejects_unknown_get_property() { + match parse_get_property("placeholder") { + Ok(_) => panic!("expected invalid get property"), + Err(err) => assert_eq!(err.code(), "INVALID_ARGS"), + } + } + + #[test] + fn rejects_unknown_is_property() { + match parse_is_property("selected") { + Ok(_) => panic!("expected invalid is property"), + Err(err) => assert_eq!(err.code(), "INVALID_ARGS"), + } + } + + #[test] + fn rejects_unknown_mouse_button() { + assert_eq!( + parse_mouse_button("primary").unwrap_err().code(), + "INVALID_ARGS" + ); + } + + #[test] + fn parses_xy_with_whitespace() { + assert_eq!(parse_xy(" 10.5, 20 ").unwrap(), (10.5, 20.0)); + } + + #[test] + fn rejects_bad_xy_shape_and_numbers() { + assert_eq!(parse_xy("10").unwrap_err().code(), "INVALID_ARGS"); + assert_eq!(parse_xy("x,20").unwrap_err().code(), "INVALID_ARGS"); + assert_eq!(parse_xy("10,y").unwrap_err().code(), "INVALID_ARGS"); + } +} diff --git a/src/help_after.txt b/src/help_after.txt index 9ddbd88..016ae3d 100644 --- a/src/help_after.txt +++ b/src/help_after.txt @@ -1,17 +1,17 @@ OBSERVATION snapshot Accessibility tree as JSON with @ref IDs screenshot PNG screenshot of an application window - find Search elements by role, name, value, or text - get Read element property: text, value, title, bounds, role, states - is Check state: visible, enabled, checked, focused, expanded + find Search elements by role, name, value, or text (--limit defaults to 50) + get --property

Read element property: text, value, title, bounds, role, states + is --property

Check state: visible, enabled, checked, focused, expanded list-surfaces Available surfaces for an app INTERACTION click Click element (kAXPress) - double-click Double-click element - triple-click Triple-click element (select line/paragraph) - right-click Right-click and open context menu - type Focus element and type text + double-click Open via AXOpen; physical double-click uses mouse-click + triple-click Triple-click element; POLICY_DENIED if physical input is disabled + right-click Right-click; includes menu when verified + type Insert text; may use focus fallback only for explicit policy paths set-value Set value attribute directly clear Clear element value to empty string focus Set keyboard focus @@ -41,8 +41,8 @@ APP & WINDOW launch Launch app and wait until window is visible close-app Quit app gracefully (--force to kill) list-windows All visible windows (--app to filter) - list-apps All running GUI applications - focus-window Bring window to front + list-apps All running GUI applications (--app substring filter) + focus-window Bring window to front and confirm focus resize-window Resize window (--width, --height) move-window Move window (--x, --y) minimize Minimize window @@ -63,13 +63,16 @@ CLIPBOARD WAIT wait [ms] Pause for N milliseconds wait --element Block until element appears (--timeout ms) + wait --element --snapshot Wait against a specific snapshot refmap wait --window Block until window appears wait --text <text> Block until text appears in app - wait --notification Block until a new notification arrives + wait --menu --app <app> Block until a menu surface is open + wait --menu-closed --app <app> Block until a menu surface is dismissed + wait --notification Block until a new notification arrives (--text filters it) SYSTEM status Adapter health, platform, and permission state - permissions Check accessibility permission (--request to prompt) + permissions Check nested permission states: {state,...} version Version string (--json for machine-readable) skills Bundled skill docs for AI agents (list, get, path) @@ -78,8 +81,9 @@ BATCH REF IDs snapshot assigns @e1, @e2, ... to interactive elements in depth-first order. - Use a ref wherever <ref> appears. Refs are snapshot-scoped; run snapshot - again after UI changes. + Use a ref wherever <ref> appears. Refs are snapshot-scoped. Pass + --snapshot <id> to pin a command to a known snapshot; when omitted, + ref commands use the latest saved snapshot. Run snapshot again after UI changes. KEY COMBOS Single keys: return, escape, tab, space, delete, up, down, left, right @@ -91,6 +95,7 @@ EXAMPLES agent-desktop skills get desktop --full Load the primary skill (start here) agent-desktop snapshot --app "System Settings" -i agent-desktop find --role button --name "OK" + agent-desktop find --role button --limit 20 agent-desktop click @e5 agent-desktop check @e3 agent-desktop type @e2 "hello@example.com" diff --git a/src/main.rs b/src/main.rs index 0a2a45d..805319d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,23 @@ -mod batch_dispatch; -mod batch_dispatch_ext; +mod batch; mod cli; mod cli_args; +mod cli_args_actions; mod cli_args_notifications; mod cli_args_skills; +mod cli_args_system; +mod command_policy; mod dispatch; mod dispatch_notifications; +mod dispatch_parse; -use agent_desktop_core::adapter::PlatformAdapter; +use agent_desktop_core::{ + adapter::PlatformAdapter, + error::AppError, + output::{ErrorPayload, Response, ENVELOPE_VERSION}, +}; use clap::{CommandFactory, Parser}; use cli::{Cli, Commands}; +use cli_args_skills::SkillsAction; use std::io::{BufWriter, Write}; fn main() { @@ -24,13 +32,10 @@ fn main() { } let msg = e.to_string(); let first_line = msg.lines().next().unwrap_or("parse error"); - let json = serde_json::json!({ - "version": "1.0", - "ok": false, - "command": "unknown", - "error": { "code": "INVALID_ARGS", "message": first_line } - }); - emit_json(&json); + emit_response(&Response::err( + "unknown", + ErrorPayload::new("INVALID_ARGS", first_line), + )); std::process::exit(2); } }; @@ -54,16 +59,11 @@ fn main() { ); finish(cmd_name, result); } - Commands::Status => { - let adapter = build_adapter(); - let result = agent_desktop_core::commands::status::execute(&adapter); - finish(cmd_name, result); - } Commands::Skills(a) => { - let result = match a.action.unwrap_or(cli::SkillsAction::List) { - cli::SkillsAction::List => agent_desktop_core::commands::skills::list(), - cli::SkillsAction::Path => agent_desktop_core::commands::skills::path(), - cli::SkillsAction::Get(g) => agent_desktop_core::commands::skills::get( + let result = match a.action.unwrap_or(SkillsAction::List) { + SkillsAction::List => agent_desktop_core::commands::skills::list(), + SkillsAction::Path => agent_desktop_core::commands::skills::path(), + SkillsAction::Get(g) => agent_desktop_core::commands::skills::get( agent_desktop_core::commands::skills::GetArgs { name: g.name, full: g.full, @@ -79,65 +79,51 @@ fn main() { fn run_with_adapter(cmd: Commands, cmd_name: &str) { let adapter = build_adapter(); - - if let agent_desktop_core::adapter::PermissionStatus::Denied { suggestion } = - adapter.check_permissions() - { - match &cmd { - Commands::Permissions(_) | Commands::Version(_) | Commands::Status => {} - _ => { - let json = serde_json::json!({ - "version": "1.0", - "ok": false, - "command": cmd_name, - "error": { - "code": "PERM_DENIED", - "message": "Accessibility permission not granted", - "suggestion": suggestion - } - }); - emit_json(&json); - std::process::exit(1); - } - } + let report = adapter.permission_report(); + if let Err(err) = command_policy::preflight(&cmd, &report) { + finish(cmd_name, Err(err)); + return; } - let result = dispatch::dispatch(cmd, &adapter); + let result = dispatch::dispatch(cmd, &adapter, &report); finish(cmd_name, result); } fn finish(cmd_name: &str, result: Result<serde_json::Value, agent_desktop_core::error::AppError>) { match result { Ok(data) => { - let response = serde_json::json!({ - "version": "1.0", - "ok": true, - "command": cmd_name, - "data": data - }); - emit_json(&response); + emit_response(&Response::ok(cmd_name, data)); std::process::exit(0); } Err(e) => { - let mut error = serde_json::json!({ - "code": e.code(), - "message": e.to_string(), - }); + let mut payload = ErrorPayload::new(e.code(), e.to_string()); if let Some(s) = e.suggestion() { - error["suggestion"] = serde_json::Value::String(s.to_string()); + payload = payload.with_suggestion(s); } - let response = serde_json::json!({ - "version": "1.0", - "ok": false, - "command": cmd_name, - "error": error - }); - emit_json(&response); + if let AppError::Adapter(adapter_error) = &e { + payload.platform_detail = adapter_error.platform_detail.clone(); + } + emit_response(&Response::err(cmd_name, payload)); std::process::exit(1); } } } +fn emit_response(response: &Response) { + match serde_json::to_value(response) { + Ok(value) => emit_json(&value), + Err(err) => emit_json(&serde_json::json!({ + "version": ENVELOPE_VERSION, + "ok": false, + "command": "internal", + "error": { + "code": "INTERNAL", + "message": format!("Failed to serialize response: {err}") + } + })), + } +} + fn emit_json(value: &serde_json::Value) { let stdout = std::io::stdout(); let mut writer = BufWriter::new(stdout.lock());