diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index 5ca40fc..acd441e 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -23,6 +23,11 @@ jobs: contents: read steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # Full history so the secret scan below can walk commits, not just + # the final tree. A shallow clone would silently reduce it to a + # working-tree scan. + fetch-depth: 0 - name: Check release metadata consistency run: scripts/check-release-consistency.sh @@ -41,6 +46,22 @@ jobs: arguments: --locked command-arguments: advisories licenses bans sources + - name: Secret and privacy scan + env: + GITLEAKS_VERSION: 8.30.1 + GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + run: | + set -euo pipefail + archive="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -fsSL -o "$archive" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${archive}" + echo "${GITLEAKS_SHA256} ${archive}" | sha256sum --check --status + tar -xzf "$archive" gitleaks + # Scan commit history, not just the checked-out tree: a value added + # in one commit and removed in a later one stays reachable in branch + # history, and a tree-only scan would pass it. + ./gitleaks git . --config .gitleaks.toml --redact --no-banner --exit-code 1 + - name: Workflow security audit uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 with: diff --git a/.gitignore b/.gitignore index d22d5b9..d740e6d 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,14 @@ docs/* !docs/phases.md !docs/solutions/ !docs/solutions/** +!docs/plans/ +!docs/plans/** +!docs/brainstorms/ +!docs/brainstorms/** + +# macOS resource forks — never commit +._* +.DS_Store todos/ .cursor/ .context/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..bf5533f --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,84 @@ +title = "agent-desktop secret and privacy scanning" + +# Inherit gitleaks' built-in credential rules (API keys, tokens, private keys), +# then add project-specific privacy rules below. This repo is public and ships +# planning documents, probe evidence, and traces that are written on developer +# machines, so operator identity and filesystem layout are treated as leaks +# alongside conventional secrets. +# +# Regexes are Go RE2: no lookahead, no lookbehind, no backreferences. Anything +# that would need a negative lookahead is expressed as an allowlist entry. +[extend] +useDefault = true + +[[rules]] +id = "ad-home-directory-path" +description = "Absolute home-directory path exposing an operator account name" +# The first character after the home root must not be a dot, so relative +# fragments like "$suite_root/home/.cache" are not mistaken for "/home/". +regex = '''(?i)(/Users/|/home/|[A-Za-z]:\\Users\\)[A-Za-z0-9_-][A-Za-z0-9._-]{1,}''' +tags = ["privacy", "path", "identity"] + +[[rules]] +id = "ad-windows-account-sid" +description = "Windows account SID with machine-unique sub-authorities" +regex = '''S-1-5-21-[0-9]{5,}-[0-9]{5,}-[0-9]{5,}-[0-9]{3,}''' +tags = ["privacy", "identity"] + +[[rules]] +id = "ad-personal-email" +description = "Personal email address in tracked content" +regex = '''[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}''' +tags = ["privacy", "identity"] + +[[rules]] +id = "ad-machine-hostname" +description = "Developer machine hostname captured from a probe or trace" +# Case-sensitive on purpose: Windows default hostnames are uppercase +# (DESKTOP-A1B2C3D). A case-insensitive form matches this project's own crate +# names (agent-desktop-core, agent-desktop-ffi, ...) and is pure noise. +regex = '''\b(DESKTOP|LAPTOP)-[A-Z0-9]{7}\b''' +tags = ["privacy", "identity"] + +# --------------------------------------------------------------------------- +# Allowlist: documentation placeholders, generated artifacts, and OS vocabulary. +# The rules above intentionally match broadly; these entries keep the scan +# usable on content that only *looks* like a leak. +# --------------------------------------------------------------------------- +[allowlist] +description = "Documentation placeholders and generated artifacts" +# Match allowlist regexes against the whole line, not just the captured secret, +# so a placeholder is recognised from its surrounding context. +regexTarget = "line" +paths = [ + '''Cargo\.lock''', + '''package-lock\.json''', + '''(^|/)tests/fixtures/''', + # Never-committed local state and build output. Listed so a local + # `gitleaks dir .` scan matches what CI sees on tracked content. + '''(^|/)target/''', + '''(^|/)node_modules/''', + '''(^|/)\.claude/''', +] +# Probe captures under probes/*/captures/ are deliberately NOT allowlisted: +# they are UIA tree dumps taken on a developer desktop and are the most likely +# place for a redaction failure, so gitleaks is the backstop behind the probe +# corpus's own redaction gate. +regexes = [ + # Explicit redaction markers and angle-bracket placeholders. + ''']*>''', + '''S-1-5-21-<[^>]*>''', + # Generic doc examples that teach path shape rather than expose one. + '''(?i)/Users/(name|user|username|you|youruser|me|someone)''', + '''(?i)/home/(name|user|username|you|youruser|runner|me|someone)''', + '''(?i)[A-Za-z]:\\Users\\(name|user|username|you|youruser|runner)''', + # CI runner home paths are not operator identity. + '''/Users/runner''', + '''/home/runner''', + # Well-known localized group principals are OS vocabulary, not identity. + '''(?i)BUILTIN\\Admin[a-z]*''', + # Addresses that are meant to be public. + '''(?i)[A-Za-z0-9._%+-]+@example\.(com|org|net)''', + '''(?i)noreply@''', + '''(?i)users\.noreply\.github\.com''', +] diff --git a/docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md b/docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md new file mode 100644 index 0000000..5630368 --- /dev/null +++ b/docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md @@ -0,0 +1,131 @@ +--- +date: 2026-02-19 +topic: architecture-validation +category: design +tags: [rust, architecture, platform-adapter, refmap, testing] +--- + +# Architecture Validation: agent-desktop PRD v2 + +## What We're Validating + +The PRD v2 is a complete engineering blueprint. This session pressure-tested three +gaps the PRD leaves underspecified that need decisions before code is written. + +## Decisions Made + +### 1. PlatformAdapter Trait — Stay Unified + +**Question:** Should the trait split into `PlatformObserver` (reads) + `PlatformActor` (writes)? + +**Decision:** Keep the unified 13-method trait as the PRD specifies. + +**Rationale:** Splitting does not help command extensibility. The §4.5 extensibility +pattern (one new file + 2 registration lines) never touches the trait — commands call +existing adapter methods. Splitting adds two trait objects, more indirection, and no +benefit for the primary extensibility goal. The 13 methods are manageable. + +--- + +### 2. NativeHandle Persistence — Optimistic with STALE_REF + +**Question:** `AXUIElement` on macOS is a live CFTypeRef. It cannot be serialized +across process invocations. How should action commands resolve `@e1` on a fresh CLI +invocation? + +**Decision:** Optimistic re-identification. The RefMap stores metadata per ref: +`(pid, role, name, bounds_hash)`. Action commands use this to locate the element in +the current AX tree. If the element is not found or metadata mismatches, return the +`STALE_REF` error (already in the PRD error taxonomy) with suggestion "Run 'snapshot' +to refresh, then retry with updated ref." + +**Rationale:** Fast path — most of the time the UI hasn't changed, so no re-traversal +needed. Clean failure path — `STALE_REF` is already documented and agents know how to +handle it. In Phase 4, the daemon holds the RefMap in memory, making this moot. + +**RefMap JSON entry shape:** +```json +{ + "@e3": { + "pid": 1234, + "role": "textfield", + "name": "Search", + "bounds_hash": "a3f9c2", + "available_actions": ["SetValue", "SetFocus"] + } +} +``` + +**Refmap write behavior:** Each snapshot REPLACES the refmap file entirely (not merges). +Agents should snapshot before acting. + +--- + +### 3. Testing Strategy — MockAdapter + Golden Fixtures + +**Question:** macOS GitHub Actions runners have no active display session. How do you +test `SnapshotEngine`, `RefAllocator`, and serialization without live AX access? + +**Decision:** Both approaches in parallel: + +- **MockAdapter** (`agent-desktop-core` test module): An in-memory `PlatformAdapter` + implementation returning a hardcoded `AccessibilityNode` tree. Exercises the full + pipeline (adapter → filter → ref-allocate → serialize) with no OS dependency. + Used for unit tests. + +- **Golden JSON fixtures** (`tests/fixtures/`): Real snapshots captured once from + Finder, TextEdit, etc. Checked into the repo. Used to regression-test that + serialization format changes don't silently alter the JSON contract. + +macOS CI integration tests (GitHub Actions macOS runner) test the real AX adapter +against live apps. + +--- + +### 4. Workspace Bootstrap — All Platform Crates from Day One + +**Decision:** Phase 1 creates all crates upfront: + +``` +crates/ + core/ # agent-desktop-core — fully implemented in P1 + macos/ # agent-desktop-macos — fully implemented in P1 + windows/ # agent-desktop-windows — stub: all methods return Err(not_supported) + linux/ # agent-desktop-linux — stub: all methods return Err(not_supported) + mcp/ # agent-desktop-mcp — stub (implemented in P3) +src/ # agent-desktop binary +``` + +**Rationale:** Enforces the platform isolation boundary (`core` never imports platform +crates) from the first commit. Prevents accidental coupling. The `PLATFORM_UNSUPPORTED` +error already exists in the PRD error taxonomy for stub responses. + +--- + +## Architectural Validations (PRD Is Correct) + +The following PRD decisions were validated as sound: + +- **Additive phase model** — Phase 1 builds the complete vertical. Phases 2-4 add + adapters/transports without modifying core. This is the right design. +- **Command extensibility** — One file + 2 registration lines per command. Elegant. + Strongly enforce this in code review. +- **Ref system** — Depth-first `@e1, @e2` on interactive-only roles. Combined with + the <500 token budget via compact serialization, this is the right approach. +- **Dual-mode entry** — `--mcp` flag triggers MCP server mode, otherwise CLI. + Invariant: every MCP tool maps 1:1 to a CLI command. +- **Sync trait for Phase 1** — All macOS AX APIs are synchronous. The async runtime + (tokio) is only needed for Linux AT-SPI (Phase 2) and MCP server (Phase 3). + The trait stays sync; adapters handle async internally in Phase 2 via `block_on`. + +## Open Questions for Planning + +- Token estimation: the <500 token budget (G4) requires a measurement strategy during + development. Should `tiktoken-rs` or a simple char-count heuristic be used for the + optional warning? +- Ref stability for multi-window snapshots: confirm that a `snapshot --window w-123` + only writes that window's refs to the refmap (not a global merge across windows). + +## Next Steps + +→ `/workflows:plan` to create the Phase 1 implementation plan (10-week milestone breakdown) diff --git a/docs/brainstorms/2026-02-19-ax-tree-accuracy-speed-brainstorm.md b/docs/brainstorms/2026-02-19-ax-tree-accuracy-speed-brainstorm.md new file mode 100644 index 0000000..083cd44 --- /dev/null +++ b/docs/brainstorms/2026-02-19-ax-tree-accuracy-speed-brainstorm.md @@ -0,0 +1,176 @@ +--- +date: 2026-02-19 +topic: ax-tree-accuracy-speed +--- + +# AX Tree Accuracy & Speed: First-Principles Fix + +## What We're Building + +A corrected, fast accessibility tree traversal engine that returns complete, readable node data +on every call. The current implementation has two compounding bugs that make the tool nearly +useless in practice: node names are missing for most elements, and traversal is 4–5 seconds +for a medium-sized UI. + +--- + +## Root Cause Diagnosis + +### Bug 1 — Empty names + +`fetch_node_attrs` tries `kAXTitleAttribute` then `kAXDescriptionAttribute` and calls it done. + +The macOS AX tree does **not** guarantee that either attribute carries the visible label text. +For the vast majority of real-world controls: + +| Element type | Where the label actually lives | +|---|---| +| `AXOutlineRow` (Finder sidebar) | `kAXTitleAttribute` ← **should work**, but our batch parse drops it | +| `AXRow` (Finder column browser) | child `AXCell` → child `AXStaticText` → `kAXValueAttribute` | +| `AXButton` with image only | `kAXDescriptionAttribute` | +| `AXTextField` | `kAXPlaceholderValueAttribute` if empty, else `kAXValueAttribute` | +| `AXStaticText` | `kAXValueAttribute` (not title!) | +| `AXMenuItem` | `kAXTitleAttribute` | + +The batch fetch uses `AXUIElementCopyMultipleAttributeValues`. The result is a `CFArray` of +`CFType` values. Our parsing calls `item.downcast::()`. This is correct for string +attrs — but **boolean attrs** (`kAXEnabledAttribute`, `kAXFocusedAttribute`) are `CFBoolean`, +so they downcast to `None`. That's fine because we re-fetch them individually. But the real +problem is more subtle: **some elements return their string attrs as `CFString` subclasses +or as `AXTextMarker` types** that do not downcast to plain `CFString`. In those cases we get +`None` even when data is present. + +The fix requires a multi-attribute fallback chain **plus** a child-text fallback. + +### Bug 2 — Unknown roles + +`roles.rs` maps ~22 role strings but the AX API has ~55. Anything not mapped becomes +`"unknown"`. In Finder's column browser, most elements are `AXBrowser`, `AXColumn`, `AXRow` +(table row variant), `AXLayoutItem` — all currently `"unknown"`. This makes the tree +unreadable and prevents `find --role` from working. + +Unmapped roles in the existing codebase: +`AXBrowser`, `AXColumn`, `AXRow` (table row), `AXGrid`, `AXHandle`, `AXPopover`, +`AXDockItem`, `AXRuler`, `AXRulerMarker`, `AXTimeField`, `AXDateField`, `AXHelpTag`, +`AXMatte`, `AXDrawer`, `AXLayoutArea`, `AXLayoutItem`, `AXLevelIndicator`, +`AXRelevanceIndicator`, `AXSearchField` (already covered by textfield but missing as alias), +`AXSwitch`, `AXMenuButton`. + +### Bug 3 — Speed + +Every node requires 2–4 round-trips across the Mach IPC boundary: +1. `AXUIElementCopyMultipleAttributeValues` — 1 call for 6 attrs +2. `copy_children` tries up to 3 attr fetches (`kAXChildren`, `kAXContents`, + `AXChildrenInNavigationOrder`) even when the first one succeeds + +Each Mach IPC call costs ~1–5 ms. A 150-node tree = 300–600 round-trips = 2–5 seconds. + +The fix is: stop on first successful children attribute, and batch-fetch **more** useful attrs +in the single `AXUIElementCopyMultipleAttributeValues` call so we never need per-attr fallback +calls for the common case. + +### Non-bug: `-i` flag + +Already implemented. `#[arg(long, short = 'i')]` on `interactive_only`. Works as +`agent-desktop snapshot -i`. The long form is `--interactive-only`, not `--interactive`. +We should add an alias `--interactive` to match user expectation. + +--- + +## Approach A: Fix name resolution with fallback chain (Recommended) + +**What it is:** Extend `fetch_node_attrs` to try 5 name sources in order, including +reading the first `AXStaticText` child. Expand `roles.rs` to cover all standard AX roles. +Stop `copy_children` on first non-empty result. Add `--interactive` as alias. + +**Pros:** +- Targets the exact root cause. Minimal code change. +- No architecture change — same `build_subtree` recursion. +- Risk is low. Every change is additive. + +**Cons:** +- The child-text fallback adds 1 extra IPC call per node that has no direct name. + For nodes that DO have `kAXTitleAttribute`, this extra call is skipped. + +**Speed estimate:** ~2x improvement from stopping children fetch early. +Name accuracy: ~95% of real apps will have readable names. + +--- + +## Approach B: Batch all attrs including children in one call + +**What it is:** Use `AXUIElementCopyMultipleAttributeValues` to fetch role, subrole, title, +description, value, placeholder, help, enabled, focused, selected, expanded, children — all +in a single call. Parse `CFBoolean` and `CFArray` correctly alongside `CFString`. + +**Pros:** +- Reduces per-node IPC calls from 2–4 down to 1. +- Could bring 150-node tree from 4s down to <1s. + +**Cons:** +- Requires correctly parsing mixed-type `CFArray` results (CFString, CFBoolean, CFArray). +- `AXUIElementCopyMultipleAttributeValues` doesn't support fetching array attrs + (kAXChildren) — children still need a separate call. +- Higher implementation risk; the CF type introspection is fragile. + +--- + +## Approach C: AXUIElementCopyAttributeNames + full dynamic attribute dump + +**What it is:** For each node, first call `AXUIElementCopyAttributeNames` to get the list of +all attributes the element supports, then batch-fetch only those. This is what macOS +Accessibility Inspector does internally. + +**Pros:** +- Most complete and accurate — never misses a custom attribute. +- Would fully expose Electron app data, web areas, custom controls. + +**Cons:** +- Doubles the IPC calls per node (one for names, one for values). +- Much slower than Approach A or B for the common case. +- Over-engineered for Phase 1. + +--- + +## Why Approach A, then B incrementally + +Start with A because it fixes the accuracy problem with minimal risk and gives us a working, +readable tree. Then layer in B's batch parsing improvements as a separate optimization pass +once the output is verified correct. + +The correctness of the tree is more valuable right now than raw speed. A 2-second tree with +readable names beats a 0.5-second tree of `"unknown"` nodes. + +--- + +## Key Decisions + +- **Name fallback chain**: `kAXTitleAttribute` → `kAXDescriptionAttribute` → + `kAXValueAttribute` → first AXStaticText child's `kAXValueAttribute` → + `kAXPlaceholderValueAttribute` +- **Expand roles.rs**: Add all ~33 missing role mappings from `role_constants.rs` +- **Add subrole field**: Include `kAXSubroleAttribute` in output so agents can distinguish + `AXOutlineRow` vs `AXTableRow` vs plain `AXRow` +- **children fetch**: Break on first non-empty result (already partially there, needs + confirmation that `kAXChildrenAttribute` wins 99% of the time and others are rarely needed) +- **`--interactive` alias**: Add alongside existing `--interactive-only` and `-i` +- **Test harness**: Write a Rust integration test that snapshots Finder and asserts: + - ≥10 nodes have non-empty names + - Zero nodes have role `"unknown"` + - Total time < 3 seconds + +--- + +## Open Questions + +- Do we want to include `kAXSubrole` in the JSON output, or keep the schema flat? +- Should we emit a `label` field (the human-readable role description) separately from `role`? +- For Finder's column browser specifically: are file rows exposed as `AXRow` children of + `AXColumn` children of `AXBrowser`? Need to confirm with a debug attr-dump. +- Should `find` command also search `value` field, not just `name`? + +--- + +## Next Steps + +→ `/workflows:plan` to break this into implementation tasks diff --git a/docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md b/docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md new file mode 100644 index 0000000..76c9b3c --- /dev/null +++ b/docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md @@ -0,0 +1,467 @@ +--- +date: 2026-02-23 +topic: macos-ax-first-robustness +--- + +# macOS AX-First Robustness + +> Current contract note (2026-05-12): this brainstorm is historical. The current +> implementation keeps CLI ref commands headless by default; focus, cursor, +> keyboard, pasteboard, and CGEvent paths require explicit physical/headed +> commands or explicit FFI policy selection. + +## What We're Building + +Harden the macOS adapter so every command optimizes for AX-first execution (headless-capable, background-safe), with multi-step fallback chains, actionable error suggestions, and resilience against transient IPC failures. + +## Why This Approach + +agent-desktop's core value proposition is headless/background desktop automation. AX API actions work on unfocused windows; CGEvent coordinate-based input requires the window to be in front. Every command that currently does a single AX call and gives up is a missed opportunity — and a broken experience when running in the background. + +`smart_activate()` (click) already has a 14-step chain. `ax_scroll()` has an 8-step chain. The other commands (`set-value`, `type`, `focus`, `press`, `clear`, `hover`, `drag`) are single-shot with no fallback. This brainstorm covers bringing them all up to parity. + +## Current State Audit + +### Commands WITH Robust Fallback Chains +| Command | Chain | Steps | CGEvent Fallback | +|---------|-------|-------|------------------| +| click | `smart_activate()` | 14 | Yes (last resort) | +| double-click | `smart_double_activate()` | AXOpen -> 2x smart_activate -> CGEvent | Yes | +| right-click | `smart_right_activate()` | AXShowMenu -> focus+retry -> select+menu -> focus+menu -> parent -> child -> CGEvent | Yes | +| triple-click | `smart_triple_activate()` | 3x smart_activate -> CGEvent | Yes | +| scroll | `ax_scroll()` | 8 | Yes (mouse wheel, last) | +| select | `select_value()` | Role-specific (combobox/popup/list) | No | +| expand/collapse | AXExpand -> AXDisclosing | 2 steps | No | +| check/uncheck | State check -> smart_activate if needed | 2 | Yes (via smart_activate) | +| toggle | Role check -> smart_activate | 2 | Yes (via smart_activate) | + +### Commands WITHOUT Fallback Chains (Single-Shot) +| Command | Current Behavior | Needs Chain | +|---------|-----------------|-------------| +| set-value | Single `AXUIElementSetAttributeValue` | Yes | +| type | Focus element + `synthesize_text()` (system-wide CGEvent keys) | Yes | +| clear | Delegates to `ax_set_value(el, "")` | Yes | +| focus | Single `AXUIElementSetAttributeValue(kAXFocusedAttribute)` | Yes | +| scroll-to | Single `AXScrollToVisible` | Yes | +| press | System-wide `AXUIElementPostKeyboardEvent` | Partial | +| hover | Returns `ActionNotSupported` at element level | Needs rethink | +| drag | Returns `ActionNotSupported` at element level | Needs rethink | +| key-down/key-up | Returns `ActionNotSupported` at element level | Needs rethink | + +### Commands Missing Error Suggestions +| Command | Current Error | Missing Suggestion | +|---------|--------------|-------------------| +| set-value | "SetValue failed (err=N)" | "Try 'clear' then 'type', or check element is a text field." | +| focus | "SetFocus failed (err=N)" | "Try 'click' to focus the element instead." | +| type | (no error path for AX focus fail) | "If typing fails, try 'set-value' for direct text insertion." | +| press | "AXUIElementPostKeyboardEvent failed" | "Ensure the target app is focused, or use 'press' with --app flag." | +| clear | Same as set-value | "Try selecting all text (press cmd+a) then press delete." | +| scroll-to | Has suggestion already | OK | +| hover/drag/key-down/key-up | "requires adapter-level handling" | "Use 'mouse-move --xy X,Y' for hover" / "Use 'drag --from-xy --to-xy'" | + +## Window Focus Policy + +**Principle:** Only focus the window when falling back to CGEvent coordinate-based input. + +| Action Type | Needs Window Focus | Why | +|-------------|-------------------|-----| +| AXUIElementPerformAction | No | AX IPC targets the element directly | +| AXUIElementSetAttributeValue | No | AX IPC targets the element directly | +| AXUIElementPostKeyboardEvent (to app) | No | Targets app AXElement by PID | +| CGEventPost (mouse/keyboard) | **Yes** | System-wide events go to frontmost window | +| `synthesize_text()` (system-wide) | **Yes** | Posts to system-wide AX element | +| `synthesize_mouse()` | **Yes** | CGEvent coordinate-based | + +**Current right-click**: `smart_right_activate()` tries AXShowMenu first (no focus needed). Falls back to CGEvent right-click only as last resort (focuses window at that point). This is correct — AXShowMenu works on unfocused windows. + +**Type command issue**: Currently uses system-wide `AXUIElementPostKeyboardEvent` via `synthesize_text()`. This means keys go to whatever is focused system-wide, NOT specifically to the target app. The chain should try `AXSetValue` first, then app-targeted key posting, then system-wide as last resort. + +## Centralized Chain Architecture + +### Problem with Per-Command Chains + +The current codebase has 3 hand-written chains (`smart_activate` 14 steps, `smart_right_activate` 7 steps, `ax_scroll` 8 steps) with ~100-150 LOC of duplicated patterns: + +- **AXScrollToVisible preamble** — 3 places, 3 different error strategies +- **list_ax_actions / has_ax_action** — near-duplicate functions across files +- **Attribute set boilerplate** — 6+ instances of CFString + AXUIElementSetAttributeValue + error check +- **Child/parent walk loops** — 5 near-identical traversals with different predicates +- **Sleep timing** — 7 instances, 4 different durations, no policy + +Writing per-command chains would add 6 more instances of this same boilerplate. Instead: centralize. + +### Design: Centralized Element Discovery + Chain Executor + +Two layers: (1) discover what an element supports once, (2) execute a declarative chain. + +#### Layer 1: Element Capabilities (one-time discovery) + +```rust +// New file: crates/macos/src/actions/discovery.rs + +pub struct ElementCaps { + pub actions: Vec, // AXPress, AXConfirm, AXOpen, AXShowMenu, etc. + pub settable_value: bool, // kAXValueAttribute is settable + pub settable_focus: bool, // kAXFocusedAttribute is settable + pub settable_selected: bool, // AXSelected is settable + pub settable_disclosing: bool, // AXDisclosing is settable + pub role: Option, // normalized role string + pub has_children: bool, + pub pid: Option, +} + +pub fn discover(el: &AXElement) -> ElementCaps { + // Single batch call: list actions + check settable attrs + read role + // Replaces scattered is_attr_settable + list_ax_actions + element_role calls +} +``` + +Every chain step gets `&ElementCaps` instead of re-querying the element. This avoids redundant IPC calls to the accessibility server. + +#### Layer 2: Chain Executor (declarative steps) + +```rust +// New file: crates/macos/src/actions/chain.rs + +pub enum ChainStep { + /// Try AXUIElementPerformAction with given action name + PerformAction(&'static str), + /// Try AXUIElementSetAttributeValue + SetAttr { attr: &'static str, value: AttrValue }, + /// Try the step on the element, if fails try with focus first + WithFocus(Box), + /// Try the step on up to N children + OnChildren { step: Box, limit: usize }, + /// Try the step on up to N ancestors + OnAncestors { step: Box, limit: usize }, + /// Custom logic (for command-specific steps that don't fit the pattern) + Custom(fn(&AXElement, &ElementCaps) -> bool), + /// Explicit physical policy step only + CGEvent(CGFallback), +} + +pub enum AttrValue { + Bool(bool), + Str(String), +} + +pub enum CGFallback { + Click(MouseButton, u32), + MouseWheel { dy: i32, dx: i32 }, + None, +} + +pub struct AXChain { + pub steps: Vec, + pub suggestion: &'static str, +} + +pub fn execute_chain( + el: &AXElement, + caps: &ElementCaps, + chain: &AXChain, +) -> Result<(), AdapterError> { + // 1. Pre-action: AXScrollToVisible (best-effort) + // 2. Pre-action: AXUIElementSetMessagingTimeout(el, 3.0) + // 3. Walk steps, try each with retry on kAXErrorCannotComplete + // 4. If all fail: return error with chain.suggestion +} +``` + +#### Layer 3: Command Definitions (data, not code) + +Each command declares its chain as a static definition. No per-command function bodies needed. + +```rust +// In dispatch.rs or a new chain_defs.rs + +fn click_chain() -> AXChain { + AXChain { + steps: vec![ + PerformAction("AXPress"), + PerformAction("AXConfirm"), + PerformAction("AXOpen"), + PerformAction("AXPick"), + Custom(try_show_alternate_ui), + OnChildren { step: Box::new(PerformAction("AXPress")), limit: 3 }, + SetAttr { attr: "AXSelected", value: AttrValue::Bool(true) }, + OnAncestors { step: Box::new(SetAttr { attr: "AXSelectedRows", .. }), limit: 2 }, + Custom(try_custom_actions), + WithFocus(Box::new(PerformAction("AXPress"))), + Custom(try_keyboard_activate), + OnAncestors { step: Box::new(PerformAction("AXPress")), limit: 2 }, + CGEvent(CGFallback::Click(MouseButton::Left, 1)), + ], + suggestion: "Element may not be interactable. Try 'mouse-click --xy X,Y'.", + } +} + +fn set_value_chain(value: &str) -> AXChain { + AXChain { + steps: vec![ + SetAttr { attr: "AXValue", value: AttrValue::Str(value.into()) }, + WithFocus(Box::new(SetAttr { attr: "AXValue", value: AttrValue::Str(value.into()) })), + ], + suggestion: "Try 'clear' then 'type', or check element is a text field.", + } +} + +fn focus_chain() -> AXChain { + AXChain { + steps: vec![ + SetAttr { attr: "AXFocused", value: AttrValue::Bool(true) }, + PerformAction("AXPress"), // clicking often sets focus as side effect + ], + suggestion: "Try 'click' to focus the element instead.", + } +} + +fn clear_chain() -> AXChain { + AXChain { + steps: vec![ + SetAttr { attr: "AXValue", value: AttrValue::Str(String::new()) }, + WithFocus(Box::new(SetAttr { attr: "AXValue", value: AttrValue::Str(String::new()) })), + Custom(select_all_then_delete), // Cmd+A, Delete via app-targeted keys + ], + suggestion: "Try 'press cmd+a' then 'press delete'.", + } +} + +fn right_click_chain() -> AXChain { + AXChain { + steps: vec![ + PerformAction("AXShowMenu"), + WithFocus(Box::new(PerformAction("AXShowMenu"))), + Custom(try_select_then_show_menu), + OnAncestors { step: Box::new(PerformAction("AXShowMenu")), limit: 3 }, + OnChildren { step: Box::new(PerformAction("AXShowMenu")), limit: 5 }, + CGEvent(CGFallback::Click(MouseButton::Right, 1)), + ], + suggestion: "Try 'mouse-click --button right --xy X,Y'.", + } +} + +fn scroll_to_chain() -> AXChain { + AXChain { + steps: vec![ + PerformAction("AXScrollToVisible"), + Custom(visible_in_scroll_context), + ], + suggestion: "Element may not be in a scrollable container.", + } +} + +fn expand_chain() -> AXChain { + AXChain { + steps: vec![ + PerformAction("AXExpand"), + SetAttr { attr: "AXDisclosing", value: AttrValue::Bool(true) }, + ], + suggestion: "Try 'click' to open it instead.", + } +} + +fn collapse_chain() -> AXChain { + AXChain { + steps: vec![ + PerformAction("AXCollapse"), + SetAttr { attr: "AXDisclosing", value: AttrValue::Bool(false) }, + ], + suggestion: "Try 'click' to close it instead.", + } +} +``` + +### Shared Utilities (extracted from current duplicated code) + +```rust +// New file: crates/macos/src/actions/ax_helpers.rs + +/// Set a boolean attribute on an element. Returns true on success. +pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool + +/// Set a string attribute on an element. Returns true on success. +pub fn set_ax_string(el: &AXElement, attr: &str, value: &str) -> bool + +/// Perform a named AX action. Returns true on success. +pub fn try_ax_action(el: &AXElement, action: &str) -> bool + +/// List all available AX actions on an element. +pub fn list_ax_actions(el: &AXElement) -> Vec +// (replaces both list_ax_actions in activate.rs AND has_ax_action in dispatch.rs) + +/// Try a predicate on each child element, up to limit. +pub fn try_each_child(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool + +/// Try a predicate on ancestors, up to limit. +pub fn try_each_ancestor(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool + +/// Retry an AX operation up to N times on kAXErrorCannotComplete. +pub fn with_retry(f: F, max_retries: u32) -> Result + where F: Fn() -> Result + +/// Set messaging timeout to prevent 6s hangs on unresponsive apps. +pub fn set_messaging_timeout(el: &AXElement, seconds: f32) + +/// Best-effort AXScrollToVisible before acting. +pub fn ensure_visible(el: &AXElement) +``` + +### What This Replaces + +| Current Code | Replaced By | +|-------------|-------------| +| `smart_activate()` 64 LOC | `click_chain()` 15 LOC + chain executor | +| `smart_right_activate()` 30 LOC | `right_click_chain()` 10 LOC + chain executor | +| `smart_double_activate()` 12 LOC | Composing click_chain with count=2 | +| `smart_triple_activate()` 7 LOC | Composing click_chain with count=3 | +| `try_child_activation()` 10 LOC | `OnChildren` step in chain | +| `try_parent_activation()` 13 LOC | `OnAncestors` step in chain | +| `try_child_show_menu()` 8 LOC | `OnChildren` step in right_click_chain | +| `try_parent_show_menu()` 13 LOC | `OnAncestors` step in right_click_chain | +| `is_attr_settable` + set pattern (6 instances) | `set_ax_bool()` / `set_ax_string()` | +| `has_ax_action()` + `list_ax_actions()` (2 files) | Single `list_ax_actions()` in ax_helpers | +| Individual perform_action match arms | Chain definitions (data, not code) | + +### What Stays Custom + +Some commands have truly unique logic that doesn't fit the chain pattern: + +- **scroll** (`ax_scroll`): Its 8-step chain is scroll-specific (scroll bars, page actions, value shifts). Stays as custom function, but uses shared helpers. +- **select** (`select_value`): Role-specific branching (combobox vs popup vs list). Stays custom, uses shared helpers. +- **type**: Mixed AX + keyboard synthesis + non-ASCII clipboard paste. Uses chain for the AX steps, but the keyboard/clipboard fallback is a `Custom` step. +- **hover/drag/key-down/key-up**: Adapter-level (coordinate-based), not element actions. Just need better error suggestions at the element-action level. + +### New File Layout + +``` +crates/macos/src/actions/ +├── mod.rs # re-exports +├── dispatch.rs # perform_action match arms (slimmed down, delegates to chains) +├── chain.rs # ChainStep enum + execute_chain() executor +├── chain_defs.rs # All chain definitions (click, set-value, focus, clear, etc.) +├── discovery.rs # ElementCaps discovery (one-time per action) +├── ax_helpers.rs # Shared AX utilities (set_ax_bool, try_each_child, retry, etc.) +├── activate.rs # REMOVED (absorbed into chain_defs.rs + ax_helpers.rs) +└── extras.rs # select_value + ax_scroll (kept, refactored to use ax_helpers) +``` + +### Benefits + +1. **No redundancy**: Each AX pattern (set attr, perform action, walk children) implemented once +2. **Adding new commands is trivial**: Define a chain (5-15 LOC), no new functions needed +3. **Easy to tune**: Reorder chain steps, add/remove steps without touching executor +4. **Testable**: Can unit-test the executor with mock elements and verify step ordering +5. **Self-documenting**: Chain definitions read like a recipe — "try AXPress, try AXConfirm, try children, fall back to CGEvent" + +### Type Command Special Case + +The `type` command is the most complex because it mixes AX actions with keyboard synthesis: + +```rust +fn type_chain(text: &str) -> AXChain { + AXChain { + steps: vec![ + // Step 1: Append via AX (handles non-ASCII natively) + Custom(|el, caps| ax_append_value(el, text)), + // Step 2: Focus + app-targeted keyboard (ASCII only, or clipboard paste for non-ASCII) + Custom(|el, caps| focus_then_type_to_app(el, caps, text)), + // Step 3: Focus + system-wide keyboard (last resort, needs element focused) + Custom(|el, caps| { + set_ax_bool(el, "AXFocused", true); + synthesize_text(text).is_ok() + }), + ], + suggestion: "Try 'set-value' for direct text insertion.", + } +} +``` + +### hover / drag / key-down / key-up + +These are inherently coordinate-based (adapter-level, not element actions). They don't need AX chains. Just improve error messages in the element-level `perform_action` match: + +```rust +Action::Hover | Action::Drag(_) | Action::KeyDown(_) | Action::KeyUp(_) => { + return Err(AdapterError::new( + ErrorCode::ActionNotSupported, + format!("{} is handled at the adapter level, not as an element action", label), + ) + .with_suggestion(match action { + Action::Hover => "Use 'hover @ref' which resolves ref to coordinates automatically.", + Action::Drag(_) => "Use 'drag --from @ref --to @ref' for ref-based drag.", + Action::KeyDown(_) | Action::KeyUp(_) => "Use 'key-down'/'key-up' commands directly.", + _ => unreachable!(), + })); +} + +## Additional Gaps to Fix + +### 1. AX Messaging Timeout +**Problem:** Unresponsive apps cause 6-second hangs per AX call (Apple default). +**Fix:** Call `AXUIElementSetMessagingTimeout(element, 3.0)` on every resolved element before performing actions. 3 seconds is generous enough for slow apps but prevents indefinite hangs. +**Where:** In `resolve_element_impl()` or at the start of `perform_action()`. + +### 2. Retry on kAXErrorCannotComplete (-25204) +**Problem:** Transient IPC failure treated as terminal. Apple docs say this is a temporary condition. +**Fix:** Wrap AX calls in a retry helper: up to 2 retries with 100ms backoff for `kAXErrorCannotComplete`. Only retry idempotent operations (AXPress, AXSetValue, etc.). +**Where:** New utility function in `actions/` module, used by `perform_action()` and chain steps. + +### 3. Non-ASCII Text Input +**Problem:** `synthesize_text()` silently drops any character without a keycode mapping (all non-ASCII). +**Fix:** For the `type` command chain, step 1 (AXSetValue append) handles non-ASCII natively. If falling back to key synthesis, detect non-ASCII characters and use clipboard-paste: save current clipboard, set clipboard to the non-ASCII segment, Cmd+V, restore clipboard. +**Where:** `input/keyboard.rs` `synthesize_text()` or the type chain in `dispatch.rs`. + +### 4. NSPasteboard FFI for Clipboard +**Problem:** Current clipboard uses `pbpaste`/`pbcopy` shell subprocesses — slow (~50ms per call), fragile, and creates visible process spawns. +**Fix:** Use `NSPasteboard.generalPasteboard` via objc2/Cocoa FFI directly. This is ~10x faster and avoids subprocess overhead. +**Where:** Replace `input/clipboard.rs` implementation. + +### 5. Stale Element Auto-Recovery +**Problem:** When an element ref becomes stale (`STALE_REF`), the agent must manually re-snapshot. Many actions could auto-recover by re-resolving the element. +**Fix:** In `execute_action_impl()`, if `resolve_element` returns `STALE_REF`, attempt one automatic re-resolution using the stored `(pid, role, name)` tuple. If the re-resolved element's bounds are close (within 50px), proceed with the action. Otherwise, return `STALE_REF` as before. +**Caution:** Only do this for single-element actions. Don't auto-recover during batch operations (could lead to wrong-element interactions). +**Where:** `adapter.rs` `execute_action_impl()` wrapper. + +### 6. Post-Action State Hints +**Problem:** After a successful action, the agent has no feedback about what changed. It must re-snapshot to verify. +**Fix:** For certain actions, include a `hint` field in `ActionResult`: +- click/toggle/check/uncheck: read back the element's value/state after action +- set-value: read back value to confirm it was set +- type: read back value to confirm text was appended + +This is optional enrichment — the `data` field can include `{ "post_state": { "value": "new text" } }`. +**Where:** After the action succeeds in `perform_action()`, do one quick attribute read. + +## Key Decisions + +- **Centralized chain architecture**: One executor + declarative step definitions, not per-command functions. Adding a command = defining a chain (5-15 LOC). +- **One-time element discovery**: `ElementCaps` struct queried once per action, shared across all chain steps. Eliminates redundant AX IPC calls. +- **AX-first, explicit physical paths only**: All default CLI ref commands try pure AX approaches and fail clearly rather than silently moving focus or the cursor. +- **Window focus only for explicit physical policy**: Never call `ensure_app_focused()` from the default ref command path. +- **App-targeted keys over system-wide**: `AXUIElementPostKeyboardEvent(app_element, ...)` targets a specific app. System-wide posting is last resort. +- **Non-ASCII via clipboard**: The most reliable cross-app method for typing non-ASCII characters on macOS. +- **3-second AX timeout**: Balances responsiveness with tolerance for slow apps. +- **Single retry for transient errors**: kAXErrorCannotComplete gets one retry with 100ms backoff, handled inside the executor. +- **Stale ref auto-recovery is opt-in**: Only for single-element actions, with bounds proximity check. +- **activate.rs absorbed**: All 400 LOC of activate.rs becomes ~50 LOC of chain definitions + shared helpers. No more per-command fallback functions. + +## Open Questions + +- Should the retry helper also handle `kAXErrorNotImplemented` (-25208) by skipping to next chain step instead of retrying? +- Should post-action state hints be opt-in via a `--verify` flag, or always-on? +- For non-ASCII clipboard paste: should we always restore the previous clipboard, or is that too slow for bulk typing? +- The NSPasteboard FFI change — should we add `objc2` as a dependency, or use raw `objc_msgSend` to keep deps minimal? +- `ChainStep::Custom` closures — use `fn` pointers (no captures, simple) or `Box` (can capture values like text string)? + +## Next Steps + +1. Build the foundation: `ax_helpers.rs` (shared utilities) + `discovery.rs` (ElementCaps) +2. Build `chain.rs` (executor) +3. Migrate `smart_activate` → `click_chain()` definition in `chain_defs.rs` (proves the pattern) +4. Migrate `smart_right_activate` → `right_click_chain()` +5. Add new chains: set-value, focus, clear, scroll-to, expand, collapse +6. Build type chain (complex: AX + keyboard + clipboard) +7. Refactor `ax_scroll` to use shared helpers (stays custom, but less duplicated) +8. Add error suggestions to hover/drag/key-down/key-up +9. Additional gaps: AX timeout, retry, non-ASCII, NSPasteboard, stale recovery, post-action hints diff --git a/docs/brainstorms/2026-02-23-release-automation-brainstorm.md b/docs/brainstorms/2026-02-23-release-automation-brainstorm.md new file mode 100644 index 0000000..b62d05d --- /dev/null +++ b/docs/brainstorms/2026-02-23-release-automation-brainstorm.md @@ -0,0 +1,105 @@ +# Release Automation & npm Distribution Brainstorm + +Date: 2026-02-23 + +## What We're Building + +An automated release pipeline that: +1. Uses **Conventional Commits** to determine version bumps (SemVer) +2. Uses **release-please** to create Release PRs with auto-generated CHANGELOGs +3. On Release PR merge: builds macOS binaries, creates GitHub Releases with tarballs, publishes to npm +4. Distributes via a **single npm package** (`agent-desktop`) with a postinstall script that downloads the correct platform binary from GitHub Releases + +## Why This Approach + +- **release-please** gives a natural gate (Release PR) before shipping, while automating CHANGELOG and version bumps +- **Single npm package + postinstall download** (like agent-browser) — one package to maintain, simpler publishing, npx support works out of the box +- **GitHub Release tarballs** serve dual purpose: npm postinstall downloads from them, and users can also download directly +- **Conventional Commits** already align with the project's imperative-mood commit style (`feat:`, `fix:`, `style:`, `docs:`) + +## Key Decisions + +### Versioning +- **SemVer** (`MAJOR.MINOR.PATCH`), starting from current `0.1.0` +- Single source of truth: `workspace.package.version` in root `Cargo.toml` +- release-please syncs version to both `Cargo.toml` and `package.json` + +### Commit Convention +- `feat:` → minor bump +- `fix:` → patch bump +- `feat!:` or `BREAKING CHANGE:` footer → major bump +- `style:`, `docs:`, `refactor:`, `chore:`, `ci:` → no release (unless paired with feat/fix) + +### npm Distribution Pattern (agent-browser style) + +Single package: `agent-desktop` + +``` +npm install -g agent-desktop # Global install +npx agent-desktop snapshot ... # Quick use via npx +``` + +**How it works:** +1. User installs `agent-desktop` from npm +2. `postinstall` script detects platform (`darwin-arm64`, `darwin-x64`, etc.) +3. Downloads the matching binary from the GitHub Release (`v{version}/agent-desktop-{platform}`) +4. Places it in the package's `bin/` directory +5. A thin JS wrapper (`bin/agent-desktop.js`) resolves and spawns the native binary +6. On global installs, postinstall patches npm's symlink to point directly to the native binary (zero Node.js overhead) + +**Fallback:** If postinstall download fails (corporate firewall, etc.), the JS wrapper shows a helpful error with manual download instructions. + +### GitHub Actions Workflow (new: `release.yml`) + +**Trigger:** Push to `main` + +**Jobs:** +1. **release-please** — creates/updates Release PR, or creates a GitHub Release on PR merge +2. **build** (if release created) — matrix build for each platform target + - `aarch64-apple-darwin` (macOS ARM) + - `x86_64-apple-darwin` (macOS Intel) + - Future: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`, `x86_64-pc-windows-msvc` +3. **publish-github** — attach platform tarballs to the GitHub Release +4. **publish-npm** — sync version to package.json, build JS wrapper, `npm publish` + +### File Structure (new files) + +``` +agent-desktop/ +├── .github/ +│ └── workflows/ +│ ├── ci.yml # Existing (unchanged) +│ └── release.yml # New: release-please + build + publish +├── .release-please-manifest.json # Version tracking for release-please +├── release-please-config.json # release-please configuration +├── npm/ # npm package scaffolding +│ ├── package.json # name: "agent-desktop", bin, postinstall +│ ├── bin/ +│ │ └── agent-desktop.js # JS wrapper: detects platform, spawns native binary +│ └── scripts/ +│ └── postinstall.js # Downloads platform binary from GitHub Release +``` + +### Secrets Required +- `NPM_TOKEN` — npm publish token (stored as GitHub repo secret) +- `GITHUB_TOKEN` — already available in Actions (for release-please and GH Releases) + +### Version Sync Strategy +- release-please bumps version in root `Cargo.toml` (`workspace.package.version`) +- CI script propagates the version to `npm/package.json` before publish +- `Cargo.lock` updated automatically by the version bump + +### Installation Methods (at launch) + +| Method | Command | Notes | +|--------|---------|-------| +| npm (global) | `npm install -g agent-desktop` | Recommended, postinstall downloads native binary | +| npx | `npx agent-desktop snapshot --app Finder -i` | Slightly slower (Node.js wrapper overhead) | +| yarn | `yarn global add agent-desktop` | Same as npm | +| bun | `bun install -g agent-desktop` | May need `--trust` for postinstall scripts | +| Direct download | Download from GitHub Releases | No Node.js required | +| Build from source | `cargo build --release` | For contributors | + +## Open Questions + +None — all key decisions resolved during brainstorm. diff --git a/docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md b/docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md new file mode 100644 index 0000000..9c971a3 --- /dev/null +++ b/docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md @@ -0,0 +1,225 @@ +# Brainstorm: Phase 2 — Windows Adapter + +**Date:** 2026-02-25 +**Status:** Draft +**Phase:** 2 (Windows Adapter) +**Participants:** Lahfir, Claude + +--- + +## What We're Building + +A Windows `PlatformAdapter` implementation that enables all 50 agent-desktop commands to work on Windows via the UI Automation COM API. The adapter uses `uiautomation` (v0.24+) for UIA operations and the `windows` crate for low-level Win32 APIs. It follows the same architecture as the macOS adapter: implements the 22-method `PlatformAdapter` trait, lives entirely within `crates/windows/`, and never touches core or macOS code. + +## Why This Approach + +### Foundation Assessment + +The Phase 1 architecture was purpose-built for this exact expansion: + +- **PlatformAdapter trait**: 22 methods, all with default `not_supported()` returns — enables incremental implementation +- **Core isolation**: Zero platform leaks, CI-enforced via `cargo tree` check +- **Dispatch wiring**: `build_adapter()` and target-gated deps already handle Windows — zero changes needed +- **Windows stub**: Already compiles, implements the trait, and is a workspace member +- **Error codes**: `PlatformNotSupported` and `ActionNotSupported` both available +- **RefMap**: Already handles `USERPROFILE` for Windows paths, has `#[cfg(not(unix))]` fallback + +### Technology Choice: `uiautomation` + `windows` (Layered) + +As specified in the PRD (Section 7.2), using `uiautomation` (v0.24+) for UI Automation operations, complemented by the `windows` crate for low-level Win32 APIs. + +**Why layered, not either/or:** + +| Responsibility | Crate | Rationale | +|---|---|---| +| Tree traversal, CacheRequest, UIA patterns, element finding | `uiautomation` | ~30% less code than raw COM, safe API, auto COM init, escape hatch via `Into` | +| Input synthesis (SendInput), clipboard, screenshot, process lifecycle, window mgmt | `windows` | `uiautomation` doesn't wrap these at the level we need | + +**Key properties of `uiautomation`:** +- v0.24.3 (Jan 2026), 184 stars, 0 open issues, quarterly releases +- Thin wrapper (~9K SLoC) over `windows` crate — no new dependency layer +- Covers all 31 UIA control patterns (Invoke, Value, ExpandCollapse, Selection, Scroll, Toggle, etc.) +- `UITreeWalker` with `UICacheRequest` for batch attribute retrieval +- `UIMatcher` fluent API for element finding and condition building +- Escape hatch: `UIElement: Into` — can drop to raw COM for any edge case +- Auto COM initialization via `UIAutomation::new()` (calls `CoInitializeEx`) + +**Why not raw `windows` for everything:** For UIA operations (tree traversal, patterns, element finding), `uiautomation` eliminates ~330 LOC of unsafe boilerplate, BSTR handling, control type mapping, condition building, and COM factory code. + +**Why not `uiautomation` for everything:** It doesn't wrap `SendInput`, `BitBlt`, `CreateProcessW`, or the Win32 clipboard API at the level agent-desktop needs. + +**Dependency config:** +```toml +# crates/windows/Cargo.toml +[target.'cfg(target_os = "windows")'.dependencies] +uiautomation = { version = "0.24", default-features = false, features = ["pattern", "control"] } +windows = { version = "0.62", features = [ + "Win32_Foundation", + "Win32_System_Threading", + "Win32_System_Com", + "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging", + "Win32_System_DataExchange", + "Win32_Graphics_Gdi", + "Win32_System_Memory", +] } +``` + +## Key Decisions + +### 1. Modifier::Cmd → Ctrl on Windows + +**Decision:** `Cmd` maps to `Ctrl` on Windows. + +**Rationale:** Agents write cross-platform automations. `cmd+c` (macOS copy) should "just work" as `ctrl+c` on Windows. The mapping happens in the Windows adapter's key synthesis layer, not in core. The core `Modifier::Cmd` enum variant stays unchanged. + +### 2. Platform-Specific Blocked Combo Lists + +**Decision:** Each platform adapter defines its own blocked combos. Core provides a validation hook but not the list. + +**Current state:** `BLOCKED_COMBOS` in `crates/core/src/commands/press.rs` is macOS-specific (`cmd+q`, `cmd+shift+q`, etc.). + +**Change needed:** Move the blocked list concept into platform adapters. The Windows adapter blocks: `alt+f4`, `ctrl+alt+delete`, `win+l`, etc. The core keeps a trait method or shared validation function signature, but the actual combos are platform-owned. + +### 3. Surfaces Are Cross-Platform + +**Decision:** `list-surfaces` is fully implementable on Windows. No commands are inherently macOS-only except `pinch`. + +**Windows surface equivalents:** +| macOS Surface | Windows Equivalent | +|---------------|-------------------| +| Menu bar | Win32 menus / UIA Menu pattern | +| Sheet | Modal dialog | +| Alert/Dialog | MessageBox / Task Dialog | +| Popover | Flyout / Popup | + +### 4. `pinch` Command Is macOS-Only + +**Decision:** The PRD's `pinch ` command (Section 5.5) is described as "Pinch zoom gesture (macOS trackpad)". On Windows, this returns `ACTION_NOT_SUPPORTED` with a clear message. Windows does not have trackpad gesture synthesis via accessibility APIs. + +### 5. Implementation Priority: Observation First + +**Decision:** Build observation commands first, then interactions. + +**Priority order:** +1. **Observation tier:** `snapshot`, `list-windows`, `list-apps`, `find`, `get`, `is`, `list-surfaces`, `screenshot` +2. **Core interaction tier:** `click`, `type`, `set-value`, `focus`, `select`, `toggle` +3. **Extended interaction tier:** `double-click`, `triple-click`, `right-click`, `expand`, `collapse`, `check`, `uncheck`, `clear` +4. **Input tier:** `press`, `key-down`, `key-up`, `hover`, `drag`, `mouse-move`, `mouse-click`, `mouse-down`, `mouse-up`, `scroll`, `scroll-to` +5. **System tier:** `launch`, `close-app`, `focus-window`, `resize-window`, `move-window`, `minimize`, `maximize`, `restore`, `clipboard-get`, `clipboard-set`, `clipboard-clear`, `wait` +6. **Meta tier:** `status`, `permissions`, `version`, `batch` (most of these already work via core) + +### 6. Development Workflow: Cross-Compile + Windows CI + +**Decision:** Develop on macOS, cross-compile to verify builds, validate with Windows CI runner. + +**Workflow:** +- Write code on macOS +- Cross-compile with `cargo check --target x86_64-pc-windows-gnu` (GNU target works without MSVC toolchain) +- Windows CI runner (`windows-latest`) executes actual UIA integration tests +- No Windows dev machine required for day-to-day work + +### 7. Zero Core/macOS Changes + +**Non-negotiable constraint:** Phase 2 touches only: +- `crates/windows/` — all new code +- `.github/workflows/ci.yml` — add Windows runner +- `docs/` — documentation updates + +Nothing in `crates/core/`, `crates/macos/`, `crates/linux/`, or `src/` is modified. The one exception is if `BLOCKED_COMBOS` needs to move from core to platform adapters — this is a refactor that benefits all platforms and must be done carefully. + +### 8. COM Threading / Send+Sync Strategy + +**Decision:** Stateless `WindowsAdapter`, per-call COM initialization. + +Both `UIElement` and `IUIAutomationElement` are `!Send, !Sync` — this is fundamental to COM, not a crate limitation. The strategy: +- `WindowsAdapter` stores zero COM state (no `UIAutomation` or `UIElement` fields) +- Each `PlatformAdapter` method creates a `UIAutomation` instance on entry +- All `UIElement` handles are consumed within the method call and dropped before returning +- `NativeHandle` continues to use `unsafe impl Send + Sync` with the raw pointer approach +- For `resolve_element`, store `(pid, control_type, name, bounds_hash)` in `RefEntry` and re-find via `UIMatcher` on each resolve + +This is identical to how the macOS adapter works — `AXUIElementRef` is also `!Send`. + +## Windows UIA Architecture Mapping + +### PlatformAdapter → Windows Implementation + +| Trait Method | Crate | Implementation | +|---|---|---| +| `list_windows` | `uiautomation` + `windows` | `EnumWindows` + `UIAutomation` root element for accessible properties | +| `list_apps` | `windows` | `EnumProcesses` + `OpenProcess` + `QueryFullProcessImageName` | +| `get_tree` | `uiautomation` | `UITreeWalker` with `UICacheRequest` for batch attribute retrieval | +| `execute_action` | `uiautomation` | `UIInvokePattern::invoke()`, `UIValuePattern::set_value()`, etc. | +| `resolve_element` | `uiautomation` | `UIMatcher` re-find by `(pid, control_type, name, bounds_hash)` | +| `check_permissions` | `windows` | COM security / UAC check (UIA doesn't require special perms for most apps) | +| `focus_window` | `windows` | `SetForegroundWindow` + `uiautomation` `UIElement::set_focus()` | +| `launch_app` | `windows` | `CreateProcessW` / `ShellExecuteW` | +| `close_app` | `windows` | `WM_CLOSE` / `TerminateProcess` (force) | +| `screenshot` | `windows` | `BitBlt` / `PrintWindow` via GDI | +| `get_clipboard` | `windows` | `OpenClipboard` + `GetClipboardData(CF_UNICODETEXT)` | +| `set_clipboard` | `windows` | `OpenClipboard` + `SetClipboardData(CF_UNICODETEXT)` | +| `focused_window` | `windows` | `GetForegroundWindow` → `UIAutomation::element_from_handle()` | +| `press_key_for_app` | `windows` | `SendInput` with `KEYBDINPUT` | +| `mouse_event` | `windows` | `SendInput` with `MOUSEINPUT` | +| `drag` | `windows` | `SendInput` sequence: mouse down → move → mouse up | +| `window_op` | `windows` | `ShowWindow` (minimize/maximize/restore), `SetWindowPos` (move/resize) | + +### Windows Crate Folder Structure + +``` +crates/windows/src/ +├── lib.rs # mod declarations + re-exports only +├── adapter.rs # PlatformAdapter trait impl +├── tree/ +│ ├── mod.rs # re-exports +│ ├── element.rs # UIElement wrapper / attribute readers +│ ├── builder.rs # build_subtree via UITreeWalker + UICacheRequest +│ ├── roles.rs # ControlType → unified role mapping +│ ├── resolve.rs # Element re-identification via UIMatcher +│ └── surfaces.rs # Surface detection (menus, dialogs, popups) +├── actions/ +│ ├── mod.rs # re-exports +│ ├── dispatch.rs # Action → UIA Pattern dispatch +│ ├── activate.rs # Window/element activation +│ └── extras.rs # SelectionPattern, ScrollPattern, TogglePattern +├── input/ +│ ├── mod.rs # re-exports +│ ├── keyboard.rs # SendInput keyboard synthesis + Cmd→Ctrl mapping +│ ├── mouse.rs # SendInput mouse synthesis +│ └── clipboard.rs # Win32 clipboard API +└── system/ + ├── mod.rs # re-exports + ├── app_ops.rs # CreateProcessW, TerminateProcess + ├── window_ops.rs # ShowWindow, SetWindowPos, EnumWindows + ├── key_dispatch.rs # App-targeted key press via SendInput + ├── permissions.rs # COM security / UAC check + ├── screenshot.rs # BitBlt / PrintWindow screen capture + └── wait.rs # WaitForInputIdle, polling +``` + +## Pre-Work Required + +Before implementing the adapter logic: + +1. **Move BLOCKED_COMBOS to platform adapters** — refactor the current macOS-specific blocked list out of core +2. **Add Windows CI job** — `windows-latest` runner in CI with appropriate binary size check +3. **Create folder skeleton** — all `mod.rs` files and empty stubs in the structure above +4. **Add `uiautomation` + `windows` dependencies** — target-gated in `crates/windows/Cargo.toml` +5. **Generalize `AdapterError::permission_denied()`** — remove hardcoded macOS suggestion from core + +## Resolved Questions + +1. **UIA crate choice:** Use `uiautomation` (v0.24+) for UIA operations + `windows` crate for Win32 APIs. The PRD specified this, and research confirms it saves ~30% LOC for UIA operations while providing an escape hatch to raw COM. + +2. **COM initialization:** Per-call via `UIAutomation::new()` which auto-calls `CoInitializeEx`. Stateless `WindowsAdapter` with zero stored COM state — same pattern as macOS adapter. + +3. **NativeHandle on Windows:** Store raw pointer from `IUIAutomationElement`. Use `unsafe impl Send + Sync` with documented safety comment, same as macOS. For resolve, re-find via `UIMatcher` using `(pid, control_type, name, bounds_hash)` from `RefEntry`. + +4. **Cross-compilation toolchain:** Use `x86_64-pc-windows-gnu` target for macOS cross-compilation (works without MSVC). Actual UIA integration tests run on Windows CI only. + +5. **Chromium detection:** Surface in `snapshot` output. When snapshot detects a Chromium-based app with an empty/minimal tree, add a warning in the JSON response suggesting `--force-renderer-accessibility`. Agents see the guidance in context when it matters. + +## Open Questions + +None — all questions resolved. diff --git a/docs/brainstorms/2026-02-27-notification-management-brainstorm.md b/docs/brainstorms/2026-02-27-notification-management-brainstorm.md new file mode 100644 index 0000000..cd21dde --- /dev/null +++ b/docs/brainstorms/2026-02-27-notification-management-brainstorm.md @@ -0,0 +1,250 @@ +# Notification Management — macOS First + +> Brainstorm date: 2026-02-27 +> Status: Draft +> Scope: macOS notification commands (list, dismiss, dismiss-all, action, wait) +> Reference: [Phase Roadmap — Notification section](../phases.md) + +--- + +## What We're Building + +A set of notification management commands for agent-desktop, starting with macOS. These commands let AI agents read, dismiss, and interact with OS-level notifications — a capability gap in the current 50-command surface. + +**New commands (5 touch points):** + +| Command | Description | Args | +|---------|-------------|------| +| `list-notifications` | List current notifications with app, title, body, timestamp, actions | `--app`, `--limit` | +| `dismiss-notification` | Dismiss a specific notification by positional index | `` | +| `dismiss-all-notifications` | Clear all notifications, optionally filtered by app | `--app` | +| `notification-action` | Click an action button on a notification | ` ` | +| `wait --notification` | Wait for a notification to appear (event-driven) | `--app`, `--text`, `--timeout` | + +**New domain type:** + +```rust +pub struct NotificationInfo { + pub index: usize, + pub app_name: String, + pub title: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timestamp: Option, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub actions: Vec, +} +``` + +Note: `NotificationInfo` has 6 fields (under the 7-field limit). `is_persistent` from the phases.md spec was dropped — it's not reliably detectable from the AX tree and adds no value for the agent. + +**New adapter trait methods (3):** + +```rust +fn list_notifications(&self, filter: &NotificationFilter) -> Result, AdapterError>; +fn dismiss_notification(&self, index: usize, app_filter: Option<&str>) -> Result<(), AdapterError>; +fn interact_notification(&self, index: usize, action_name: &str) -> Result; +``` + +**New error codes (2):** + +- `NOTIFICATION_NOT_FOUND` — Index out of bounds or notification no longer exists +- `NOTIFICATION_UNSUPPORTED` — Platform doesn't support notification management (stub adapters) + +--- + +## Why This Approach + +**Approach chosen: AX tree traversal + NSDistributedNotificationCenter observer** + +### CRUD Operations — AX Tree + +All list/dismiss/interact operations work by programmatically opening Notification Center, traversing its accessibility tree, and performing actions via AXPress. + +**How it works:** + +1. **Open NC silently:** Use AXUIElement targeting the `com.apple.notificationcenterui` process, or AppleScript `tell application "System Events"` to click the NC clock/date area in the menu bar. The tool opens NC automatically — agents don't need to do this manually. + +2. **Traverse NC AX tree:** The Notification Center window exposes notifications as AXGroup children, each containing: + - AXStaticText elements for title and body + - AXButton elements for actions ("Reply", "Open", app-specific actions) + - AXButton for dismiss/close + +3. **Identify by positional index:** Notifications are numbered 1-based from top to bottom (newest first). The agent calls `list-notifications` to see the current state, then uses the index for dismiss/interact. This mirrors the snapshot → ref → action pattern — the index is snapshot-scoped, not stable across calls. + +4. **Close NC after operation:** After reading or acting, close NC to restore the user's screen state. Use AXPress on the NC toggle or send Escape key. + +**Why not other approaches:** +- **SQLite DB approach** requires Full Disk Access permission — unacceptable adoption barrier on top of the existing Accessibility permission requirement. +- **Pure polling for wait** wastes resources and has minimum ~500ms granularity due to NC open/close visual flicker. + +### Wait — NSDistributedNotificationCenter + +`wait --notification` uses `NSDistributedNotificationCenter` to listen for system-level notification events. This is event-driven — instant detection, no polling, no visual impact. + +**How it works:** + +1. Register an observer on `NSDistributedNotificationCenter.defaultCenter()` for notification-related events +2. When a notification arrives, check against the filter criteria (`--app`, `--text`) +3. If matched, open NC briefly to read full notification details and return them +4. If `--timeout` exceeded, return a TIMEOUT error + +**Fallback:** If the distributed notification mechanism fails (undocumented API changed in a new macOS version), fall back to AX polling with a 1-second interval. + +--- + +## Key Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Platform scope | macOS first | Matches Phase 1 additive pattern. Windows/Linux follow in their respective phases. | +| NC access | Auto-open silently | Agents shouldn't need to manually open NC. The tool handles it transparently. | +| Notification identity | Positional index (1-based) | No stable IDs exist. Index is simple, matches the snapshot-scoped ref pattern. Agent must list before acting. | +| Wait mechanism | NSDistributedNotificationCenter + AX fallback | Event-driven for responsiveness. Polling fallback for resilience. | +| System tray | Separate feature | Different AX targets, different design questions. Brainstorm independently. | +| `is_persistent` field | Dropped | Not reliably detectable from AX tree. No agent use case identified. | +| `NotificationFilter` | App name + text content | `--app` filters by source app, `--text` filters by title/body substring. `--limit` caps result count. | +| DND detection | Include as metadata | `list-notifications` response includes a `do_not_disturb: bool` field so agents know if notifications are suppressed. Read via CoreFoundation preferences. | +| NC close behavior | Always close after operation | Prevents NC from staying open and blocking the user's view. Brief flash is acceptable. | + +--- + +## Resolved Questions + +1. **AX tree structure stability:** Use heuristic role/attribute matching (find AXGroup with AXStaticText children) rather than hardcoded tree paths. More resilient to macOS version changes. No version-specific parsers. + +2. **Grouped notifications:** Flat list ordered by recency. App name is on each notification. Agents don't need to know about NC's grouping structure. + +3. **Notification banners vs NC:** NC only. Banners are transient and require a separate AX tree target. Keeping scope to Notification Center contents is sufficient for v1. + +--- + +## JSON Output Examples + +### list-notifications + +```json +{ + "version": "1.0", + "ok": true, + "command": "list-notifications", + "data": { + "count": 3, + "do_not_disturb": false, + "notifications": [ + { + "index": 1, + "app_name": "Messages", + "title": "John Doe", + "body": "Hey, are you free for lunch?", + "timestamp": "2026-02-27T10:30:00", + "actions": ["Reply", "Open"] + }, + { + "index": 2, + "app_name": "Calendar", + "title": "Team Standup in 15 minutes", + "actions": ["Join", "Snooze"] + }, + { + "index": 3, + "app_name": "Slack", + "title": "#engineering", + "body": "New message from Alice", + "actions": ["Open"] + } + ] + } +} +``` + +### dismiss-notification + +```json +{ + "version": "1.0", + "ok": true, + "command": "dismiss-notification", + "data": { + "dismissed_index": 2, + "app_name": "Calendar", + "title": "Team Standup in 15 minutes" + } +} +``` + +### notification-action + +```json +{ + "version": "1.0", + "ok": true, + "command": "notification-action", + "data": { + "index": 1, + "app_name": "Messages", + "action": "Reply", + "result": "performed" + } +} +``` + +### wait --notification (success) + +```json +{ + "version": "1.0", + "ok": true, + "command": "wait", + "data": { + "condition": "notification", + "matched": true, + "notification": { + "index": 1, + "app_name": "Messages", + "title": "John Doe", + "body": "Hey, are you free for lunch?", + "actions": ["Reply", "Open"] + } + } +} +``` + +--- + +## macOS Implementation Outline + +### New files + +``` +crates/core/src/ +├── notification.rs # NotificationInfo, NotificationFilter structs + +crates/core/src/commands/ +├── list_notifications.rs # list-notifications handler +├── dismiss_notification.rs # dismiss-notification handler +├── dismiss_all_notifications.rs # dismiss-all-notifications handler +└── notification_action.rs # notification-action handler + +crates/macos/src/notifications/ +├── mod.rs # re-exports +├── list.rs # Open NC → traverse AX tree → parse notifications → close NC +├── dismiss.rs # Open NC → find by index → AXPress close button → close NC +├── interact.rs # Open NC → find by index → find action button → AXPress → close NC +├── nc_control.rs # Open/close Notification Center helpers +└── observer.rs # NSDistributedNotificationCenter observer for wait +``` + +### Registration points (existing files to modify) + +- `crates/core/src/lib.rs` — add `pub mod notification` +- `crates/core/src/adapter.rs` — add 3 trait methods with `not_supported` defaults +- `crates/core/src/error.rs` — add `NotificationNotFound`, `NotificationUnsupported` variants +- `crates/core/src/commands/mod.rs` — add 4 `pub mod` declarations +- `crates/macos/src/lib.rs` — add `pub mod notifications` +- `crates/macos/src/adapter.rs` — implement 3 trait methods delegating to notifications module +- `src/cli.rs` — add 4 command variants +- `src/cli_args.rs` — add arg structs +- `src/dispatch.rs` — add 4 match arms +- `src/dispatch.rs` — extend `wait` command match arm for `--notification` flag diff --git a/docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md b/docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md new file mode 100644 index 0000000..2090363 --- /dev/null +++ b/docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md @@ -0,0 +1,65 @@ +# Electron Tree Compaction Brainstorm + +**Date:** 2026-03-01 +**Status:** Decided +**Trigger:** Electron depth-skip fix (commit a19c1b5) made Slack snapshots work but raised concern about context bloat for AI agents. + +## Problem Statement + +After enabling Electron/web app support via depth-skip for non-semantic `AXGroup` wrappers, Slack snapshots with `-i` return **268 nodes / ~6,700 tokens** — 4x the cost of Finder's 96 nodes / ~1,625 tokens. 40% of Slack's nodes are structural wrappers carrying no useful info. + +The question: should we build a smart architecture (e.g., a standalone Electron-to-AX bridge) to reduce this, or is there a simpler solution? + +## Data + +| Mode | Nodes | Refs | Tokens | Notes | +|------|-------|------|--------|-------| +| Slack `-i` (current) | 268 | 162 | ~6,700 | 40% structural noise | +| Slack `-i` + compact | 204 | 162 | ~5,475 | 18% savings | +| Slack full (no `-i`) | 528 | 162 | ~12,000 | 69% structural noise | +| Finder `-i` | 96 | 73 | ~1,625 | 24% structural (baseline) | + +Snapshot speed: Slack ~220ms, Finder ~260ms. **Speed is not an issue.** + +## What We're Building + +**Implement `--compact` flag** (already wired in CLI as a no-op) to collapse single-child non-interactive pass-through nodes during tree serialization. + +**Collapse rule:** If a node has no `ref_id`, no `name`, no `value`, and exactly one child — skip it and hoist the child. If it has a `name` but the child doesn't, promote the name to the child and skip. + +This is a tree serialization optimization, not an Electron-specific feature. Benefits all apps. + +## Why This Approach + +1. **The flag already exists** — `--compact` is wired through CLI, `TreeOptions`, and the snapshot pipeline. Just needs implementation. +2. **18% token savings for free** — 64 nodes removed from Slack with zero information loss. +3. **No separate package needed** — The Electron bridge idea (CDP-based, standalone crate) is YAGNI. The actual token cost is dominated by real content (162 interactive elements with descriptive names), not structural noise. +4. **Composable with `--surface`** — Agents that need even smaller snapshots should scope to a surface (sheet, menu, alert) rather than us over-pruning the full tree. + +## Approaches Considered + +### 1. Implement `--compact` with chain collapsing (CHOSEN) +- Collapse unnamed/valueless single-child non-interactive nodes +- 15-18% token reduction, zero information loss, trivial to implement +- Works for all apps, not Electron-specific + +### 2. Standalone Electron-to-AX bridge package +- Separate crate that connects via CDP, extracts real DOM structure, maps to AX tree +- Would require `--remote-debugging-port` (invasive), becomes agent-browser territory +- **Rejected:** YAGNI. Pure AX approach already works. Adds dependency and complexity. + +### 3. Flat interactive-only output (`--flat`) +- Output just refs as `[{ref, role, name, value, path}]` — no tree +- Minimal tokens (~2,000 for Slack) but agents lose spatial/hierarchy context +- **Deferred:** Could add later if compact isn't enough. Not needed now. + +## Key Decisions + +- **Chain collapsing preserves tree structure** — agents still see hierarchy, just without empty wrapper noise +- **Not Electron-specific** — universal tree optimization via `--compact` flag +- **No separate bridge package** — unnecessary given 220ms speed and ~5,475 post-compact tokens +- **Agents control scope via `--surface`** — surface-scoped snapshots are the right lever for smaller output, not tree pruning + +## Open Questions + +None — approach is clear and implementation is straightforward. diff --git a/docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md b/docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md new file mode 100644 index 0000000..ac5bb19 --- /dev/null +++ b/docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md @@ -0,0 +1,185 @@ +# ClawHub Skill Publishing & Scalable Skill Architecture Brainstorm + +**Date:** 2026-03-02 +**Status:** Decided +**Trigger:** agent-desktop has a mature skill ready for public distribution. Need a scalable architecture for multi-platform, multi-app skill publishing to ClawHub. + +## Problem Statement + +The `agent-desktop` skill exists locally in `skills/agent-desktop/` with a SKILL.md and 5 reference files. A separate `agent-desktop-macos` skill lives in `.claude/skills/` (not git-tracked). As we add Windows/Linux platforms and app-specific knowledge (Slack, Notion, VS Code quirks), we need: + +1. A **scalable directory structure** under `skills/` that supports platform and app-specific sub-skills +2. **Automated CI publishing** to ClawHub on every release (no manual steps) +3. A **scaffolding pattern** so adding a new platform or app skill is trivial + +## Data + +| Metric | Value | +|--------|-------| +| Current skill files | 1 SKILL.md + 5 reference docs + 1 platform SKILL.md | +| Total content | ~42KB across all files | +| Commands documented | 54 | +| ClawHub skills total | 5,700+ | +| Desktop automation skills on ClawHub | Very few (niche) | +| ClawHub CLI batch command | `clawhub sync --root skills/ --all` | + +## What We're Building + +A **nested skill hierarchy** where platform skills contain app-specific references, with CI-driven auto-publishing to ClawHub on every release. + +### Directory Structure + +``` +skills/ +├── agent-desktop/ # Core skill (platform-agnostic) +│ ├── SKILL.md # Commands, observe-act loop, ref system, JSON contract +│ └── references/ +│ ├── commands-observation.md +│ ├── commands-interaction.md +│ ├── commands-system.md +│ └── workflows.md # 12 common automation patterns +│ +├── agent-desktop-macos/ # macOS platform skill +│ ├── SKILL.md # TCC permissions, AX API, smart activation chain, surfaces +│ └── references/ +│ ├── slack.md # Slack on macOS: Electron quirks, clipboard paste pattern +│ ├── notion.md # Notion on macOS: block navigation, slash commands +│ ├── vscode.md # VS Code on macOS: command palette, editor interaction +│ └── notifications.md # NC lifecycle, dismiss strategies, stacked notifications +│ +├── agent-desktop-windows/ # Windows platform skill (Phase 2) +│ ├── SKILL.md # UIA setup, permission model, COM automation +│ └── references/ +│ ├── slack.md # Slack on Windows +│ ├── teams.md # Teams on Windows (UIA-native) +│ └── vscode.md # VS Code on Windows +│ +└── agent-desktop-linux/ # Linux platform skill (Phase 3) + ├── SKILL.md # AT-SPI setup, D-Bus, Wayland vs X11 + └── references/ + ├── slack.md # Slack on Linux + └── vscode.md # VS Code on Linux +``` + +### Key design principles + +1. **App quirks are platform-scoped** — Slack on macOS (AX + Electron depth-skip) differs from Slack on Windows (UIA + Chromium detection). App references live inside the platform skill, not in a separate top-level skill. + +2. **Core skill stays platform-agnostic** — `agent-desktop/` documents commands, JSON contract, ref system. Zero platform detail. The `references/macos.md` that currently lives here moves to `agent-desktop-macos/SKILL.md`. + +3. **Each `skills/*/` folder is independently publishable** — ClawHub `clawhub sync --root skills/ --all` discovers and publishes each subfolder as a separate skill. + +4. **Git-tracked source of truth** — everything under `skills/` is committed. The `.claude/skills/` and `.agents/skills/` symlinks are derived (gitignored). + +## Why This Approach + +1. **Matches how the project actually scales** — each platform adapter (Phase 2, 3) brings platform-specific AX behavior AND app-specific quirks. The skill hierarchy mirrors the crate hierarchy. + +2. **App knowledge compounds** — every time we test Slack/Notion/VS Code, we document quirks in the platform's `references/` folder. Future agents benefit automatically. + +3. **`clawhub sync` handles multi-skill publishing** — one CI step publishes all skills in `skills/`. No per-skill config. Adding a new skill = adding a new folder. + +4. **Fewer top-level skills** — users install `agent-desktop` (core) + `agent-desktop-macos` (their platform). App knowledge comes free inside the platform skill. No need to install `agent-desktop-slack` separately. + +## Approaches Considered + +### 1. Flat app-specific skills (agent-desktop-slack, agent-desktop-notion, etc.) +- One top-level skill per app, published separately +- **Rejected:** App behavior is platform-dependent. Slack on macOS vs Windows differs fundamentally. Flat app skills would either duplicate platform info or be incomplete. + +### 2. Nested hierarchy — platform > apps (CHOSEN) +- Platform skills contain app-specific references +- App knowledge scoped to platform where it's actually useful +- Fewer skills to install, more content per skill + +### 3. Monolithic single skill with everything +- One massive `agent-desktop` skill with all platform + app info +- **Rejected:** Too much context loaded at once. Users on macOS don't need Windows/Linux info in context. + +## CI Automation + +Add a `publish-skills` job to `.github/workflows/release.yml`: + +```yaml +publish-skills: + needs: [release-please] + if: needs.release-please.outputs.release_created == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm i -g clawhub + - run: | + clawhub sync \ + --root skills/ \ + --all \ + --bump patch \ + --changelog "${{ needs.release-please.outputs.tag_name }}" + env: + CLAWHUB_TOKEN: ${{ secrets.CLAWHUB_TOKEN }} +``` + +This runs automatically when release-please creates a new release. All skills under `skills/` are published in one step. + +### Symlink Automation + +Add a post-checkout script or Makefile target to wire `.claude/skills/`: + +```bash +# scripts/link-skills.sh +for skill_dir in skills/*/; do + name=$(basename "$skill_dir") + mkdir -p .claude/skills + ln -sfn "../../$skill_dir" ".claude/skills/$name" +done +``` + +## Key Decisions + +- **Nested hierarchy (platform > apps)** — app references scoped to platform +- **CI auto-publish via `clawhub sync`** on every release — zero manual steps +- **Move `references/macos.md` from core skill to `agent-desktop-macos/`** — core stays platform-agnostic +- **Move `agent-desktop-macos` from `.claude/skills/` to `skills/`** — git-tracked, publishable +- **Version synced to npm/Cargo version** via release-please +- **`CLAWHUB_TOKEN` as GitHub secret** for CI authentication +- **Symlink script** to wire `skills/` → `.claude/skills/` for local development + +## Scaffolding: Adding a New App Reference + +To add Slack macOS knowledge: + +1. Create `skills/agent-desktop-macos/references/slack.md` +2. Document: Electron quirks, clipboard paste pattern, block structure, known AX issues +3. Reference it from `skills/agent-desktop-macos/SKILL.md` reference table +4. Commit → next release auto-publishes to ClawHub + +To add a new platform (e.g., Windows): + +1. Create `skills/agent-desktop-windows/SKILL.md` with platform-specific frontmatter +2. Create `skills/agent-desktop-windows/references/` for app-specific docs +3. Run `scripts/link-skills.sh` to wire local symlinks +4. Commit → next release auto-publishes + +## Migration Steps (from current state) + +1. Move `.claude/skills/agent-desktop-macos/SKILL.md` → `skills/agent-desktop-macos/SKILL.md` +2. Move `skills/agent-desktop/references/macos.md` → merge into `skills/agent-desktop-macos/SKILL.md` +3. Remove `macos.md` from core skill's reference table in SKILL.md +4. Add ClawHub metadata (`version`, `tags`, `requirements`) to all SKILL.md frontmatters +5. Create `scripts/link-skills.sh` for local symlink management +6. Add `publish-skills` job to `release.yml` +7. Store `CLAWHUB_TOKEN` in GitHub repo secrets +8. First publish: manual `clawhub sync --root skills/ --all` to verify +9. After verification: CI handles all future publishes + +## Open Questions + +None — architecture is clear. + +## Sources + +- [ClawHub Docs](https://docs.openclaw.ai/tools/clawhub) — publish CLI, sync command, token management +- [ClawHub GitHub](https://github.com/openclaw/clawhub) — skill directory structure +- [Anthropic Skills Repo](https://github.com/anthropics/skills) — SKILL.md format standard +- [ClawHub Publisher Skill](https://playbooks.com/skills/openclaw/skills/openclaw-clawhub-publisher) — batch publish, CI integration diff --git a/docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md b/docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md new file mode 100644 index 0000000..2fcfac2 --- /dev/null +++ b/docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md @@ -0,0 +1,209 @@ +# Progressive Skeleton Traversal + +**Date:** 2026-03-10 +**Status:** Brainstorm +**Author:** Lahfir + +## What We're Building + +A two-phase tree traversal system that lets AI agents explore dense accessibility trees incrementally instead of consuming the entire tree at once. The system introduces: + +1. **Skeleton mode** — A shallow overview snapshot (2-3 levels deep) where truncated nodes show `children_count` instead of full subtrees. Named containers at the truncation boundary receive refs so agents can drill into them. + +2. **Ref-rooted subtree** — A `--root @ref` flag on `snapshot` that starts traversal from a previously-discovered element instead of the window root. Combined with refmap merging, this lets agents accumulate knowledge across multiple targeted drill-downs. + +3. **Epoch-tagged refmap merging** — Instead of replacing the entire refmap on each snapshot, drill-down snapshots merge new refs into the existing map. A monotonic epoch counter and root-ref tagging enable staleness detection and scoped invalidation. + +### The Problem + +Dense Electron apps (Slack, VS Code, Discord) produce massive accessibility trees — 500+ nodes, 12,000+ tokens per full snapshot. Even with `--interactive-only --compact`, Slack still produces ~200 nodes (~5,400 tokens). Agents pay per-token and struggle to navigate these trees efficiently. + +Current optimization layers (depth-skip, compact, surfaces, interactive-only) reduce output size but still require full tree traversal. There's no way to say "I only care about the sidebar right now." + +### Target Workflow + +```bash +# Step 1: Cheap overview of the entire app (~500 tokens) +agent-desktop snapshot --app Slack --skeleton -i +# Returns 2-3 levels with children_count on truncated containers + +# Step 2: Agent identifies sidebar (@e3) as interesting, drills in (~800 tokens) +agent-desktop snapshot --root @e3 -i --compact +# Returns @e3's full subtree, merges refs into existing refmap + +# Step 3: Agent identifies a specific channel list (@e12), drills deeper (~400 tokens) +agent-desktop snapshot --root @e12 -i --compact +# Returns @e12's subtree, merges again + +# Step 4: Agent acts on a discovered element +agent-desktop click @e45 +# Uses ref from step 3 — still valid because refmap accumulated + +# Total: ~1,700 tokens across 3 snapshots vs ~5,400 tokens for one full snapshot +# Plus: agent only processed relevant information at each step +``` + +## Why This Approach + +### Why not Semantic Zone Auto-Detection? + +Zone detection (auto-labeling "sidebar", "toolbar", "content" via role heuristics) is appealing but fragile: +- Not all apps use standard accessibility roles for layout +- Platform differences in role semantics make cross-platform heuristics unreliable +- Introduces a new concept (zones) alongside existing refs +- Can be layered on top of Progressive Skeleton later as sugar + +### Why not Query-Driven Traversal? + +Targeted search (`find --near "Channels"`) is token-efficient but requires agents to know what they're looking for. It fails at the "orient myself in an unfamiliar app" use case — which is the most common first step. + +### Why Progressive Skeleton wins + +1. **Composable** — Works with ALL existing flags (`--compact`, `--interactive-only`, `--max-depth`, `--surface`) +2. **Platform-agnostic** — Just new `TreeOptions` fields; each adapter implements the same interface +3. **Agent-controlled** — Agents decide exploration strategy based on their goals +4. **Familiar pattern** — Mirrors tree exploration in file systems (expand on demand) +5. **Minimal API surface** — Two new flags (`--skeleton`, `--root`), one new refmap behavior (merge) +6. **Incremental value** — Each feature is independently useful; skeleton without drill-down still saves tokens + +## Key Decisions + +### 1. Skeleton Output Format + +Truncated nodes show `children_count` (direct child count) instead of full children arrays: + +```json +{ + "role": "group", + "name": "Channels", + "ref": "@e3", + "children_count": 47 +} +``` + +**Rationale:** Child count is O(1) to fetch on all platforms (it's a property of the element, not requiring subtree traversal). Gives agents enough signal to prioritize drill-downs. More detailed stats (interactive count, role distribution) would require full traversal, defeating the purpose. + +### 2. Skeleton Ref Assignment + +At the skeleton boundary, **named containers with children** receive refs even if they're non-interactive. These are "drill-down targets" — the agent uses them with `--root @ref` to explore deeper. + +Rules for skeleton ref assignment: +- Interactive elements: always get refs (existing behavior) +- Named containers at truncation boundary with `children_count > 0`: get refs (NEW) +- Anonymous wrappers (no name, no value): never get refs (existing behavior) + +### 3. RefMap Merge Strategy + +Each snapshot call increments a monotonic `epoch` counter stored in the refmap file. RefEntries are tagged with: + +- `epoch: u32` — when this ref was created +- `root_ref: Option` — which drill-down root created this ref (`None` for skeleton/full snapshots) + +**Merge behavior:** +- **Full/skeleton snapshot** (`--skeleton` or no `--root`): replaces entire refmap (current behavior). Resets epoch. +- **Drill-down** (`--root @e3`): loads existing refmap, removes all refs where `root_ref == "@e3"` (clear stale subtree), increments epoch, adds new refs with `root_ref: "@e3"`, saves merged refmap. + +**Staleness detection:** +- On `STALE_REF`, error includes epoch info for recovery hints +- Agent can compare epochs across refs to detect relative freshness +- Re-drilling a region automatically invalidates only that region's refs + +### 4. `--root` Implies Merge + +When `--root @ref` is used, merge mode is automatic — there's no separate `--merge` flag. Rationale: drilling into a subtree without merging would discard all other refs, which is never what an agent wants. + +### 5. Platform-Agnostic Design + +New `PlatformAdapter` trait method: + +```rust +fn get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions) -> Result { + Err(AdapterError::not_supported("get_subtree")) +} +``` + +Each platform adapter implements this by: +- macOS: extract `AXUIElementRef` from handle, call `build_subtree()` from that element +- Windows (Phase 2): extract `IUIAutomationElement` from handle, walk with TreeWalker from that element +- Linux (Phase 3): extract AT-SPI accessible from handle, walk from that element + +The `TreeOptions` struct gains: +```rust +pub struct TreeOptions { + pub max_depth: u8, + pub include_bounds: bool, + pub interactive_only: bool, + pub compact: bool, + pub surface: SnapshotSurface, + pub skeleton: bool, // NEW: shallow overview with children_count + pub root_ref: Option, // NEW: start from this ref instead of window root +} +``` + +### 6. Skeleton Depth + +Skeleton mode sets an internal depth limit independent of `--max-depth`. Default skeleton depth: **3 levels** from the root (or from `--root` element). + +- Level 0: Root element (window or ref target) +- Level 1: Major sections (toolbar, sidebar, content) +- Level 2: Sub-sections (channel list, message list) +- Level 3: Individual groups/items (truncated here with `children_count`) + +Agent can override: `--skeleton --max-depth 2` for even shallower overview. + +## Token Budget Analysis + +Based on real Slack data (528 nodes full tree): + +| Mode | Nodes | Est. Tokens | Savings vs Full | +|------|-------|-------------|-----------------| +| Full snapshot | 528 | ~12,000 | baseline | +| + interactive-only | 268 | ~6,700 | 44% | +| + compact | 204 | ~5,400 | 55% | +| **Skeleton (depth 3)** | ~25-40 | ~500-800 | **93-96%** | +| Drill-down (sidebar) | ~30-60 | ~600-1,200 | scoped | +| Drill-down (content) | ~50-100 | ~1,000-2,000 | scoped | +| **Skeleton + 1 drill-down** | ~55-100 | ~1,100-2,000 | **83-91%** | +| **Skeleton + 2 drill-downs** | ~85-160 | ~1,700-3,200 | **73-86%** | + +Even with 2 targeted drill-downs, agents use 1,700-3,200 tokens instead of 5,400. And they only process relevant information at each step, improving reasoning quality. + +## Implementation Scope + +### Core Changes +- `TreeOptions`: add `skeleton: bool`, `root_ref: Option` +- `RefMap` / `RefEntry`: add `epoch: u32`, `root_ref: Option` fields +- `snapshot.rs`: skeleton mode in `allocate_refs()` (assign refs to named containers, replace children with `children_count`) +- `snapshot.rs`: merge logic in `run()` (load existing refmap, scoped invalidation, merge) +- `AccessibilityNode`: add `children_count: Option` field with serde skip-if-none +- `PlatformAdapter`: add `get_subtree()` method with default `not_supported` + +### macOS Changes +- `adapter.rs`: implement `get_subtree()` — resolve handle to AXElement, call `build_subtree()` +- `builder.rs`: support `skeleton` flag in `build_subtree()` — at depth limit, count children instead of recursing + +### CLI Changes +- `cli_args.rs`: add `--skeleton` and `--root` flags to `SnapshotArgs` +- `dispatch.rs`: pass new args through to `TreeOptions` + +### find Command Changes +- `find.rs`: add `--root @ref` support — resolve ref, build subtree from it, then search in-memory +- `cli_args.rs`: add `--root` flag to `FindArgs` + +### No Changes Needed +- `resolve.rs` — existing resolution logic works as-is +- `get.rs` — works with merged refmaps transparently +- All action commands — work with any valid ref from the merged map +- `compact`, `interactive-only` — compose naturally with skeleton/drill-down + +## Resolved Questions + +1. **Should `find` support `--root @ref` too?** **Yes.** Agents can search within a drill-down region: `find --root @e3 --role button`. Low effort since find already operates on in-memory trees — just change where the tree comes from. + +2. **What happens when the skeleton root ref itself is stale?** **Return STALE_REF, no auto-recovery.** The suggestion tells the agent to re-run skeleton. Auto-fallback hides UI state changes the agent should know about. Agents already handle STALE_REF. + +3. **Should `children_count` count ALL descendants or just direct children?** **Direct children only.** O(1) on all platforms — it's an attribute of the element. Agent sees "this group has 47 direct children" which is sufficient to judge density. Total descendant count would require subtree traversal, defeating the purpose. + +4. **Naming: `--skeleton` vs `--overview` vs `--shallow`?** **`--skeleton`.** Most descriptive — "shows structure without flesh." Technical but precise. Matches the pattern of other flags. + +5. **Should skeleton be the default when `--compact --interactive-only` are both set?** **No, always opt-in.** Agents explicitly choose skeleton mode. Existing behavior unchanged. No surprise breakage for agents already using snapshot. diff --git a/docs/brainstorms/2026-04-24-phase2-windows-crossplatform-requirements.md b/docs/brainstorms/2026-04-24-phase2-windows-crossplatform-requirements.md new file mode 100644 index 0000000..28e1c45 --- /dev/null +++ b/docs/brainstorms/2026-04-24-phase2-windows-crossplatform-requirements.md @@ -0,0 +1,119 @@ +--- +date: 2026-04-24 +topic: phase2-windows-crossplatform +--- + +# Phase 2 Windows + Cross-Platform Parity Requirements + +## Problem Frame + +agent-desktop is shipping as a strong macOS + FFI tool, but it is not yet a credible cross-platform automation runtime. Phase 2 must bring Windows online without weakening the core contract: agents observe and act through structured refs, not through foregrounded UI poking. + +The next phase is larger than "add Windows." It also closes the parity gaps exposed by the FFI release: unstable element identity, polling-based waits, missing text-range operations, slower screenshots, incomplete permission reporting, and command-surface duplication risk. The goal is one coherent v0.2.0 release that makes Windows first-class while strengthening macOS in the same primitives. + +The authoritative implementation plan already exists at `docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md`. This brief captures the product and architecture decisions that should guide execution. + +## Release Gates + +Phase 2 remains one v0.2.0 GA milestone, but execution should use internal gates so Windows value does not get buried under unrelated breadth: + +- G1. Windows existing-command parity: the current macOS command surface works on Windows for Explorer, Notepad, Settings, VS Code, and Edge. +- G2. Headless contract: non-mouse commands on both macOS and Windows either use accessibility/headless APIs or return structured unsupported/action errors with guidance. Cursor or focus fallbacks are reserved for explicit mouse/window-focus commands. +- G3. Cross-platform primitives: stable selectors, event-backed waits, text ranges, modern screenshots, richer permission/error reporting, and command registration parity are implemented where the execution-prep spikes validate the API path. +- G4. Release readiness: Windows CI, Windows x86_64 and ARM64 artifacts, npm platform selection, FFI ABI handshaking, README, skills, and roadmap sync are complete. + +G1 is the Windows-first MVP gate. G2-G4 decide whether the work is ready to be called Phase 2 GA rather than an internal checkpoint or prerelease. + +## Requirements + +- R1. Windows must support the existing command surface with the same JSON envelope and error contract as macOS. +- R2. Cross-platform parity is structural, not byte-identical: roles, refs, available actions, and stable identifiers must be comparable across equivalent apps while preserving platform-specific truth. +- R3. Headless-first behavior is non-negotiable: non-mouse commands must avoid focus steal, visible activation, and physical cursor movement. Existing macOS focus/cursor fallbacks must be migrated, gated behind explicit mouse/focus commands, or replaced with structured unsupported/action errors. +- R4. Windows skeleton traversal and ref-rooted drill-down must preserve the existing progressive traversal contract, including depth-3 skeletons, `children_count`, scoped invalidation, and the refmap size guard. +- R5. Stable selector fields must be added and used to reduce `STALE_REF` churn, especially in Electron, WebView2, localized, and custom-rendered apps. +- R6. Push-based event watching must complement polling waits through the public CLI shape `wait --event --ref @e... --timeout `. `watch_element` is the adapter primitive behind that command surface, not a separate Phase 2 CLI command. +- R7. Text range primitives must support selection, caret reads, range reads, and insert-at-caret on both macOS and Windows. +- R8. Modern per-window screenshot APIs must become the default on both platforms, with legacy paths retained as explicit fallbacks. +- R9. New action coverage must include `SelectRange` and `InsertAtCaret` for text workflows, plus `ShowMenu`, `WindowRaise`, `Cancel`, `DeliverFiles`, `LongPress`, and `ForceClick` only where the unit can prove a headless or platform-native path. If a variant has no native path on a platform, it must return a structured unsupported/action error rather than violating R3. +- R10. Missing surfaces and OS integration points must be promoted only when tied to Phase 2 workflows: Toolbar on both platforms for browser/editor automation; Windows SystemTray and notification parity for OS-level agent workflows; Spotlight, Dock, and MenuBarExtras on macOS as parity backfills only if their unit has a concrete target workflow and acceptance test. +- R11. Permission and error reporting must distinguish accessibility denial, permission revocation, screen-recording denial, automation denial, resource exhaustion, and AX messaging timeouts. +- R12. Command registration and FFI parity must use deterministic `build.rs` filesystem enumeration, not `inventory` or `linkme`, so one command file remains the durable source of truth across CLI, FFI, and future MCP. +- R13. Phase 2 must ship as v0.2.0 with Windows CI, Windows release artifacts, npm platform selection, FFI ABI handshaking, and updated skills/docs. + +## Success Criteria + +- `snapshot --app Explorer`, Notepad, Settings, VS Code, and Edge returns usable trees with refs on Windows. +- All existing commands run through Windows CI, and core still has zero platform crate leakage. +- Non-mouse command integration tests assert no focus change and no cursor movement; any fallback that would move focus/cursor returns a structured error with recovery guidance instead. +- Stable selectors measurably reduce stale-ref rate versus the Phase 1 resolver baseline. +- `wait --event value-changed --ref @e... --timeout 3000` receives events within 500 ms on macOS and Windows after the event-threading spike validates the path. +- Modern screenshot becomes the default after backend-specific benchmarks validate the threshold; legacy remains selectable. Do not hard-code a sub-50ms cold target before the screenshot spike because current ScreenCaptureKit crate docs report broader first-frame latency ranges. +- R9 action variants each have a target workflow, native/headless implementation path, or documented structured unsupported result before their unit opens. +- R10 surfaces each have a target workflow and acceptance test before their unit opens. +- Adding a new command requires one core command file plus type-level marshaling where necessary; it does not require hand-maintained per-transport command lists. +- v0.2.0 release artifacts include Windows x86_64 and ARM64 deliverables, with npm install working on Windows. + +## Scope Boundaries + +- Linux adapter remains Phase 3. +- MCP server mode remains Phase 4. +- Daemon, sessions, audit log, policy engine, OCR fallback, and package-manager distribution remain later production-readiness work. +- UI recording, macro replay, and GUI/TUI surfaces remain non-goals. +- Platform divergence is acceptable when the OS genuinely differs, but the JSON envelope, error semantics, and command names must stay consistent. + +## Key Decisions + +- Treat `docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md` as authoritative where earlier brainstorm or roadmap wording conflicts. +- Ship Phase 2 as one coherent release, not a 2a/2b split. Manage risk with reviewable units, not with hidden deferrals. +- Preserve headless-first as the phase's top architectural constraint. Any approach requiring foreground activation or physical cursor movement for ordinary commands is rejected. +- Use Windows UIA-first behavior and SendInput only as a fallback. UIA patterns are the reliability baseline. +- Use `ControlViewWalker` for Windows tree traversal, fresh cache requests per drill-down, and no cross-apartment UIA element caching. +- Reject `inventory` and `linkme`; use deterministic `build.rs` enumeration of one-command-per-file sources. +- Rename the old `FileDrop` concept to `DeliverFiles` because macOS drag sessions violate headless-first. +- Keep public APIs synchronous for CLI/FFI. Async or streaming behavior belongs to MCP/daemon phases. +- Keep `WindowInfo.pid` as `i32`; narrow Windows PIDs at the adapter boundary with explicit failure if reality ever exceeds the representation. +- Add FFI ABI handshaking before the breaking v0.2.0 shape changes so consumers fail closed. +- Do not refactor existing flat `AccessibilityNode` fields just to satisfy aesthetic field-count pressure; nest only the new selector group. +- Update docs and skills as a release gate, not as cleanup afterthoughts. + +## External Research Notes + +Checked on 2026-04-25: + +- Microsoft UI Automation guidance says desktop-wide UIA clients should make UIA calls from a separate COM MTA thread that owns no windows, add/remove event handlers on that same non-UI thread, and handle COM apartment-affinity invalidation. This supports the Windows event-threading and no-cross-apartment-caching decisions. +- Microsoft `SendInput` documentation says input injection is subject to UIPI and only works into equal-or-lower integrity processes; it also does not reliably identify UIPI as the failure reason. This supports UIA-first behavior and structured fallback errors. +- Microsoft `IGraphicsCaptureItemInterop::CreateForWindow` targets a single `HWND` and requires Windows 10 version 1903 / build 18362 or later. Phase 2 should make that the minimum for modern per-window Windows screenshots, with legacy fallback for unsupported environments. +- Apple documents `SCScreenshotManager.captureImage(contentFilter:configuration:)` for single-frame capture through ScreenCaptureKit. The current `screencapturekit` crate docs list version 1.5.4 and report typical Apple Silicon first-frame latency around 30-100ms at 1080p, so the screenshot unit must benchmark before pinning a colder threshold. +- `uiautomation` 0.24.4 currently depends on `windows` 0.62.2, and the `windows` crate's current docs list 0.62.2. The Phase 2 dependency pins should be rechecked in the unit, but the existing direction is still plausible. +- GitHub's hosted-runner reference now lists `windows-11-arm` for ARM64 Windows in public and private repositories. Phase 2 should try ARM64 CI on that runner; keep build-only fallback only if repository, minutes, or toolchain constraints block reliable CI. + +## Dependencies / Assumptions + +- Windows support targets interactive desktop sessions, not Session 0 or Server Core. +- Modern Windows screenshot support requires Windows Graphics Capture availability, with `CreateForWindow` requiring Windows 10 version 1903 / build 18362 or later. +- Some Chromium/Electron apps still require renderer accessibility to expose useful trees; Phase 2 should detect and guide rather than pretend otherwise. +- ARM64 Windows should use GitHub's `windows-11-arm` runner for at least CI smoke coverage if reliable in this repository; fall back to build-only only if runner/toolchain constraints block it. +- v0.2.0 may break JSON and FFI ABI in documented ways because the project is still pre-1.0. + +## Outstanding Questions + +### Resolve Before Planning + +- None for product direction. The existing Phase 2 plan can proceed through Unit 1 / Unit 2 prep work. + +### Execution-Prep Gates + +- [Affects R3][Technical] Audit existing macOS non-mouse action fallbacks and decide which become headless-only, which move behind explicit mouse/focus commands, and which return structured unsupported/action errors. +- [Affects R6][Technical] Validate macOS `AXObserver` threading and teardown against real apps before opening the final event-backed wait implementation. +- [Affects R7][Technical] Confirm exact Windows UIA text-range behavior in Notepad, Settings, and WebView2 controls, including nil/empty/error cases. +- [Affects R8][Needs research] Verify ScreenCaptureKit and `windows-capture` API pins and benchmark cold/warm screenshot latency before setting final thresholds. +- [Affects R9][Technical] For each new action variant, record the target workflow and native/headless path or structured unsupported result before implementation. +- [Affects R10][Technical] Confirm Windows notification and tray UIA trees across current Windows 11 builds and tie every macOS-only surface backfill to a target workflow before implementation. + +### Deferred to Implementation + +- [Affects R12][Technical] Keep checking generated wrapper/header drift in CI as command registry codegen lands. + +## Next Steps + +Use `docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md` for execution planning, but apply the gates above before opening affected implementation units. If moving directly into work, start with the v0.1.14 FFI prep / Unit 1 sequence before opening Windows adapter implementation. diff --git a/docs/plans/2026-02-19-feat-agent-desktop-phase1-foundation-plan.md b/docs/plans/2026-02-19-feat-agent-desktop-phase1-foundation-plan.md new file mode 100644 index 0000000..f6e3989 --- /dev/null +++ b/docs/plans/2026-02-19-feat-agent-desktop-phase1-foundation-plan.md @@ -0,0 +1,1897 @@ +--- +title: "feat: agent-desktop Phase 1 — Foundation + macOS MVP" +type: feat +date: 2026-02-19 +phase: 1 +duration_weeks: 10 +deepened: 2026-02-19 +--- + +# feat: agent-desktop Phase 1 — Foundation + macOS MVP + +## Enhancement Summary + +**Deepened on:** 2026-02-19 (round 1), 2026-02-19 (round 2 — code quality audit) +**Research agents used:** 15 (Architecture Strategist, Performance Oracle, Security Sentinel, Code Simplicity Reviewer, Agent-Native Reviewer, Pattern Recognition Specialist, Best Practices Researcher, Framework Docs Researcher, Spec Flow Analyzer, Agent-Native Architecture Skill, Dead Code/LOC Auditor, Test-Before-Implement Researcher, Modular Architecture Researcher, Code Quality Reviewer, Architecture Extensibility Reviewer) +**Context7 queries:** clap 4 derive patterns, serde performance patterns +**Web research:** macOS AX FFI best practices, cargo-dist distribution, rmcp latest version, accessibility-sys alternatives, thiserror 2.0 changes + +### Round 2 Structural Findings (Fix During Implementation) + +- **Dead code:** `clipboard.rs` (zero callers), `batch::execute` stub (never called) +- **LOC violation:** `tree.rs` at 403 lines — split into `ax_element.rs` / `ax_attrs.rs` / `ax_tree.rs` +- **Dispatch duplication:** 187/397 lines in `dispatch.rs` are a near-verbatim parallel dispatch table — collapse to single code path via `Commands` enum deserialization +- **Probe binaries:** `axprobe.rs` / `axprobe2.rs` in `src/bin/` compile on every build — move to `examples/` with `required-features = ["dev-tools"]` +- **Bugs:** `wait.rs` silently discards RefMap load errors; `is_check.rs` reads stale state; `press.rs` undocumented null-handle convention + +### Blockers (Fix Before Writing Code) + +1. **Circular dependency in CommandRegistry** — `CommandRegistry::dispatch` references `crate::cli::Commands` from the binary crate, creating a circular dep (core ↔ binary). **Fix:** Drop `Command` trait and `CommandRegistry` entirely; dispatch via `match` in the binary crate. +2. **Missing `resolve_handle` on PlatformAdapter** — `click.rs` calls `adapter.resolve_handle(entry)` but the method does not exist on the 13-method trait. **Fix:** Add `fn resolve_element(&self, entry: &RefEntry) -> Result` as the 14th trait method. +3. **`AppError` type referenced but never defined** — `command.rs` and `click.rs` return `AppError` but only `AdapterError` and `ErrorCode` are defined in `error.rs`. **Fix:** Define `AppError` enum with `#[from]` impls for `AdapterError`, `std::io::Error`, `serde_json::Error`. +4. **`unwrap()` in `refmap_path()`** — Violates the plan's own "zero unwrap in non-test code" invariant. Panics if `HOME` is unset. **Fix:** Return `Result`, propagate error. + +### Critical Improvements + +5. **Use `AXUIElementCopyMultipleAttributeValues`** for batch attribute fetch (3-5x faster tree traversal, likely required to hit 2s Xcode target). +6. **Remove `IsTerminal` stdin detection** — Piped input (`< /dev/null`) falsely triggers MCP mode. Use only `--mcp` flag. +7. **Atomic RefMap writes** — Use temp-file + `rename()` to prevent corruption on concurrent access. +8. **File permissions** — RefMap at `0o600`, directory at `0o700`. Currently world-readable. +9. **`AXElement` inner field** — Change from `pub` to `pub(crate)` to prevent double-free via raw pointer extraction. +10. **NativeHandle soundness** — Add `PhantomData<*const ()>` to opt out of auto-`Send`/`Sync`. + +### Key Simplifications (YAGNI) + +11. **Delete:** `Command` trait, `CommandRegistry`, `command.rs`, `crates/mcp/` stub, `--mcp` flag, token estimation, `manage_window`, `synthesize_input`. +12. **Defer:** `schemars` and `schemas/` to Phase 3 (no consumer in P1). `tokio` to Phase 2/3 (all P1 ops are sync). +13. **Simplify:** `Response` → `Response` with `serde_json::Value`. Entry point from ~40 lines to ~15 lines. Pipeline from 5 stages to 3. + +### Version Corrections + +14. **rmcp is at 0.15.0** (February 2026), NOT 0.8+ as PRD claimed. Now the official Rust SDK at `modelcontextprotocol/rust-sdk`. +15. **core-foundation** may be at 0.10.0 (plan says 0.9). Verify with `cargo search`. +16. **core-foundation-sys** is at 0.8.x (plan says 0.9). Version mismatch between the two crates. +17. Add `panic = "abort"` to release profile (200-500KB smaller binary). + +### Agent-Native Gaps + +18. **Add `batch` command** — Single process invocation for observe-act-verify sequences. Eliminates N startup costs. +19. **Fix snapshot envelope** — `ref_count`/`tree` must be inside `data`, not top-level siblings of `version`/`ok`. +20. **Post-action state in responses** — Every action command should return the element's state after the action (eliminates verify round-trip). +21. **Exit code contract** — 0=success, 1=structured error, 2=argument error. +22. **Default `snapshot` to focused window** of frontmost app when no args given. + +--- + +## Overview + +Build the complete vertical for `agent-desktop`: a Rust CLI + MCP server that enables +AI agents to observe and control desktop applications via native OS accessibility trees. +Phase 1 targets macOS exclusively, delivers 30 production-ready commands, establishes +every shared abstraction, and ships a binary distribution via cargo-dist. + +Phases 2–4 are strictly additive: new platform adapters, new transport, new +production-readiness work — nothing in core is rebuilt. + +**Source:** PRD v2.0 (February 2026) + Architecture Brainstorm 2026-02-19. + +--- + +## Architecture Decisions + +From the brainstorm session (`docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md`): + +| Decision | Choice | Rationale | +|---|---|---| +| `PlatformAdapter` trait | Unified (13 methods) | Command extensibility never touches the trait | +| NativeHandle persistence | Optimistic + `STALE_REF` | Store `(pid, role, name, bounds_hash)`; return `STALE_REF` on mismatch | +| Test strategy | MockAdapter + golden fixtures | Offline unit tests + output contract regression | +| Workspace bootstrap | All platform crates from day one | Enforces adapter boundary at compile time | + +### Research Insights — Architecture + +**From Architecture Strategist:** +- The core-to-platform isolation is the strongest architectural property. Enforce with CI: `cargo tree -p agent-desktop-core` must contain no platform crate names. +- Binary crate's `Cargo.toml` must use **target-gated dependencies** for platform crates (not unconditional deps with `#[cfg]` in source). +- Consider extracting `RefMapStore` trait to decouple persistence from core (enables in-memory store for MCP/daemon in Phase 3/4). +- `src/` as a workspace member for the binary crate is non-standard. Consider workspace root or `crates/cli/`. + +**From Code Simplicity Reviewer:** +- Drop `Command` trait + `CommandRegistry` — use plain functions + `match`. The trait's associated types prevent `dyn Command`, making the registry useless. Each command is already a standalone `execute()` function in practice. +- Delete `crates/mcp/` stub — unlike platform stubs which enforce adapter boundary, the MCP crate enforces nothing. Create in Phase 3. +- Remove `manage_window` and `synthesize_input` from trait — these serve Phase 2 commands only. Trait can grow additively. +- Trait drops from 13 to 11 methods after removing these + adding `resolve_element`. + +**From Pattern Recognition Specialist:** +- Extract `resolve_ref()` and `wrap_response()` helpers to eliminate 100-150 lines of boilerplate across 10+ ref-based command files. +- Provide default impls on `PlatformAdapter` that return `not_supported()` — eliminates 65-line stub implementations per platform. +- `Response::ok(command, data)` and `Response::err(command, error)` constructors prevent envelope construction duplication. + +--- + +## Dependency Versions + +All versions confirmed against latest stable (February 2026). Use `workspace = true` +inheritance throughout. + +### `Cargo.toml` (workspace root) + +```toml +[workspace] +members = [ + "crates/core", + "crates/macos", + "crates/windows", + "crates/linux", + "crates/mcp", + "src", +] +resolver = "2" + +[workspace.package] +edition = "2021" +rust-version = "1.78" +license = "Apache-2.0" + +[workspace.dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +tokio = { version = "1", features = ["rt", "io-std", "macros", "sync", "time"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +schemars = "0.8" +base64 = "0.22" +anyhow = "1" + +# Phase 3 only — verify actual published version before adding: +# rmcp = { version = "0.8", features = ["server", "transport-io"] } + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +strip = true +``` + +### `rust-toolchain.toml` (workspace root) + +```toml +[toolchain] +channel = "stable" +profile = "minimal" +targets = ["aarch64-apple-darwin", "x86_64-apple-darwin"] +``` + +### Platform crate dependencies + +**`crates/macos/Cargo.toml`** +```toml +[dependencies] +agent-desktop-core = { path = "../core" } +thiserror.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +accessibility-sys = "0.1" +core-foundation = "0.9" +core-foundation-sys = "0.9" +``` + +**`crates/windows/Cargo.toml`** (Phase 2 stub) +```toml +[dependencies] +agent-desktop-core = { path = "../core" } +thiserror.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +uiautomation = "0.24" +``` + +**`crates/linux/Cargo.toml`** (Phase 2 stub) +```toml +[dependencies] +agent-desktop-core = { path = "../core" } +thiserror.workspace = true + +[target.'cfg(target_os = "linux")'.dependencies] +atspi = "0.28" +zbus = "5" +``` + +**`crates/mcp/Cargo.toml`** (Phase 3 stub) +```toml +[dependencies] +agent-desktop-core = { path = "../core" } +serde.workspace = true +serde_json.workspace = true +schemars.workspace = true +# rmcp added in Phase 3 after version verification +``` + +### Version selection rationale + +| Crate | Version | Why | +|---|---|---| +| `clap` | `4` | 4.5.x stable; derive API is mature; 40+ subcommands fit cleanly | +| `thiserror` | `2` | 2.0 released 2024; breaking changes are minor; `#[error(transparent)]` cleaner | +| `base64` | `0.22` | Engine API is required; free-function API removed in 0.22 | +| `schemars` | `0.8` | 1.0-alpha is unstable; 0.8 is production-stable; required by rmcp | +| `tokio` | minimal features | Do NOT use `full`; adds TLS/net/process not needed until Phase 3 | +| `rmcp` | verify before Phase 3 | PRD says 0.8+; actual crates.io version may differ; pin exact minor | +| `accessibility-sys` | `0.1` | Thin stable FFI to Apple AXUIElement.h; safer than higher-level `accessibility` crate | +| `core-foundation` | `0.9` | Required companion for CFTypeRef, CFString, CFArray management | + +### Research Insights — Dependencies + +**Version corrections (February 2026):** + +| Crate | Plan Says | Actual/Recommended | Action | +|---|---|---|---| +| `rmcp` | `0.8+` | **0.15.0** (official SDK at `modelcontextprotocol/rust-sdk`) | Update Phase 3 comment. Pin exact: `"=0.15.0"` | +| `core-foundation` | `0.9` | Possibly `0.10.0` | Run `cargo search core-foundation` to confirm | +| `core-foundation-sys` | `0.9` | `0.8.x` (NOT 0.9.x!) | `core-foundation` 0.10.x depends on `core-foundation-sys` 0.8.x | +| `thiserror` | `2` | Correct (`2.0.x`) | Requires Rust 1.78+ (plan already specifies) | +| `schemars` | `0.8` | Correct (1.0 is alpha) | Defer to Phase 3 — no P1 consumer | +| `tokio` | `1` (5 features) | Correct but **not needed in P1** | Remove from P1; all ops are sync | + +**Recommended release profile addition:** +```toml +[profile.release] +panic = "abort" # 200-500KB smaller binary, no unwind tables +``` + +**Pre-implementation verification checklist:** +```bash +cargo search accessibility-sys +cargo search core-foundation +cargo search core-foundation-sys +cargo search rmcp +# Then create throwaway project and run: +cargo tree -d # check for duplicate/conflicting versions +``` + +The `core-foundation` / `core-foundation-sys` / `accessibility-sys` version triangle is the highest-risk compatibility issue. Verify with `cargo tree` before committing versions. + +**From Best Practices Researcher — version-pinned dependency table:** + +| Crate | Pinned Version | Rationale | +|---|---|---| +| `clap` | `"4.5"` | 4.5.x stable; derive API mature | +| `serde` | `"1.0"` | Stable forever | +| `serde_json` | `"1.0"` | Stable forever | +| `thiserror` | `"2.0"` | 2.0 cleaner `#[error(transparent)]`; requires Rust 1.78+ | +| `tracing` | `"0.1"` | Stable | +| `tracing-subscriber` | `"0.3"` + `env-filter` | Stable | +| `base64` | `"0.22"` | Engine API required | +| `anyhow` | `"1.0"` | Test helpers only | +| `accessibility-sys` | `"0.1"` | Thin stable FFI | +| `core-foundation` | `"0.10"` or `"0.9"` | Verify compat | +| `core-graphics` | `"0.24"` | For CGEvent, CGWindowList | +| `rustc-hash` | `"2.0"` | FxHashSet for cycle detection (zero-dep, 2KB) | + +--- + +## Workspace Layout + +Matches PRD §4.1 exactly. + +``` +agent-desktop/ +├── Cargo.toml # workspace root, shared deps +├── Cargo.lock +├── rust-toolchain.toml +├── clippy.toml # project-wide lint config +├── schemas/ +│ ├── snapshot_response.json # generated, checked in +│ ├── action_response.json +│ └── error_response.json +├── docs/ +│ ├── brainstorms/ +│ └── plans/ +├── tests/ +│ ├── fixtures/ # golden JSON snapshots for regression tests +│ │ ├── finder_documents.json +│ │ ├── textedit_untitled.json +│ │ └── system_settings.json +│ └── integration/ +│ ├── macos_snapshot.rs +│ ├── macos_actions.rs +│ └── cross_platform.rs # stub, enabled in Phase 2 +├── crates/ +│ ├── core/src/ +│ │ ├── lib.rs # pub re-exports only +│ │ ├── node.rs # AccessibilityNode, Rect, WindowInfo +│ │ ├── adapter.rs # PlatformAdapter trait +│ │ ├── action.rs # Action enum, ActionResult, InputEvent, WindowOp +│ │ ├── refs.rs # RefAllocator, RefMap, RefEntry +│ │ ├── snapshot.rs # SnapshotEngine (filter, allocate, serialize) +│ │ ├── error.rs # ErrorCode enum, AdapterError, AppError +│ │ ├── output.rs # Response envelope, JSON formatting +│ │ ├── command.rs # Command trait + CommandRegistry +│ │ └── commands/ +│ │ ├── mod.rs # register_all() +│ │ ├── snapshot.rs +│ │ ├── click.rs +│ │ ├── type_text.rs +│ │ ├── set_value.rs +│ │ ├── press.rs +│ │ ├── find.rs +│ │ ├── get.rs +│ │ ├── is_check.rs +│ │ ├── screenshot.rs +│ │ ├── scroll.rs +│ │ ├── select.rs +│ │ ├── toggle.rs +│ │ ├── expand.rs +│ │ ├── collapse.rs +│ │ ├── focus.rs +│ │ ├── launch.rs +│ │ ├── close_app.rs +│ │ ├── list_windows.rs +│ │ ├── list_apps.rs +│ │ ├── focus_window.rs +│ │ ├── clipboard.rs +│ │ ├── wait.rs +│ │ ├── status.rs +│ │ ├── permissions.rs +│ │ └── version.rs +│ ├── macos/src/ +│ │ ├── lib.rs +│ │ ├── adapter.rs # MacOSAdapter: PlatformAdapter impl +│ │ ├── tree.rs # AXUIElement traversal, AXElement newtype +│ │ ├── actions.rs # AXPress, SetValue, SetFocus, Expand, Select +│ │ ├── roles.rs # AXRole string → unified role mapping +│ │ ├── input.rs # CGEvent keyboard/mouse synthesis +│ │ ├── screenshot.rs # ScreenshotBackend legacy capture +│ │ └── permissions.rs # AXIsProcessTrusted, TCC guidance +│ ├── windows/src/ +│ │ ├── lib.rs +│ │ └── adapter.rs # WindowsAdapter stub → PLATFORM_UNSUPPORTED +│ ├── linux/src/ +│ │ ├── lib.rs +│ │ └── adapter.rs # LinuxAdapter stub → PLATFORM_UNSUPPORTED +│ └── mcp/src/ +│ ├── lib.rs +│ └── server.rs # MCP stub → Phase 3 +└── src/ + ├── main.rs # entry point: mode detection, dispatch + └── cli.rs # clap derive structs for all 30 commands +``` + +--- + +## Core Data Models + +### `crates/core/src/node.rs` + +```rust +use serde::{Deserialize, Serialize}; +use schemars::JsonSchema; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct AccessibilityNode { + #[serde(skip_serializing_if = "Option::is_none")] + pub ref_id: Option, // @e1, @e2 — only on interactive roles + + pub role: String, // normalized: button, textfield, checkbox, etc. + + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub states: Vec, // focused, selected, expanded, checked, etc. + + #[serde(skip_serializing_if = "Option::is_none")] + pub bounds: Option, // only when --include-bounds + + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub children: Vec, +} + +### Research Insights — AccessibilityNode + +**From Performance Oracle — consider Role enum + States bitfield:** +Roles come from a fixed set (~30 values). Using `role: String` allocates heap memory per node. A `Role` enum eliminates one allocation per node (2000 saved on Xcode-scale trees): +```rust +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Role { + Button, TextField, Checkbox, Link, MenuItem, Tab, Slider, + ComboBox, TreeItem, Cell, Window, Group, Toolbar, StaticText, + Image, Table, ScrollArea, + #[serde(other)] + Unknown, +} +``` +Similarly, `states: Vec` can become a bitfield (2 bytes vs 24+ bytes + heap): +```rust +bitflags::bitflags! { + pub struct States: u16 { + const FOCUSED = 0b0001; const SELECTED = 0b0010; + const EXPANDED = 0b0100; const CHECKED = 0b1000; + const DISABLED = 0b0001_0000; + } +} +``` +**Estimated savings:** 40-60% fewer heap allocations during tree construction. + +**Trade-off:** Custom serde impls needed to maintain JSON array-of-strings format for `states`. The `#[serde(other)]` on `Role` handles unknown AX roles gracefully. Consider whether this optimization is worth the implementation cost for Phase 1 — `String` types work fine and are simpler. Apply this optimization if benchmarks show allocation pressure. + +**From Pattern Recognition Specialist — remove `JsonSchema` derive:** +`schemars` is deferred to Phase 3. Remove `#[derive(JsonSchema)]` from all structs in Phase 1. + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Rect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WindowInfo { + pub id: String, // w-4521 format + pub title: String, + pub app: String, + pub pid: i32, + pub bounds: Option, + pub is_focused: bool, +} +``` + +### `crates/core/src/refs.rs` + +```rust +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Stable metadata for re-identifying an AXUIElement across invocations. +/// Stored in ~/.agent-desktop/last_refmap.json for CLI mode. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RefEntry { + pub pid: i32, + pub role: String, + pub name: Option, + pub bounds_hash: Option, // FxHash of bounds for disambiguation + pub available_actions: Vec, // ["Click", "SetValue"] etc. + // Internal: NativeHandle is NOT serialized — resolved at action time +} + +pub struct RefMap { + inner: HashMap, // "@e1" → RefEntry + counter: u32, +} + +impl RefMap { + pub fn new() -> Self { + Self { inner: HashMap::new(), counter: 0 } + } + + pub fn allocate(&mut self, entry: RefEntry) -> String { + self.counter += 1; + let ref_id = format!("@e{}", self.counter); + self.inner.insert(ref_id.clone(), entry); + ref_id + } + + pub fn get(&self, ref_id: &str) -> Option<&RefEntry> { + self.inner.get(ref_id) + } + + pub fn save(&self) -> Result<(), AppError> { + let path = refmap_path()?; + let dir = path.parent().ok_or(AppError::Internal("invalid refmap path".into()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().recursive(true).mode(0o700).create(dir)?; + } + #[cfg(not(unix))] + std::fs::create_dir_all(dir)?; + + let json = serde_json::to_string(&self)?; // compact, not pretty + let tmp = path.with_extension("tmp"); + + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let mut file = std::fs::OpenOptions::new() + .write(true).create(true).truncate(true).mode(0o600).open(&tmp)?; + std::io::Write::write_all(&mut file, json.as_bytes())?; + } + #[cfg(not(unix))] + std::fs::write(&tmp, &json)?; + + std::fs::rename(&tmp, &path)?; // atomic replace + Ok(()) + } + + pub fn load() -> Result { + let path = refmap_path()?; + let json = std::fs::read_to_string(path)?; + let inner: HashMap = serde_json::from_str(&json)?; + let counter = inner.keys() + .filter_map(|k| k.strip_prefix("@e")) + .filter_map(|n| n.parse::().ok()) + .max() + .unwrap_or(0); + Ok(Self { inner, counter }) + } +} + +fn refmap_path() -> Result { + let home = dirs::home_dir() + .ok_or(AppError::Internal("HOME directory not found".into()))?; + Ok(home.join(".agent-desktop").join("last_refmap.json")) +} + +### Research Insights — RefMap + +**Changes from original:** +- **Atomic writes** — temp file + `rename()` prevents corruption on concurrent access or kill. +- **File permissions** — `0o600` for RefMap, `0o700` for directory (was world-readable). +- **Compact JSON** — `to_string` instead of `to_string_pretty` (30-40% smaller, faster to parse). +- **Fixed `unwrap()`** — `refmap_path()` returns `Result` instead of panicking. +- **Fixed counter** — Derived from highest ref number in keys, not `inner.len()` (prevents collisions on sparse maps). + +**From Architecture Strategist — RefMap counter serialization:** +Serialize the counter alongside the map to make the format self-consistent: +```rust +#[derive(Serialize, Deserialize)] +pub struct RefMap { + inner: HashMap, + counter: u32, +} +``` + +**From Security Sentinel — RefMap integrity:** +- Add file size limit on load (reject >1MB to prevent memory exhaustion). +- In Phase 3/4, consider HMAC for file integrity checking. +- Include `source_app` and `source_window_id` in `RefEntry` to prevent the silent wrong-target bug when two agents clobber each other's RefMap. + +**From Spec Flow Analyzer — concurrent access race (Race 3):** +Agent A snapshots Finder, Agent B snapshots TextEdit (overwrites RefMap), Agent A clicks @e5 — silently hits TextEdit element. **Mitigation:** Add `source_app` field to RefEntry and validate before action execution. +``` + +### `crates/core/src/adapter.rs` + +The single most important abstraction. Never import platform crates from core. + +```rust +use crate::{ + action::{Action, ActionResult, InputEvent, WindowOp}, + node::AccessibilityNode, + node::WindowInfo, + error::AdapterError, +}; + +pub struct WindowFilter { + pub focused_only: bool, + pub app: Option, +} + +pub struct TreeOptions { + pub max_depth: u8, + pub include_bounds: bool, + pub interactive_only: bool, + pub compact: bool, +} + +pub enum ScreenshotTarget { + Screen(usize), + Window(String), // window ID + Element(NativeHandle), + FullScreen, +} + +pub enum PermissionReport { + Granted, + Denied { suggestion: String }, +} + +/// Opaque handle to a native UI element within a snapshot context. +/// Not Send/Sync on macOS (AXUIElement). Methods that receive a NativeHandle +/// must be called on the thread that created it. +pub struct NativeHandle(pub(crate) *const std::ffi::c_void); + +pub struct ImageBuffer { + pub data: Vec, + pub format: ImageFormat, + pub width: u32, + pub height: u32, +} + +pub enum ImageFormat { Png, Jpg } + +pub trait PlatformAdapter: Send + Sync { + fn list_windows(&self, filter: &WindowFilter) -> Result, AdapterError>; + fn list_apps(&self) -> Result, AdapterError>; + fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions) + -> Result; + fn execute_action(&self, handle: &NativeHandle, request: ActionRequest) + -> Result; + fn resolve_element(&self, entry: &RefEntry) -> Result; + fn permission_report(&self) -> PermissionReport; + fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError>; + fn launch_app(&self, id: &str, wait: bool) -> Result; + fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError>; + fn screenshot(&self, target: ScreenshotTarget) -> Result; + fn get_clipboard(&self) -> Result; + fn set_clipboard(&self, text: &str) -> Result<(), AdapterError>; +} +``` + +### Research Insights — PlatformAdapter Trait + +**Changes from original (13 methods → 12 methods):** +- **Added** `resolve_element` — every action command needs ref-to-native-handle resolution. This is platform-specific (macOS walks AX tree, Windows queries UIA). +- **Removed** `synthesize_input` — Phase 2 concern. P1 keyboard needs route through `execute_action` with `Action::PressKey(KeyCombo)`. +- **Removed** `manage_window` — all window geometry commands (`resize`, `move`, `minimize`) are Phase 2. Only P1 window op is `focus_window` which has its own method. + +**From Performance Oracle — `resolve_element` must use early-termination search:** +```rust +fn resolve_element(&self, entry: &RefEntry) -> Result { + let root = element_for_pid(entry.pid); + resolve_recursive(&root, entry, 0, 20) + .ok_or_else(|| AdapterError::stale_ref()) +} +``` +Search stops the moment the matching element is found (O(k) average vs O(n) full traversal). + +**From Security Sentinel — NativeHandle soundness fix:** +```rust +use std::marker::PhantomData; + +pub struct NativeHandle { + pub(crate) ptr: *const std::ffi::c_void, + _not_send_sync: PhantomData<*const ()>, +} +``` +Raw pointers are `!Send + !Sync`. For Phase 1 (single-threaded CLI), add `unsafe impl Send/Sync` with safety documentation. Revisit for Phase 3 async runtime. + +**From Architecture Strategist — provide default implementations:** +```rust +pub trait PlatformAdapter: Send + Sync { + fn list_windows(&self, _: &WindowFilter) -> Result, AdapterError> { + Err(AdapterError::not_supported("list_windows")) + } + // ... defaults for all methods +} +``` +This eliminates 60+ lines of boilerplate per stub adapter. + +### `crates/core/src/error.rs` + +```rust +use serde::Serialize; +use thiserror::Error; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ErrorCode { + PermissionDenied, + ElementNotFound, + ApplicationNotFound, + ActionFailed, + ActionNotSupported, + StaleRef, + WindowNotFound, + PlatformNotSupported, + Timeout, + InvalidArgs, + Internal, +} + +#[derive(Debug, Error)] +#[error("{message}")] +pub struct AdapterError { + pub code: ErrorCode, + pub message: String, + pub suggestion: Option, + pub platform_detail: Option, +} + +impl AdapterError { + pub fn new(code: ErrorCode, message: impl Into) -> Self { + Self { code, message: message.into(), suggestion: None, platform_detail: None } + } + + pub fn with_suggestion(mut self, s: impl Into) -> Self { + self.suggestion = Some(s.into()); + self + } + + pub fn stale_ref(ref_id: &str) -> Self { + Self::new(ErrorCode::StaleRef, format!("{ref_id} not found in current RefMap")) + .with_suggestion("Run 'snapshot' to refresh, then retry with updated ref") + } + + pub fn not_supported(msg: &str) -> Self { + Self::new(ErrorCode::PlatformNotSupported, msg) + .with_suggestion("This platform adapter ships in Phase 2") + } +} + +#[derive(Debug, Error)] +pub enum AppError { + #[error(transparent)] + Adapter(#[from] AdapterError), + + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("{0}")] + Internal(String), +} +``` + +### Research Insights — Error Design + +**Changes from original (12 variants → 10):** +- **Merged** `TreeTimeout` + `Timeout` → single `Timeout` (the `message` field provides context). +- **Removed** `ClipboardEmpty` — empty clipboard is a valid state, not an error. Return `{ "ok": true, "data": { "text": null } }`. +- **Added** `InvalidArgs` — for clap parse failures wrapped in JSON envelope. +- **Renamed** for consistency: `PermDenied` → `PermissionDenied`, `AppNotFound` → `ApplicationNotFound`, `PlatformUnsupported` → `PlatformNotSupported`. + +**From Agent-Native Reviewer — exit code contract:** +- Exit 0 = `ok: true`, valid JSON on stdout +- Exit 1 = `ok: false`, valid JSON on stdout (structured error) +- Exit 2 = binary-level failure (no JSON guarantee — clap errors, panics) + +**From Agent-Native Reviewer — add `retry_command` to ErrorPayload:** +```rust +pub struct ErrorPayload { + pub code: String, + pub message: String, + pub suggestion: Option, + pub retry_command: Option, // e.g., "snapshot --app Finder" + pub platform_detail: Option, +} +``` +Gives agents a machine-executable recovery path instead of parsing natural language from `suggestion`. + +**From Security Sentinel — strip `platform_detail` in MCP mode:** +Only include raw AXError codes / HRESULT values when `--verbose` is set or in CLI mode. MCP responses should omit platform internals. + +### `crates/core/src/output.rs` + +```rust +use serde::Serialize; +use schemars::JsonSchema; + +/// The versioned response envelope. All commands produce this structure. +#[derive(Debug, Serialize, JsonSchema)] +pub struct Response { + pub version: &'static str, // "1.0" + pub ok: bool, + pub command: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub app: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct AppContext { + pub name: String, + pub window: Option, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct WindowContext { + pub id: String, + pub title: String, +} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ErrorPayload { + pub code: String, + pub message: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub suggestion: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub platform_detail: Option, +} +``` + +--- + +## Command Extensibility Pattern + +Every command is one file under `crates/core/src/commands/`. No existing files change +when a command is added — only `mod.rs` and `src/cli.rs` receive new lines. + +### Dispatch via `match` (replaces `Command` trait + `CommandRegistry`) + +```rust +pub fn dispatch( + cmd: Commands, + adapter: &dyn PlatformAdapter, +) -> Result { + match cmd { + Commands::Snapshot(args) => commands::snapshot::execute(args, adapter), + Commands::Click(args) => commands::click::execute(args.into(), adapter), + Commands::Find(args) => commands::find::execute(args, adapter), + // ... one arm per command + } +} +``` + +Adding a command = add one file + add one `Commands` variant + add one match arm. Same cost as the registry approach but without runtime indirection or trait object gymnastics. + +### Shared helpers: `crates/core/src/commands/helpers.rs` + +```rust +pub fn resolve_ref( + ref_id: &str, + adapter: &dyn PlatformAdapter, +) -> Result<(RefEntry, NativeHandle), AppError> { + validate_ref_id(ref_id)?; + let refmap = RefMap::load()?; + let entry = refmap.get(ref_id).ok_or(AppError::stale_ref(ref_id))?.clone(); + let handle = adapter.resolve_element(&entry)?; + Ok((entry, handle)) +} + +fn validate_ref_id(ref_id: &str) -> Result<(), AppError> { + let valid = ref_id.starts_with("@e") + && ref_id.len() <= 10 + && ref_id[2..].chars().all(|c| c.is_ascii_digit()); + if !valid { + return Err(AppError::invalid_input("ref_id must match @e{N}")); + } + Ok(()) +} +``` + +### Example: `crates/core/src/commands/click.rs` + +```rust +use crate::{adapter::PlatformAdapter, error::AppError, commands::helpers::resolve_ref}; + +pub struct ClickArgs { + pub ref_id: String, +} + +pub fn execute(args: ClickArgs, adapter: &dyn PlatformAdapter) -> Result { + let (_entry, handle) = resolve_ref(&args.ref_id, adapter)?; + let result = adapter.execute_action(&handle, crate::action::Action::Click)?; + Ok(serde_json::to_value(result)?) +} +``` + +### Research Insights — Command Pattern + +**Why `Command` trait + `CommandRegistry` were removed:** +- The trait's associated types (`type Args`, `type Output`) prevent `dyn Command` — the registry cannot store heterogeneous commands without type erasure. +- The `CommandRegistry::dispatch` referenced `crate::cli::Commands` from the binary crate, creating a circular dependency. +- The actual `click.rs` code was already a plain function, not a trait impl. The plan contradicted itself. +- `Response` envelope wrapping now happens once in the dispatch layer, not duplicated in every command file. + +**From Agent-Native Reviewer — add `batch` command:** +``` +agent-desktop batch '[ + {"command": "snapshot", "args": {"app": "Finder"}}, + {"command": "click", "args": {"ref": "@e3"}}, + {"command": "get", "args": {"ref": "@e3", "property": "value"}} +]' +``` +Returns array of responses. Eliminates N process spawns for observe-act-verify sequences. Add `--stop-on-error` flag. + +**From Agent-Native Reviewer — post-action state in responses:** +Every action command should return the element's state after the action: +```json +{ + "data": { + "action": "click", + "ref": "@e5", + "post_state": { "role": "checkbox", "states": ["checked", "focused"] } + } +} +``` +Eliminates the verify round-trip (2 tool calls → 1 per action). + +--- + +## SnapshotEngine Processing Pipeline + +Lives entirely in `crates/core/src/snapshot.rs`. No platform code. + +**3 stages** (simplified from original 5): +1. **Raw tree** — `adapter.get_tree(window, opts)` returns full `AccessibilityNode` +2. **Filter + allocate refs** — single depth-first pass: prune invisible/offscreen nodes, enforce `max_depth`, assign `@e1, @e2, …` to interactive roles, build `RefMap` +3. **Serialize** — `serde_json::to_writer(BufWriter::new(stdout.lock()), &response)` — stream directly to stdout, no intermediate String allocation + +**Removed stages:** +- Stage 4 (serialize) was a one-liner calling `serde_json::to_value()` — not a pipeline stage. +- Stage 5 (token estimate) was YAGNI — agents see the output directly and know their own token budget. Validate P1-O4 (<500 tokens for Finder) in integration tests using actual tiktoken, not a runtime heuristic. + +**Interactive roles** (only these receive refs): +`button, textfield, checkbox, link, menuitem, tab, slider, combobox, treeitem, cell` + +**RefMap write behavior:** Each snapshot REPLACES the full file. Action commands that +follow a snapshot always operate on fresh refs. Multi-window workflows require +separate snapshots (each replaces the previous RefMap). + +### Research Insights — Snapshot Pipeline + +**From Performance Oracle — budget-aware tree construction:** +Integrate token budget checking into the filter stage. If estimated chars exceed `max_tokens * 4`, stop adding children to deeper subtrees. This prevents serializing trees that will exceed the budget. + +**From Agent-Native Reviewer — default behavior:** +- `snapshot` with no arguments should default to the focused window of the frontmost application (the 80% case). +- Add `--subtree ` flag to snapshot only the subtree rooted at a specific element. +- Add `--filter-role ` to only include specific roles. + +**From Spec Flow Analyzer — `find` semantics:** +`find` should perform a fresh snapshot internally (using the last-used window or requiring `--app`), update the RefMap, and return matching elements with their refs. Include an ancestry `path` field for disambiguation: +```json +{ "ref": "@e7", "role": "button", "name": "Save", "path": ["window:Documents", "toolbar", "button:Save"] } +``` + +**From Performance Oracle — serialization performance:** +Use `serde_json::to_writer(BufWriter::new(stdout.lock()), &data)` instead of `to_string()` + `println!`. Eliminates intermediate String allocation (saves 2-50KB depending on tree size). For 1000-node trees: `to_writer` ~1.8ms vs `to_string` ~2.5ms. + +--- + +## macOS Adapter Implementation + +### `crates/macos/src/` — Tree module split + +`tree.rs` exceeds 400 LOC and is split into three files (see "Code Quality and Structural Refactors" section). The patterns below apply to the split files: `AXElement` and `element_for_pid` go in `ax_element.rs`, attribute helpers in `ax_attrs.rs`, traversal logic in `ax_tree.rs`. + +### `crates/macos/src/ax_element.rs` — Key patterns + +```rust +use accessibility_sys::{ + AXUIElementRef, AXUIElementCreateApplication, + AXUIElementCopyAttributeValue, + kAXChildrenAttribute, kAXRoleAttribute, + kAXTitleAttribute, kAXDescriptionAttribute, + kAXValueAttribute, kAXEnabledAttribute, + kAXErrorSuccess, kAXErrorNoValue, kAXErrorAttributeUnsupported, +}; +use core_foundation::{base::{CFTypeRef, CFRelease, TCFType}, string::CFString}; +use std::collections::HashSet; + +/// Owns an AXUIElementRef. Releases on drop. +pub struct AXElement(pub AXUIElementRef); + +impl Drop for AXElement { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { CFRelease(self.0 as CFTypeRef) } + } + } +} + +// SAFETY: AXUIElement is NOT thread-safe. The MacOSAdapter is designed so +// all AX calls happen on the calling thread within a single method invocation. +// No AXElement is stored in MacOSAdapter fields. + +pub fn element_for_pid(pid: i32) -> AXElement { + AXElement(unsafe { AXUIElementCreateApplication(pid) }) +} + +/// Traverse with depth limit and cycle detection. +/// visited: raw pointer addresses (usize) prevent infinite loops on malformed trees. +pub fn build_subtree( + el: &AXElement, + depth: usize, + max_depth: usize, + visited: &mut HashSet, +) -> Option { + if depth > max_depth { return None; } + if !visited.insert(el.0 as usize) { return None; } // cycle + + let role = copy_string_attr(el, unsafe { kAXRoleAttribute })?; + // ... build AccessibilityNode, recurse into children +} +``` + +### `crates/macos/src/permissions.rs` + +```rust +use accessibility_sys::{AXIsProcessTrusted, AXIsProcessTrustedWithOptions}; + +pub fn is_trusted() -> bool { + unsafe { AXIsProcessTrusted() } +} + +/// Prompt system dialog. Only call via `permissions --request`. +pub fn request_trust() -> bool { + // Build CFDictionary with kAXTrustedCheckOptionPrompt = kCFBooleanTrue + // ... omitted for brevity + unsafe { AXIsProcessTrustedWithOptions(options) } +} +``` + +### AX attribute error handling + +```rust +match err { + kAXErrorSuccess => { /* use value */ } + kAXErrorNoValue => return None, // attribute has no value + kAXErrorAttributeUnsupported => return None, // element lacks this attribute + kAXErrorInvalidUIElement => return Err(AdapterError::stale()), // element gone + kAXErrorCannotComplete => return Err(AdapterError::timeout()), // app busy/hung + _ => return Err(AdapterError::ax(err)), +} +``` + +### Research Insights — macOS Adapter + +**From Performance Oracle (CRITICAL) — batch attribute fetch:** +`AXUIElementCopyMultipleAttributeValues` fetches all requested attributes in a single IPC round-trip. Reduces 7 calls per node to 1. For 2000 nodes (Xcode), drops from 14,000 IPC calls to 2,000. **3-5x faster traversal.** + +This API is NOT in `accessibility-sys` 0.1. Declare manually: +```rust +extern "C" { + fn AXUIElementCopyMultipleAttributeValues( + element: AXUIElementRef, + attributes: CFArrayRef, + options: u32, + values: *mut CFArrayRef, + ) -> AXError; +} +``` + +**Traversal performance projections:** + +| Scenario | Nodes | FFI Calls (current) | FFI Calls (batch) | Time (current) | Time (batch) | +|---|---|---|---|---|---| +| Finder Documents | ~100 | ~700 | ~100 | 35-70ms | 10-20ms | +| TextEdit simple | ~50 | ~350 | ~50 | 18-35ms | 5-10ms | +| System Settings | ~500 | ~3,500 | ~500 | 175-350ms | 50-100ms | +| Xcode (depth=8) | ~1,500 | ~10,500 | ~1,500 | 525-1,050ms | 150-300ms | + +Without batch fetch, the 2-second Xcode target is achievable only at depth ≤8. With it, depth 12-15 becomes feasible. + +**From Security Sentinel — `AXElement` safety fixes:** +- Make inner field private: `pub struct AXElement(AXUIElementRef)` (not `pub AXUIElementRef`) +- Implement `Clone` with `CFRetain`: +```rust +impl Clone for AXElement { + fn clone(&self) -> Self { + if !self.0.is_null() { + unsafe { CFRetain(self.0 as CFTypeRef); } + } + AXElement(self.0) + } +} +``` + +**From Best Practices Researcher — CFTypeRef memory management:** +- `Create` / `Copy` functions → `wrap_under_create_rule` (you own it, Drop releases) +- `Get` functions → `wrap_under_get_rule` (borrowed, do NOT release) +- `AXUIElementCopyAttributeValue` returns OWNED `CFTypeRef` → use create rule + +**From Framework Docs Researcher — missing FFI declarations needed:** +- `AXUIElementCopyMultipleAttributeValues` (batch fetch) +- `CGWindowListCopyWindowInfo` (from CoreGraphics, not Accessibility) +- `CGEventCreateKeyboardEvent` / `CGEventCreateMouseEvent` (from CoreGraphics) +- NSWorkspace bindings for `launch_app` / `list_apps` (use `objc` crate or shell out to `open -a`) + +**From Performance Oracle — cycle detection optimization:** +Replace `HashSet` with `FxHashSet` from `rustc-hash` (2x faster hash ops, zero downside since keys are machine pointers, not user input). + +**From Security Sentinel — `kAXErrorCannotComplete` (-25204):** +Most common error in practice. Target app is busy or hung. Retry with backoff or return `TIMEOUT`. + +--- + +## Entry Point — `src/main.rs` + +```rust +mod cli; + +use clap::Parser; +use cli::{Cli, Commands}; + +fn main() { + let cli = match Cli::try_parse() { + Ok(cli) => cli, + Err(e) => { + if e.kind() == clap::error::ErrorKind::DisplayHelp + || e.kind() == clap::error::ErrorKind::DisplayVersion + { + e.exit(); + } + let json = serde_json::json!({ + "version": "1.0", + "ok": false, + "command": "unknown", + "error": { "code": "INVALID_ARGS", "message": e.to_string().lines().next().unwrap_or("Unknown error") } + }); + println!("{json}"); + std::process::exit(2); + } + }; + + init_tracing(cli.verbose); + + match cli.command { + Some(Commands::Version(args)) => handle_version(args), + Some(Commands::Status) => handle_status(), + Some(cmd) => { + let adapter = build_adapter(); + let result = dispatch(cmd, &adapter); + match result { + Ok(data) => { + let response = serde_json::json!({ + "version": "1.0", "ok": true, + "command": cmd_name, "data": data + }); + println!("{response}"); + std::process::exit(0); + } + Err(e) => { + println!("{}", e.to_json()); + std::process::exit(1); + } + } + } + None => { Cli::command().print_help().unwrap(); } + } +} + +fn build_adapter() -> impl agent_desktop_core::adapter::PlatformAdapter { + #[cfg(target_os = "macos")] + { agent_desktop_macos::MacOSAdapter::new() } + + #[cfg(not(target_os = "macos"))] + compile_error!("Unsupported platform") +} + +fn init_tracing(verbose: bool) { + use tracing_subscriber::{fmt, EnvFilter}; + fmt() + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new(if verbose { "debug" } else { "warn" })) + ) + .with_writer(std::io::stderr) + .with_ansi(false) + .init(); +} +``` + +### Research Insights — Entry Point + +**Changes from original:** +- **Removed** `--mcp` flag + `IsTerminal` stdin detection — MCP is Phase 3. Piped input (`< /dev/null`) falsely triggered MCP mode. Add `--mcp` flag in Phase 3 as a one-line addition. +- **Removed** `Arc` — Phase 1 is single-threaded. Return `impl PlatformAdapter` instead. +- **Added** `Cli::try_parse()` with JSON error wrapping — agents parsing stdout for JSON will get unparseable plain text from clap's default error handler. Now they get structured `INVALID_ARGS` errors. +- **Lazy adapter construction** — `Version` and `Status` commands don't need the adapter. Only construct it when needed. +- **Added** `RUST_LOG` environment variable override — `try_from_default_env().unwrap_or(filter)` lets users override via `RUST_LOG=agent_desktop_macos=trace`. +- **Added** `with_ansi(false)` — no color codes in stderr (may be redirected). + +**From Best Practices Researcher — startup performance budget:** + +| Phase | Typical Cost | Notes | +|-------|-------------|-------| +| Dynamic linker | 1-2ms | Static linking helps | +| `main()` entry | <0.1ms | | +| clap parsing | 0.5-2ms | 30 subcommands | +| tracing init | 0.2-0.5ms | env-filter parsing | +| Adapter construction | <0.1ms | Just struct init | +| **Total cold start** | **2-5ms** | Well under 10ms target | + +Removing tokio from Phase 1 saves 1-3ms startup and 200-400KB binary size. + +--- + +## CLI Structure — `src/cli.rs` + +Key patterns from research: + +- Use `global = true` on `--verbose` and `--mcp` so they work after subcommands +- `kebab-case` flags are automatic from `snake_case` field names in clap 4 +- `std::io::IsTerminal` from stable std — no `atty` dependency +- `subcommand_required = false` + explicit `None` handling (not `arg_required_else_help`) + +```rust +#[derive(Parser, Debug)] +#[command(name = "agent-desktop", version, about = "Desktop automation for AI agents")] +pub struct Cli { + #[arg(long, global = true, hide = true)] + pub mcp: bool, + + #[arg(long, short = 'v', global = true)] + pub verbose: bool, + + #[command(subcommand)] + pub command: Option, +} + +#[derive(Subcommand, Debug)] +pub enum Commands { + Snapshot(SnapshotArgs), + Find(FindArgs), + Screenshot(ScreenshotArgs), + Get(GetArgs), + Is(IsArgs), + Click(RefArgs), + DoubleClick(RefArgs), + RightClick(RefArgs), + Type(TypeArgs), + SetValue(SetValueArgs), + Focus(RefArgs), + Select(SelectArgs), + Toggle(RefArgs), + Expand(RefArgs), + Collapse(RefArgs), + Scroll(ScrollArgs), + Launch(LaunchArgs), + CloseApp(CloseAppArgs), + ListWindows(ListWindowsArgs), + ListApps(ListAppsArgs), + FocusWindow(FocusWindowArgs), + Press(PressArgs), + ClipboardGet, + ClipboardSet(ClipboardSetArgs), + Wait(WaitArgs), + Status, + Permissions(PermissionsArgs), + Version(VersionArgs), + Batch(BatchArgs), +} +``` + +### Research Insights — CLI + +**From Best Practices Researcher:** +- Use `Cli::try_parse()` instead of `Cli::parse()` to catch clap errors and wrap them in JSON envelope with `INVALID_ARGS` error code. +- `help` and `version` display remains plain text (not JSON). All other errors are JSON. +- For 30 subcommands, parsing is ~1ms. Do NOT use `arg_required_else_help = true` (prevents `--mcp` from working). +- `kebab-case` is automatic from `snake_case` field names in clap 4. + +**From Agent-Native Reviewer — `batch` command added:** +```rust +Batch(BatchArgs), // JSON array of commands, returns array of responses +``` + +**From Spec Flow Analyzer — missing global flags to consider:** + +| Flag | Purpose | +|---|---| +| `--timeout ` (global) | Default timeout for any blocking command | +| `--session ` | Scope RefMap to a session (prevents multi-agent corruption) | +| `--dry-run` | Show what would happen without executing (agent debugging) | + +**From Best Practices Researcher — subcommand grouping for `--help`:** +```rust +#[command(after_help = "\ +CATEGORIES: + Observation: snapshot, find, screenshot, get, is + Interaction: click, double-click, right-click, type, set-value, focus, select, toggle, expand, collapse, scroll, press + App/Window: launch, close-app, list-windows, list-apps, focus-window + Clipboard: clipboard-get, clipboard-set + Wait: wait + System: status, permissions, version + Batch: batch")] +``` + +--- + +## JSON Output Contract + +All commands produce this envelope. Schema files live in `schemas/` and are +generated from the Rust structs via `schemars`. Schema version is tracked via +the `version` field. + +```json +{ + "version": "1.0", + "ok": true, + "command": "snapshot", + "app": { "name": "Finder", "window": { "id": "w-4521", "title": "Documents" } }, + "ref_count": 14, + "tree": { + "role": "window", + "name": "Documents", + "children": [ + { "role": "toolbar", "children": [ + { "ref": "@e1", "role": "button", "name": "Back" }, + { "ref": "@e2", "role": "button", "name": "Forward" } + ]}, + { "ref": "@e3", "role": "textfield", "name": "Search", "value": "" } + ] + } +} +``` + +Error envelope: +```json +{ + "version": "1.0", + "ok": false, + "command": "click", + "error": { + "code": "STALE_REF", + "message": "@e3 not found in current RefMap", + "suggestion": "Run 'snapshot' to refresh, then retry with updated ref", + "retry_command": "snapshot --app Finder" + } +} +``` + +### Research Insights — JSON Contract + +**From Agent-Native Reviewer — envelope consistency fix:** +The snapshot example had `ref_count` and `tree` as **top-level** fields, but `Response` wraps data in `data`. Every command without exception must nest its output inside `data`. Use `Response` with `serde_json::Value`: + +```rust +pub struct Response { + pub version: &'static str, + pub ok: bool, + pub command: String, + pub app: Option, + pub data: Option, + pub error: Option, +} +``` + +**Corrected snapshot example:** +```json +{ + "version": "1.0", + "ok": true, + "command": "snapshot", + "app": { "name": "Finder", "window": { "id": "w-4521", "title": "Documents" } }, + "data": { + "ref_count": 14, + "tree": { "role": "window", "children": [...] } + } +} +``` + +**From Spec Flow Analyzer — `screenshot` output format:** +If `[path]` argument is provided, write to file and return `{ "path": "/tmp/screenshot.png" }` in data. If not provided, return base64-encoded data. The `base64` crate is already in dependencies for this purpose. + +**From Spec Flow Analyzer — `is` command output format:** +```json +{ "data": { "property": "visible", "result": true } } +``` + +--- + +## Testing Plan + +### Unit tests — `crates/core/src/` (no macOS required, runs on any CI) + +``` +snapshot_engine_filter_test - invisible nodes removed, depth pruning works +ref_allocator_ordering_test - depth-first order, only interactive roles get refs +ref_allocator_interactive_test - button/textfield get refs; group/statictext do not +snapshot_serialize_compact_test - null fields omitted, empty arrays omitted +snapshot_serialize_roundtrip_test - AccessibilityNode ser/de roundtrip +refmap_save_load_test - RefMap REPLACE semantics; stale detection +error_serialize_test - every ErrorCode serializes to SCREAMING_SNAKE_CASE +json_schema_validate_test - generated schema matches output +``` + +**MockAdapter** for unit tests: + +```rust +// crates/core/src/commands/tests/mock_adapter.rs +pub struct MockAdapter { + pub tree: AccessibilityNode, // hardcoded tree returned by get_tree() + pub clipboard: String, + pub windows: Vec, +} +impl PlatformAdapter for MockAdapter { ... } +``` + +### Golden fixture regression — `tests/fixtures/` + +Checked-in JSON files captured from real macOS apps. Tests load them and verify: +- Ref count matches expected +- Interactive elements have refs; static text does not +- JSON schema validates against `schemas/snapshot_response.json` + +Fixtures captured once during development and committed. Re-capture with a +`--update-fixtures` flag when intentional format changes occur. + +### Research Insights — Testing + +**From Best Practices Researcher — CI configuration for macOS tests:** +```yaml +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo test --lib --all + + integration-tests: + runs-on: macos-14 # Apple Silicon runner (has display session) + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Grant accessibility (TCC) + run: | + sudo tccutil reset Accessibility + - run: cargo test --test integration +``` + +**From Performance Oracle — add benchmarks during Week 3-4:** +```rust +// benches/traversal.rs (using criterion) +fn bench_finder_traversal(c: &mut Criterion) { + c.bench_function("finder_full_tree", |b| { + b.iter(|| adapter.get_tree(&window, &opts)) + }); +} +``` + +Add `criterion` as dev-dependency. Benchmark traversal, serialization, and refmap I/O independently. This validates whether batch fetch (CRITICAL-3) is needed on target hardware. + +**From Architecture Strategist — CI isolation enforcement:** +```yaml +- name: Verify core isolation + run: | + cargo tree -p agent-desktop-core | grep -E "accessibility-sys|core-foundation|uiautomation|atspi|zbus" && exit 1 || true +``` + +**From Spec Flow Analyzer — missing integration test:** +Add a test for the canonical agent workflow: `snapshot → find → click → snapshot → verify`. This is the most important flow to regression-test. + +### Integration tests — `tests/integration/` (macOS CI only) + +Real AX adapter tests on GitHub Actions macOS runner: + +``` +macos_snapshot_finder_test - snapshot Finder Documents window → non-empty tree +macos_snapshot_textedit_test - snapshot TextEdit → textfield refs present +macos_click_test - launch test app, click button, verify state changed +macos_type_test - type text into TextEdit, verify content +macos_clipboard_test - clipboard get/set roundtrip +macos_launch_close_test - launch Calculator, verify window appears, close +macos_perm_denied_test - simulate no TCC permission → PERM_DENIED error code +macos_large_tree_test - snapshot Xcode → completes under 2 seconds +``` + +--- + +## Phase 1 — 10-Week Implementation Timeline + +### Weeks 1–2: Scaffold + Core Types + +**Deliverables:** +- `Cargo.toml` workspace with all 6 crates + resolver 2 +- `rust-toolchain.toml` pinned to stable 1.78+ +- `crates/core/src/`: `node.rs`, `adapter.rs`, `action.rs`, `error.rs`, `output.rs` +- `crates/core/src/refs.rs`: `RefMap`, `RefEntry`, `RefAllocator` structs +- `crates/core/src/command.rs`: `Command` trait + `CommandRegistry` skeleton +- `crates/core/src/commands/mod.rs`: empty `register_all()` +- Stub adapters for windows, linux (all methods → `AdapterError::not_supported`) +- Stub MCP crate +- `src/cli.rs`: all 30 subcommand structs (no-op implementations) +- `src/main.rs`: mode detection, `build_adapter()`, `init_tracing()` +- `clippy.toml` with `unwrap_in_result = "deny"`, `panic = "deny"` for non-test +- `schemas/` directory with placeholder JSON Schema files +- CI: `cargo check --all-targets` passes on macOS + +**Key invariants to enforce from day one:** +- `crates/core` has zero imports from any platform crate (verify with `cargo tree` in CI) +- Zero `unwrap()` calls in non-test code +- All error types implement the structured pattern from `error.rs` +- Binary crate uses target-gated deps: `[target.'cfg(target_os = "macos")'.dependencies]` +- `clippy.toml` with `forbid-unwrap-in-result = true` + +### Research Insights — Weeks 1–2 + +**Simplified workspace from agent research:** +- Delete: `crates/mcp/` stub (create in Phase 3), `command.rs` (use direct dispatch), `schemas/` dir (defer to Phase 3) +- Stub crates: single `lib.rs` per stub (not separate `adapter.rs`), no platform deps in stub `Cargo.toml` +- Omit from workspace deps: `tokio`, `schemars` (not needed until Phase 2/3) +- Add to workspace deps: `rustc-hash = "2.0"` (FxHashSet for cycle detection) + +--- + +### Weeks 3–4: macOS Tree Traversal + +**Deliverables:** +- `crates/macos/src/ax_element.rs`: `AXElement` newtype with `Drop`, `Clone` (CFRetain), `element_for_pid()`, `window_element_for()` +- `crates/macos/src/ax_attrs.rs`: `copy_string_attr`, `copy_ax_array`, `fetch_node_attrs`, `read_bounds` +- `crates/macos/src/ax_tree.rs`: `build_subtree()` with cycle detection via `FxHashSet`, depth limiting, `resolve_element_name` +- `crates/macos/src/roles.rs`: AXRole string → unified role enum mapping (AXButton → "button", AXTextField → "textfield", etc.) +- `crates/macos/src/permissions.rs`: `is_trusted()`, `request_trust()` +- `crates/macos/src/adapter.rs`: `MacOSAdapter::get_tree()` and `MacOSAdapter::permission_report()` implemented; all other methods stub +- `crates/macos/examples/axprobe.rs`: probe binary moved from `src/bin/`, gated on `dev-tools` feature +- Unit tests: role mapping coverage, cycle detection (synthetic circular tree), permission mock +- Stage 1–3 probe validation completed for each AX attribute used before wiring into adapter +- Manual validation: `cargo run -- snapshot --app Finder` produces valid JSON + +**Gotchas from research:** +- `kAXErrorNoValue` is non-fatal — return `None`, not `Err` +- `kAXErrorCannotComplete` (-25204) is the most common error — app is busy/hung. Retry or return `TIMEOUT` +- `kAXVisibleChildrenAttribute` fallback for scroll views +- Never call AX APIs on tokio thread pool — all AX stays on calling thread +- `build.rs` in `crates/macos/` to link `ApplicationServices` and `CoreFoundation` +- `AXElement.0` must be `pub(crate)` not `pub` to prevent double-free +- Use `wrap_under_create_rule` for Copy/Create function results; `wrap_under_get_rule` for Get results +- Implement `AXUIElementCopyMultipleAttributeValues` as manual `extern "C"` (not in accessibility-sys 0.1) +- Add `criterion` benchmarks during this phase to validate performance targets + +--- + +### Weeks 5–6: Snapshot Engine + Ref System + +**Deliverables:** +- `crates/core/src/snapshot.rs`: full `SnapshotEngine` with 5-stage pipeline +- `crates/core/src/refs.rs`: `RefAllocator` depth-first ordering, interactive-role + filtering, `RefMap::save()` / `RefMap::load()` with REPLACE semantics +- RefMap written to `~/.agent-desktop/last_refmap.json` after each snapshot +- Token estimation (char ÷ 4 heuristic, warn at 500) +- `crates/core/src/commands/snapshot.rs`: full implementation +- Unit tests: all 8 snapshot + ref unit tests passing +- Golden fixtures captured: Finder, TextEdit, System Settings +- JSON Schema files generated and committed to `schemas/` +- Integration test: `macos_snapshot_finder_test` passes + +--- + +### Weeks 6–7: Core Interaction Commands + +**Deliverables:** +- `click.rs`, `double_click.rs`, `right_click.rs` +- `type_text.rs` (CGEventCreateKeyboardEvent synthesis) +- `set_value.rs` (AXUIElementSetAttributeValue) +- `focus.rs` (AXFocusedAttribute = true) +- `press.rs` (CGEvent key combos: cmd+c, ctrl+shift+s, etc.) +- `find.rs` (tree search by name, role, or value substring) +- `get.rs` (text, value, title, bounds, role, states) +- `is_check.rs` (visible, enabled, checked, focused, expanded) +- `crates/macos/src/input.rs`: key combo parsing and CGEvent synthesis +- Integration tests: `macos_click_test`, `macos_type_test` passing + +--- + +### Weeks 7–8: App/Window + Remaining Interaction Commands + +**Deliverables:** +- `launch.rs` (NSWorkspace / `open -a`, --wait via window polling) +- `close_app.rs` (graceful quit + SIGKILL --force) +- `list_windows.rs` (CGWindowListCopyWindowInfo filtered to visible) +- `list_apps.rs` (running processes via NSWorkspace) +- `focus_window.rs` (AXRaise + NSActivateApp) +- `screenshot.rs` (ScreenshotBackend legacy capture for window; full screen) +- `select.rs` (kAXSelectedAttribute on child elements) +- `toggle.rs` (detect role + appropriate action: AXPress for checkbox) +- `expand.rs` / `collapse.rs` (kAXExpandedAttribute toggle) +- `scroll.rs` (kAXScrollByPageAttribute or CGEventScrollWheelCreate) +- Integration tests: `macos_launch_close_test`, `macos_snapshot_xcode_test` (large tree) + +--- + +### Weeks 8–9: Clipboard, Wait, System Commands + +**Deliverables:** +- `clipboard.rs` get/set (NSPasteboard via Cocoa FFI) +- `wait.rs`: + - `wait ` — `std::thread::sleep` + - `wait --element ` — poll RefMap every 100ms up to timeout + - `wait --window ` — poll window list for title match +- `status.rs` (platform, permissions, daemon PID placeholder) +- `permissions.rs` (check + optional `--request` prompt) +- `version.rs` (from `CARGO_PKG_VERSION`, optional --json) +- Integration tests: `macos_clipboard_test`, `macos_perm_denied_test` + +--- + +### Weeks 9–10: Testing, CI, Binary Distribution, Polish + +**Deliverables:** +- All 8 unit test suites passing (`cargo test --lib`) +- All integration tests passing on GitHub Actions macOS runner +- JSON Schema validation added to CI: `cargo test --test schema_validation` +- GitHub Actions workflow (`.github/workflows/ci.yml`): + - `cargo fmt --check` + - `cargo clippy --deny warnings` + - `cargo test --lib` (any runner) + - `cargo test --test integration` (macOS runner) + - `cargo build --release` builds under 15MB +- `cargo-dist init` with `[workspace.metadata.dist]` config +- Release workflow (`.github/workflows/release.yml`) via cargo-dist: + - Produces `aarch64-apple-darwin` + `x86_64-apple-darwin` tarballs + - SHA256 checksums attached to GitHub Release +- Binary size verified: `cargo build --release && ls -lh target/release/agent-desktop` +- CLI smoke test: `./target/release/agent-desktop snapshot --app Finder | jq .ok` +- README with installation, usage, and macOS TCC setup instructions +- P1-O1 through P1-O9 success criteria all verified (PRD §6.1) + +--- + +## Phase 1 Acceptance Criteria + +From PRD §6.1: + +| ID | Criterion | Verification | +|---|---|---| +| P1-O1 | Working macOS snapshot CLI | `snapshot --app Finder` returns valid JSON with refs | +| P1-O2 | Platform adapter trait | Trait compiles with MockAdapter; MacOSAdapter satisfies all methods | +| P1-O3 | Ref-based interaction | `click @e3` invokes AXPress on resolved element | +| P1-O4 | Context efficiency | Finder Documents snapshot < 500 tokens | +| P1-O5 | Typed JSON contract | Output validates against JSON Schema; schema is versioned | +| P1-O6 | Permission detection | Missing TCC permission prints macOS setup instructions | +| P1-O7 | Command extensibility | Adding a new command = 1 new file + 2 registration lines | +| P1-O8 | 30 working commands | All P1-scoped commands from §5 pass integration tests | +| P1-O9 | CI pipeline | GitHub Actions macOS runner executes full test suite on every PR | + +--- + +## Phase 2–4 Reference + +| Phase | Duration | Key Work | +|---|---|---| +| P2: Cross-Platform | 10 weeks | Windows adapter (uiautomation 0.24), Linux adapter (atspi 0.28 + zbus 5), cross-platform CI | +| P3: MCP Server | 6 weeks | rmcp integration (verify version), stdio transport, Claude Desktop validation | +| P4: Production Readiness | 8 weeks | Persistent daemon, session isolation, brew/winget/snap packages, performance benchmarks | + +**Phase 3 note:** rmcp is confirmed at **0.15.0** (February 2026) — the official Rust +MCP SDK at `modelcontextprotocol/rust-sdk`. Pin to exact: `"=0.15.0"`. The PRD's +`0.8+` version was incorrect. Verify features: `server`, `transport-io`. +Before Phase 3, implement auth/authz policy system (Security finding F03). + +--- + +## Security Readiness (Phase 1) + +From the Security Sentinel audit (17 findings, 5 CRITICAL/HIGH for Phase 1): + +### Immediate (Week 1 — before first code commit) + +| # | Finding | Fix | +|---|---|---| +| F01 | `NativeHandle` is `Send/Sync` unsound | Add `PhantomData<*const ()>` field | +| F02 | `AXElement` double-free risk | Make inner field `pub(crate)`, implement `Clone` with `CFRetain` | +| F04 | RefMap world-readable permissions | `0o600` file, `0o700` directory | +| F05 | `home_dir().unwrap()` panic | Return `Result`, propagate error | + +### During Implementation (Weeks 2–10) + +| # | Finding | Fix | +|---|---|---| +| F06 | No input validation on `ref_id` | `validate_ref_id()` at entry of every ref-based command | +| F07 | `launch_app` accepts arbitrary paths | Validate app identifiers (bundle ID or name only), reject raw paths. Use `NSWorkspace` APIs, not `open -a` | +| F08 | `press` accepts dangerous key combos | Blocklist: `cmd+q`, `cmd+shift+q`, `cmd+opt+esc`, `ctrl+cmd+q`, `cmd+shift+delete` | +| F09 | `type_text` enables command injection | Text length limits (10,000 chars max). Log invocations with target app context | +| F12 | `close_app --force` lacks safeguards | Protected process list: `loginwindow`, `WindowServer`, `Dock`, `launchd`. PID ownership check | +| F14 | Cycle detection uses pointer address | Add hard `ABSOLUTE_MAX_DEPTH = 50` cap | +| F17 | No audit logging | Implement `~/.agent-desktop/audit.log` with `0o600` permissions | + +### Pre-Phase 3 (Before MCP Ships) + +| # | Finding | Fix | +|---|---|---| +| F03 | MCP server has no auth/authz | Policy file (`~/.agent-desktop/policy.toml`) with command allowlist/denylist | +| F16 | MCP has no message size limits | 1MB max per message, request timeout | +| F10 | Screenshot data exposure | Separate TCC permission check (`kTCCServiceScreenCapture`). Add `--no-capture` flag | +| F11 | Clipboard credential exposure | Detect `org.nspasteboard.ConcealedType` (password manager entries), refuse to read | + +--- + +## Risk Mitigations + +| Risk | Mitigation | +|---|---| +| macOS TCC blocks CI integration tests | Document setup; use GitHub Actions `macos-14` runner (ARM, has display session); `tccutil reset Accessibility` + sqlite3 TCC.db grant | +| AXUIElement cycle causes stack overflow | `FxHashSet<usize>` cycle detection; max_depth hard stop at 20; `ABSOLUTE_MAX_DEPTH = 50` cap | +| Large AX trees (Xcode) exceed 5s timeout | Default max_depth = 8; focused-window-only; `AXUIElementCopyMultipleAttributeValues` batch fetch (3-5x faster); integration test asserts < 2s | +| rmcp version mismatch | **Confirmed: rmcp 0.15.0** (Feb 2026). Pin exact: `"=0.15.0"`. Fall back to hand-rolled JSON-RPC if API changes | +| Binary > 15MB | `lto = true`, `strip = true`, `panic = "abort"` in release profile; feature-gate `schemars`; monitor with CI size check. Expected 3-6MB | +| core-foundation version mismatch | Run `cargo tree -d` to check for duplicates. `core-foundation` 0.10.x depends on `core-foundation-sys` 0.8.x (NOT matching versions) | +| Concurrent agent RefMap corruption | Atomic writes (temp + rename). Document as known limitation. Add `source_app` to RefEntry for validation | +| Focus theft during `type` command | Document that `type` is best-effort, `set-value` is atomic. Consider verifying focus before each keystroke batch | + +--- + +## Agent-Native Checklist + +From the Agent-Native Reviewer (score: 26/30, 4 structural gaps): + +### Must-Fix (before Phase 1 ships) + +- [x] **Batch command** — single process invocation for multi-step workflows +- [x] **Envelope consistency** — `ref_count`/`tree` inside `data`, not top-level +- [x] **Post-action state** — every action returns element state after the action +- [x] **Exit code contract** — 0=ok, 1=structured error, 2=argument error +- [x] **Default snapshot** — no args = focused window of frontmost app + +### Should-Fix (Phase 1) + +- [ ] **`wait --match`** — semantic criteria (`--role button --name "Save"`) instead of stale ref +- [ ] **`retry_command`** in ErrorPayload — machine-executable recovery +- [ ] **`find` ancestry path** — `["window:Documents", "toolbar", "button:Save"]` +- [ ] **`available_actions`** in tree output — opt-in with `--include-actions` +- [ ] **MCP naming convention** — `desktop_{command}` documented now, implemented in Phase 3 + +### Consider (late Phase 1 or Phase 2) + +- [ ] Namespaced refs by window (`@w4521.e1`) for multi-window workflows +- [ ] `resolve <ref>` probe command (check validity without replacing RefMap) +- [ ] `diff` command (change detection between snapshots) +- [ ] `status --app <name>` responsiveness check +- [ ] `--max-tokens` flag on snapshot + +--- + +## Code Quality and Structural Refactors + +These findings are not blockers for Phase 1 feature work, but must be resolved before the Phase 1 milestone is closed. Each has a clear fix and zero risk of behaviour change. + +### Dead Code to Remove + +**`crates/core/src/commands/clipboard.rs`** — 13 lines, exports `execute_get()` and `execute_set()` with zero callers anywhere in the workspace. The clipboard commands are implemented in the correct split files `clipboard_get.rs` and `clipboard_set.rs` (one command per file), which `dispatch.rs` calls directly. `clipboard.rs` is a leftover from before the split and should be deleted along with its `pub mod clipboard;` line in `mod.rs`. + +**`batch::execute` stub** — `crates/core/src/commands/batch.rs` exports an `execute()` function that returns `Ok(json!({"note": "..."}))`. It is never called; `dispatch.rs` handles batch routing inline. Delete the `execute` function body. Keep `BatchArgs`, `BatchCommand`, and `parse_commands` — they are live. + +### LOC Violations + +**`crates/macos/src/tree.rs` is at 403 lines** — exceeds the 400-line hard limit. Split into three files by single responsibility: + +| New file | Contents | Estimated LOC | +|---|---|---| +| `ax_element.rs` | `AXElement` newtype, `Drop`, `Clone` (with `CFRetain`), `element_for_pid`, `window_element_for` | ~55 | +| `ax_attrs.rs` | `copy_string_attr`, `copy_value_typed`, `copy_bool_attr`, `copy_ax_array`, `read_bounds`, `fetch_node_attrs` | ~200 | +| `ax_tree.rs` | `build_subtree`, `resolve_element_name`, `label_from_children`, `copy_children` | ~115 | + +**`crates/macos/src/lib.rs`** — update module declarations: +```rust +pub mod ax_element; +pub mod ax_attrs; +pub mod ax_tree; +// Remove: pub mod tree; +``` + +Internal cross-file imports use `super::` paths: +```rust +// ax_tree.rs +use super::ax_element::AXElement; +use super::ax_attrs::{copy_string_attr, fetch_node_attrs}; +``` + +**`src/dispatch.rs` is at 397 lines** — 3 lines under the limit but contains 187 lines of duplication (see below). Collapsing the duplication brings it to ~210 lines. + +### Dispatch Duplication + +`dispatch.rs` has two parallel dispatch tables: +- Lines 16–169: `dispatch()` — type-safe `Commands` enum, 29 match arms +- Lines 171–358: `dispatch_batch_command()` — string-keyed version, same 29 commands parsed differently + +**Fix:** Collapse `dispatch_batch_command()` by deserializing batch JSON into the `Commands` enum. Because `Commands` derives `Deserialize`, `serde_json::from_value(json!({"snapshot": {"app": "Finder"}}))` already works: + +```rust +pub fn dispatch_batch_command( + name: &str, + args: serde_json::Value, + adapter: &dyn PlatformAdapter, +) -> Result<serde_json::Value, AppError> { + let cmd: Commands = serde_json::from_value(serde_json::json!({ name: args })) + .map_err(|e| AppError::invalid_args(e.to_string()))?; + dispatch(cmd, adapter) +} +``` + +This eliminates ~187 lines of duplication. `batch.rs`'s `parse_commands` routes `{"command": "snapshot", "args": {...}}` entries to `dispatch_batch_command("snapshot", args, adapter)` — one unified code path. + +### Probe Binary Reorganization + +`src/bin/axprobe.rs` and `src/bin/axprobe2.rs` are development probe binaries compiled on every `cargo build`. They should live in `examples/` and only compile when explicitly requested: + +``` +crates/macos/ +└── examples/ + ├── axprobe.rs # moved from src/bin/axprobe.rs + ├── axprobe2.rs # moved from src/bin/axprobe2.rs + └── probe_utils.rs # extract shared helpers (find_pid, string readers, array readers) +``` + +In `crates/macos/Cargo.toml`: +```toml +[[example]] +name = "axprobe" +path = "examples/axprobe.rs" +required-features = ["dev-tools"] + +[features] +dev-tools = [] +``` + +Run with: `cargo run -p agent-desktop-macos --example axprobe --features dev-tools` + +### Bugs to Fix Before Phase 1 Ships + +| File | Bug | Fix | +|---|---|---| +| `wait.rs` | `if let Ok(refmap) = RefMap::load()` silently discards load errors on every polling iteration | Propagate with `?` or emit `tracing::warn!` | +| `is_check.rs` | States read from stale RefMap entry; `_handle` is resolved (liveness check) but then discarded | Add `///` doc-comment: "Returns state from last snapshot. Run `snapshot` first for live state." | +| `press.rs` | `parts.last().unwrap()` — safe but proof depends on preceding `is_empty` guard being visible | Change to `parts.last().ok_or_else(|| AppError::invalid_args("empty key combo"))` | +| `press.rs` | `NativeHandle::null()` passed for `PressKey` action — convention undocumented | Add `// SAFETY: PressKey synthesises a global CGEvent; no element handle is required.` | + +--- + +## Test-Before-Implement Development Workflow + +Every new AX capability must be confirmed at the OS level before being wired into the adapter. This 4-stage process eliminates "wrote 100 lines then discovered the attribute doesn't exist" failures. Document this in `CLAUDE.md` under `## Development Workflow`. + +### Stage 1 — Accessibility Inspector + +Open `/Applications/Xcode.app/Contents/Applications/Accessibility Inspector.app`. Point at the target application. Confirm the AX attribute exists and has a non-null value. Note the exact attribute name string (e.g. `AXMenus`, `AXFocusedWindow`). + +### Stage 2 — JXA Probe (30-second sanity check) + +```bash +osascript -l JavaScript -e ' + var sys = Application("System Events"); + var proc = sys.processes.whose({ name: "Finder" })[0]; + // inspect proc.windows(), proc.menuBars() etc. + JSON.stringify(proc.windows[0].attributes.whose({ name: "AXRole" })[0].value()) +' +``` + +If this returns the expected value, proceed to Stage 3. If it errors or returns null, the attribute is not accessible via standard AX for this app/macOS version — stop and reassess. + +### Stage 3 — Rust Probe in examples/ + +Add a probe function to `examples/axprobe.rs` that calls the candidate AX API directly and prints the result: + +```rust +// examples/axprobe.rs +fn probe_ax_menus(pid: i32) { + let app = ax_element::element_for_pid(pid); + let menus = ax_attrs::copy_ax_array(&app, "AXMenus"); + println!("AXMenus count: {:?}", menus.map(|v| v.len())); +} +``` + +Run: `cargo run -p agent-desktop-macos --example axprobe --features dev-tools -- --pid $(pgrep Finder)` + +Only proceed to Stage 4 when Stage 3 prints the expected value. + +### Stage 4 — Implement as `pub(crate)` + Unit Test + +1. Add the function to the appropriate `ax_*.rs` file with `pub(crate)` visibility +2. Write a unit test (with MockAdapter or synthetic tree if real AX not available) +3. Wire into `adapter.rs` / `adapter.get_tree()` + +### Why the Stages Matter + +- AX attribute availability varies by app (Electron apps expose nothing; native apps vary) +- Some attributes exist but always return `kAXErrorNoValue` for certain element subtypes +- Stage 2 costs 30 seconds; Stage 3 costs 5 minutes; Stage 4 costs hours — fail early +- The probe files double as executable documentation of how each AX API was validated + +--- + +## References + +- PRD v2.0: `agent_desktop_prd_v2.pdf` +- Brainstorm: `docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md` +- accessibility-sys: https://crates.io/crates/accessibility-sys +- core-foundation: https://crates.io/crates/core-foundation +- rmcp (MCP Rust SDK, v0.15.0): https://github.com/modelcontextprotocol/rust-sdk +- clap 4 derive docs: https://docs.rs/clap/latest/clap/_derive/index.html +- Apple AXUIElement reference: https://developer.apple.com/documentation/applicationservices +- Apple AXUIElementCopyMultipleAttributeValues: ApplicationServices/HIServices +- cargo-dist: https://github.com/axodotdev/cargo-dist +- Rust IsTerminal (stable 1.70+): https://doc.rust-lang.org/std/io/trait.IsTerminal.html +- rustc-hash (FxHashSet): https://crates.io/crates/rustc-hash +- bitflags: https://crates.io/crates/bitflags +- criterion (benchmarks): https://crates.io/crates/criterion diff --git a/docs/plans/2026-02-19-feat-context-menu-popup-ax-handling-plan.md b/docs/plans/2026-02-19-feat-context-menu-popup-ax-handling-plan.md new file mode 100644 index 0000000..6a0e964 --- /dev/null +++ b/docs/plans/2026-02-19-feat-context-menu-popup-ax-handling-plan.md @@ -0,0 +1,260 @@ +--- +title: "feat: Context Menu, Popup, and Overlay Handling via Accessibility APIs" +type: feat +date: 2026-02-19 +--- + +# feat: Context Menu, Popup, and Overlay Handling via Accessibility APIs + +## Overview + +Context menus, sheets, dialogs, and popovers are transient surfaces that appear and disappear outside the normal window hierarchy. The current `snapshot` command misses them entirely because `list_windows` filters `kCGWindowLayer != 0` and `window_element_for` only queries `kAXWindowsAttribute`. This plan introduces targeted surface commands that read exactly the live transient surface — no full-tree traversal required. + +## Problem Statement + +1. **Context menus are invisible to snapshot.** Right-clicking in any app opens a menu at CG layer 8. `list_windows` rejects all `layer != 0` entries. `AXWindowsAttribute` does not include menus. An agent cannot read what is in the menu. + +2. **Sheets and dialogs are not distinguished.** Save dialogs, alert panels, and popovers appear as children of windows but are not surfaced as distinct snapshot targets. An agent cannot tell if a modal is blocking interaction. + +3. **No way to wait for a menu or popup.** `wait --element` polls the ref tree. There is no primitive to block until a context menu opens or a sheet appears. + +4. **Comment noise.** Inline comments throughout the macOS crate explain implementation details that should be self-evident from names. + +## Key Research Findings + +### How macOS exposes transient surfaces via AX + +| Surface | AX API | Notes | +|---------|--------|-------| +| Context menu (open) | `AXUIElementCopyAttributeValue(app, "AXMenus", …)` | Returns array of open `AXMenu` elements. Empty when closed. O(1) — no tree scan. | +| Context menu (notification) | `kAXMenuOpenedNotification` on the app element | Delivers the menu AXElement directly. | +| Focused window's sheet/dialog | `kAXFocusedWindowAttribute` on app element | Returns whatever window (or sheet) currently has focus. | +| Sheet attached to a window | Child with `AXSubrole == "AXSheet"` | Always a child of the parent window. | +| Alert/dialog | `kAXSubrole == "AXDialog"` or `kAXModalAttribute == true` | System alerts use subrole; app dialogs may use modal attribute. | +| Popover | `kAXSubrole == "AXPopover"` | Floats above the window hierarchy. | + +### CG window layers (for reference, not for menu detection) + +| Layer | Surface | +|-------|---------| +| 0 | Normal windows | +| 3 | Panels / popovers | +| 8 | Context menus | +| 20 | Alerts | + +Using CG layers to find menus requires cross-referencing PID and is slower than reading `"AXMenus"` directly. + +### `AXUIElementCreateSystemWide()` + +Returns the system-wide accessibility element. `kAXFocusedApplicationAttribute` on it yields the currently focused app element. Useful when the caller has not already identified the target PID. + +## Proposed Solution + +### Phase 1 — `snapshot --surface` flag (new surface targeting) + +Extend `SnapshotArgs` with `--surface <surface>` (default: `window`). + +| `--surface` value | What is captured | Implementation | +|-------------------|-----------------|----------------| +| `window` | Current focused window (existing behavior) | unchanged | +| `focused` | Whatever `kAXFocusedWindowAttribute` returns (window, sheet, or dialog) | single AX attribute read, then build_subtree on result | +| `menu` | Open context menu on the target app | read `"AXMenus"` attribute on app element; error if no open menu | +| `sheet` | Sheet attached to focused window | walk `kAXChildrenAttribute` of focused window, find first `AXSubrole == "AXSheet"` | +| `popover` | Floating popover | walk children for `AXSubrole == "AXPopover"` | +| `alert` | Modal alert/dialog | walk children for `kAXModalAttribute == true` or `AXSubrole == "AXDialog"` | + +All surfaces build their subtree with the existing `build_subtree` function. No new traversal logic needed. + +### Phase 2 — `wait --menu` and `wait --popup` + +Extend the `wait` command with surface-aware variants that use `AXObserver` notifications rather than polling: + +| Flag | Notification | Timeout behavior | +|------|-------------|-----------------| +| `wait --menu` | `kAXMenuOpenedNotification` on app element | error `TIMEOUT` if menu does not appear within `--timeout` ms | +| `wait --menu-closed` | `kAXMenuClosedNotification` | waits until menu dismisses | +| `wait --popup` | poll `"AXMenus"` or watch for `AXSubrole==AXPopover` child | polling fallback (250ms interval) | + +`AXObserver` requires a `CFRunLoop`. The implementation spins a dedicated thread, runs the loop for the duration of the timeout, then shuts it down. This is safe for single-shot CLI use — no persistent daemon needed in Phase 1. + +### Phase 3 — `list-surfaces` command + +Enumerate all currently visible transient surfaces for a given app: + +```json +{ + "surfaces": [ + { "type": "menu", "title": "Edit", "item_count": 12 }, + { "type": "sheet", "title": "Save", "ref": "@e1" } + ] +} +``` + +Implemented by reading `"AXMenus"`, inspecting `kAXFocusedWindowAttribute` children for sheets/popovers/dialogs, and returning a flat list. No tree traversal. + +### Phase 4 — Comment cleanup + +Remove all inline `//` comments from macOS crate files. Retain `///` doc-comments on public items where the function name alone is insufficient. + +Files to clean: `tree.rs`, `adapter.rs`, `actions.rs`, `screenshot.rs`, `app_ops.rs`, `input.rs`, `roles.rs`, `permissions.rs`. + +## Implementation Plan + +### Step 1: Add `--surface` to `SnapshotArgs` + +**File:** `src/cli.rs` + +```rust +#[derive(clap::ValueEnum, Clone, Debug, Default)] +pub enum Surface { + #[default] + Window, + Focused, + Menu, + Sheet, + Popover, + Alert, +} + +// In SnapshotArgs: +#[arg(long, value_enum, default_value_t = Surface::Window)] +pub surface: Surface, +``` + +**File:** `crates/core/src/adapter.rs` — extend `ScreenshotTarget` analog with a `SnapshotSurface` enum mirroring the CLI enum. + +### Step 2: Surface resolution in macOS adapter + +**File:** `crates/macos/src/tree.rs` — add `menu_element_for_pid`, `focused_surface_for_pid`, `sheet_for_window`: + +```rust +/// Returns the first open context menu element for the given PID, if any. +pub fn menu_element_for_pid(pid: i32) -> Option<AXElement> { … } + +/// Returns whatever kAXFocusedWindowAttribute points to (window, sheet, or dialog). +pub fn focused_surface_for_pid(pid: i32) -> Option<AXElement> { … } + +/// Returns the first AXSheet child of the given window element. +pub fn sheet_for_window(win: &AXElement) -> Option<AXElement> { … } +``` + +These are all single attribute reads — O(1) AX calls before any tree traversal begins. + +**File:** `crates/macos/src/adapter.rs` — update `get_tree` to dispatch on `SnapshotSurface`: + +```rust +fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError> { + let el = match opts.surface { + SnapshotSurface::Window => crate::tree::window_element_for(win.pid, &win.title), + SnapshotSurface::Focused => crate::tree::focused_surface_for_pid(win.pid) + .ok_or_else(|| AdapterError::internal("no focused surface"))?, + SnapshotSurface::Menu => crate::tree::menu_element_for_pid(win.pid) + .ok_or_else(|| AdapterError::not_found("no open context menu"))?, + SnapshotSurface::Sheet => { … } + SnapshotSurface::Popover => { … } + SnapshotSurface::Alert => { … } + }; + let mut visited = FxHashSet::default(); + crate::tree::build_subtree(&el, 0, opts.max_depth, opts.include_bounds, &mut visited) + .ok_or_else(|| AdapterError::internal("empty tree")) +} +``` + +### Step 3: `wait --menu` via AXObserver + +**File:** `crates/macos/src/wait.rs` (new file, ≤ 150 LOC) + +```rust +/// Blocks until a context menu opens on the given PID or the timeout elapses. +pub fn wait_for_menu(pid: i32, timeout_ms: u64) -> Result<(), AdapterError> { … } + +/// Blocks until the context menu on the given PID closes or the timeout elapses. +pub fn wait_for_menu_closed(pid: i32, timeout_ms: u64) -> Result<(), AdapterError> { … } +``` + +Implementation uses `AXObserverCreate`, `AXObserverAddNotification` with `kAXMenuOpenedNotification`/`kAXMenuClosedNotification`, then runs a `CFRunLoop` with a deadline timer on a dedicated thread. The main thread joins with the specified timeout. + +**File:** `crates/macos/src/lib.rs` — `pub mod wait;` + +### Step 4: Extend `WaitArgs` in CLI + +**File:** `src/cli.rs` + +```rust +/// Wait for a context menu to open (--menu) or close (--menu-closed) +#[arg(long)] +pub menu: bool, +#[arg(long)] +pub menu_closed: bool, +``` + +**File:** `crates/core/src/commands/wait.rs` — dispatch to `adapter.wait_for_surface(…)` with new `WaitSurface` enum. + +### Step 5: `list-surfaces` command + +**File:** `crates/core/src/commands/list_surfaces.rs` + +```rust +pub struct ListSurfacesArgs { + #[arg(long)] + pub app: Option<String>, +} +``` + +Output: +```json +{ + "version": "1.0", + "ok": true, + "command": "list-surfaces", + "data": { + "surfaces": [ + { "type": "menu", "item_count": 8 }, + { "type": "sheet", "title": "Save" } + ] + } +} +``` + +### Step 6: Comment cleanup + +Remove all inline `//` comments from `crates/macos/src/*.rs`. Rules: +- Delete comments that restate what the code does +- Delete comments that explain macOS constant values (constants are named) +- Keep `///` doc-comments on `pub` functions where the name alone is insufficient +- Keep `// SAFETY:` and `// kAX...` references where they provide non-obvious AX API context + +## Acceptance Criteria + +- [ ] `agent-desktop snapshot --app WhatsApp --surface menu` returns the open context menu tree within 0.5s (no full tree traversal) +- [ ] `agent-desktop snapshot --app Finder --surface sheet` returns the save sheet when one is open +- [ ] `agent-desktop snapshot --app Finder --surface focused` returns a focused dialog, sheet, or window — whichever has focus +- [ ] `agent-desktop wait --app TextEdit --menu --timeout 5000` blocks until a context menu opens and exits 0; exits with `TIMEOUT` error if none appears in 5s +- [ ] `agent-desktop list-surfaces --app Finder` lists all currently visible transient surfaces +- [ ] All existing 30 commands continue to pass +- [ ] No inline `//` comments remain in `crates/macos/src/` +- [ ] Snapshot of a context menu does NOT traverse the full app tree first +- [ ] `--surface menu` returns `APP_NOT_FOUND`-equivalent error (`ELEMENT_NOT_FOUND`) when no menu is open + +## File Checklist + +| File | Change | +|------|--------| +| `src/cli.rs` | Add `--surface`, `--menu`, `--menu-closed` args | +| `crates/core/src/adapter.rs` | Add `SnapshotSurface` enum to `TreeOptions`; add `wait_for_surface` trait method | +| `crates/core/src/commands/snapshot.rs` | Pass `surface` from args to `TreeOptions` | +| `crates/core/src/commands/wait.rs` | Add `WaitSurface` dispatch | +| `crates/core/src/commands/list_surfaces.rs` | New file | +| `crates/core/src/commands/mod.rs` | Register `list_surfaces` | +| `crates/macos/src/tree.rs` | Add `menu_element_for_pid`, `focused_surface_for_pid`, `sheet_for_window` | +| `crates/macos/src/adapter.rs` | Dispatch on `SnapshotSurface` in `get_tree`; implement `wait_for_surface` | +| `crates/macos/src/wait.rs` | New file — AXObserver-based surface waiting | +| `crates/macos/src/lib.rs` | `pub mod wait` | +| `crates/macos/src/*.rs` | Remove inline comments | + +## Non-Goals + +- Does NOT use `ScreenshotBackend legacy capture` for menu capture (screenshot of a menu is separate from its AX tree) +- Does NOT implement persistent AXObserver across invocations (Phase 4 daemon scope) +- Does NOT handle Electron/custom-rendered menus that bypass the AX API +- Does NOT add Windows or Linux surface detection (Phase 2) diff --git a/docs/plans/2026-02-19-fix-bugs-add-missing-commands-plan.md b/docs/plans/2026-02-19-fix-bugs-add-missing-commands-plan.md new file mode 100644 index 0000000..546ad83 --- /dev/null +++ b/docs/plans/2026-02-19-fix-bugs-add-missing-commands-plan.md @@ -0,0 +1,987 @@ +--- +title: "fix: Phase 1 bug fixes, AX-first execution, missing commands, adapter-ready architecture" +type: fix +date: 2026-02-19 +deepened: 2026-02-19 +--- + +# fix: Phase 1 Bug Fixes + AX-First Execution + Missing Commands + Adapter-Ready Architecture + +## Enhancement Summary + +**Deepened on:** 2026-02-19 +**Plans merged:** `fix-bugs-add-missing-commands-plan.md` + `fix-subtree-traversal-error-dedup-ax-only-plan.md` +**Research agents used:** AX-only execution, select semantics, agent-browser gap analysis, sub-window tree traversal, stderr duplication, cross-platform adapter design, drag-and-drop feasibility + +### Key Improvements Over Original Plan +1. **AX-first philosophy**: Every command uses Accessibility API as primary execution path. CGEvent/osascript kept ONLY for operations with no AX equivalent (drag, hover, modified key combos without menu bar entries) +2. **Corrected root causes**: Stderr duplication is NOT a code bug (removed from plan). Sub-window tree fix is a ONE-LINE change in `list_windows_impl`, not `window_element_for` +3. **Role-aware `select`**: Branches by AX role — AXPopUpButton (press→menu→press item), AXComboBox (set kAXValueAttribute), AXList/AXTable (set kAXSelectedChildrenAttribute) +4. **New commands from agent-browser gap analysis**: `check`/`uncheck` (idempotent), `scroll-to`, `wait --text`, `find` enhancements (`--count`, `--first`, `--nth`), `hover`, `drag` +5. **Adapter extensibility**: Flat `#[non_exhaustive]` Action enum, WindowOp enum, MouseEvent struct, single `mouse_event` method + +### AX-First Principle + +> Every command MUST use the Accessibility API as its primary execution mechanism. CGEvent mouse/keyboard synthesis and osascript are NEVER the primary path. They are kept ONLY as documented fallbacks for operations that have no AX equivalent (drag, hover, arbitrary screen-coordinate clicks, modified key combos not in any menu bar). + +**AX-Only Commands** (no CGEvent/osascript at all): +- `click`, `right-click`, `toggle`, `select`, `expand`, `collapse`, `set-value`, `focus`, `type` (via kAXValueAttribute) +- `focus-window` (via kAXMainAttribute + kAXRaiseAction) +- `close-app` graceful (via AX menu bar "Quit" item or kAXCloseButtonAttribute) +- `press` for Return/Escape/Space (via kAXConfirmAction/kAXCancelAction/kAXPressAction) +- `press` for cmd+shortcuts (via AX menu bar traversal matching kAXMenuItemCmdChar) +- `scroll` (via AXScrollBar kAXIncrementAction/kAXDecrementAction) +- `check`/`uncheck` (via kAXValueAttribute set on checkbox/switch) +- Window geometry: `resize-window`, `move-window`, `minimize`, `maximize`, `restore` + +**Hybrid Commands** (AX resolve coordinates → CGEvent execute): +- `drag` — AX resolves element bounds, CGEvent performs LeftMouseDown→Dragged→Up +- `hover` — AX resolves bounds, CGEvent MouseMoved (tooltips require real cursor movement) +- `mouse-move`, `mouse-click`, `mouse-down`, `mouse-up` — coordinate-based, CGEvent only + +**CGEvent-Only Commands** (no AX equivalent exists): +- `key-down`/`key-up` — holding modifier keys has no AX equivalent +- `press` for arbitrary key combos not in menu bar (e.g., `f5`, `ctrl+left`) — falls back to `AXUIElementPostKeyboardEvent`, then returns `ACTION_NOT_SUPPORTED` if both fail + +--- + +## Phase A: Critical Bug Fixes (P0 + P1) + +### A1. Fix sub-window tree missing elements + +**Problem:** `snapshot --app "System Settings"` only captures the sidebar; the content pane is never reached. + +**Root Cause (CORRECTED from original plan):** The bug is NOT in `window_element_for` or `build_subtree`. It's in `list_windows_impl` in `crates/macos/src/adapter.rs`. The CGWindow enumeration filters out windows with empty `kCGWindowName` via a `_ => continue` match arm. System Settings has a window whose CGWindow title is empty, so it's silently skipped. The content pane is NOT a separate AXWindow — it's inside a single AXSplitGroup within the one AXWindow, and `build_subtree` already recurses `kAXChildrenAttribute`, so it would find the content pane IF the window weren't filtered out. + +**Files:** `crates/macos/src/adapter.rs` + +**Fix:** ONE-LINE change in `list_windows_impl`: +```rust +// BEFORE (adapter.rs, inside the CGWindow title match): +_ => continue, + +// AFTER: +_ => app_name.clone(), +``` + +When `kCGWindowName` is empty or missing, use the application name as fallback instead of skipping the window entirely. This ensures System Settings' content pane window (and similar empty-titled windows in other apps) appears in the window list and gets traversed. + +**Verification:** `agent-desktop snapshot --app "System Settings"` returns both sidebar AND content pane elements (AXOutline + content controls like AXCheckBox, AXPopUpButton, etc.) + +### A2. Fix `select` false success (P0) + +**Problem:** `select @e4 "Courier"` returns `ok:true` but value stays "Helvetica". + +**Root cause:** `actions.rs` maps `Action::Select` to a bare `kAXPressAction` click — it just clicks the element instead of performing role-aware selection. + +**Files:** `crates/macos/src/actions.rs` + +**Critical insight from research:** Popup menu children DON'T EXIST in the AX tree until the menu is physically opened. You cannot query kAXChildrenAttribute on a closed AXPopUpButton to enumerate its options. The `select` command's purpose is to SET a value when the caller already knows what value they want. Option DISCOVERY uses a different workflow: `click @popup` → `snapshot` → see menu items → `click @menuitem`. + +**Fix — role-aware branching:** + +1. **AXPopUpButton** (dropdown menus): + - `AXUIElementPerformAction(kAXPressAction)` to open the menu + - Wait up to 500ms for menu to appear (poll kAXChildrenAttribute on the popup) + - Walk menu children, find `AXMenuItem` whose `kAXTitleAttribute` matches the target value (case-insensitive, trimmed) + - `AXUIElementPerformAction(kAXPressAction)` on the matched menu item + - If no match found: press Escape to close menu, return `ELEMENT_NOT_FOUND` with suggestion listing available options + - Read back `kAXValueAttribute` to confirm change, include in `post_state` + +2. **AXComboBox** (editable dropdowns): + - `AXUIElementSetAttributeValue(kAXValueAttribute, value)` directly + - Read back to confirm, include in `post_state` + +3. **AXList / AXTable** (list selections): + - Find child element matching `value` by name + - `AXUIElementSetAttributeValue(kAXSelectedChildrenAttribute, [matched_child])` + - Read back to confirm + +4. **Any other role**: Return `ACTION_NOT_SUPPORTED` with suggestion: "Element role '{role}' doesn't support select. Use 'click' or 'set-value' instead." + +### A3. Fix `toggle` false success (P0) + +**Problem:** `toggle @e1` on a textfield returns `ok:true` with no effect. + +**Root cause:** `actions.rs` maps `Action::Toggle` to `kAXPressAction` unconditionally — no role check. + +**Files:** `crates/macos/src/actions.rs`, `crates/core/src/commands/toggle.rs` + +**Fix:** +- Before executing, check element role from RefEntry +- Supported roles: `checkbox`, `switch`, `radiobutton`, `togglebutton`, `menuitemcheckbox`, `menuitemradio` +- For supported roles: `AXUIElementPerformAction(kAXPressAction)`, then read back `kAXValueAttribute` to confirm state changed +- For unsupported roles: return `ACTION_NOT_SUPPORTED` with suggestion: "Toggle only works on checkboxes, switches, and radio buttons. Use 'click' for other elements." + +### A4. Fix `launch` window detection (P1) + +**Problem:** `launch Calculator` returns error "no window found" even though the app launched fine. + +**Root cause:** `app_ops.rs::launch_app_impl` non-wait path only sleeps 500ms. Many apps need 2-5s for window. + +**Files:** `crates/macos/src/app_ops.rs` + +**Fix:** +- Default behavior: poll `list_windows()` every 200ms for up to 5s (matching the wait path's pattern) +- When the app process exists but no window: return success with `"window": null` and `"note": "App launched but no window detected yet. Use 'wait --window' to poll."` +- Only return error if the `open -a` command itself fails (non-zero exit code) + +### A5. Fix `close-app` not actually closing (P1) + +**Problem:** `close-app Calculator` returns `ok:true` but app keeps running. + +**Root cause:** The graceful quit via osascript doesn't verify termination. + +**Files:** `crates/macos/src/app_ops.rs` + +**Fix (AX-first):** +- **Primary:** Find "Quit" menu item via AX menu bar traversal (`kAXMenuBarAttribute` → last `AXMenuBarItem` → walk `AXMenuItem` children → find item with title containing "Quit"), then `kAXPressAction` on it +- **Fallback:** If no menu bar accessible, use `AXUIElementSetAttributeValue(kAXHiddenAttribute)` or the existing osascript quit +- After sending quit, poll `list_apps()` every 200ms for up to 3s to verify app exited +- If still running after timeout: return `{ "ok": true, "data": { "closed": false, "note": "Quit requested but app may still be running. Use --force to kill." } }` + +### A6. Fix `expand`/`collapse` wrong error code (P1) + +**Problem:** Returns `ACTION_FAILED` with raw AX error code (-25205) instead of `ACTION_NOT_SUPPORTED`. + +**Files:** `crates/macos/src/actions.rs` + +**Fix:** +- Before attempting `AXExpand`/`AXCollapse`, call `AXUIElementCopyActionNames` and check if the action string exists +- If not present: return `AdapterError::new(ErrorCode::ActionNotSupported, "This element doesn't support expand/collapse").with_suggestion("Try 'click' to open it instead.")` +- If present but fails: keep `ACTION_FAILED` with the AX error detail + +### A7. Fix `get --property bounds` returning null + +**Problem:** `get @e1 --property bounds` returns `null` even though element has screen position. + +**Files:** `crates/core/src/commands/get.rs`, `crates/macos/src/adapter.rs` + +**Fix:** +- `get bounds` should resolve the element handle via `resolve_element`, then query `kAXPositionAttribute` + `kAXSizeAttribute` LIVE from the AX tree +- Don't rely on RefEntry's cached bounds — query the adapter directly +- Add `get_element_bounds(handle: &NativeHandle) -> Result<Option<Rect>>` to PlatformAdapter trait (with default `not_supported()`) +- macOS impl queries `AXUIElementCopyAttributeValue` for position and size + +### A8. Fix `focus-window` to use AX instead of osascript + +**Problem:** `focus_window_impl` uses `osascript` → `tell application X to activate`. + +**Files:** `crates/macos/src/app_ops.rs` + +**Fix (AX-first):** +- Get AXApplication element for the target PID +- `AXUIElementSetAttributeValue(appEl, kAXFrontmostAttribute, kCFBooleanTrue)` to bring app to front +- Get the target window element +- `AXUIElementSetAttributeValue(winEl, kAXMainAttribute, kCFBooleanTrue)` to make it the main window +- `AXUIElementPerformAction(winEl, kAXRaiseAction)` to raise it +- Remove osascript path entirely + +### A9. Replace CGEvent fallbacks in action execution + +**Problem:** `click`, `right-click`, `toggle`, `select` all fall back to `cg_mouse_click` CGEvent synthesis when AX action fails. + +**Files:** `crates/macos/src/actions.rs` + +**Fix:** +- **`Action::Click`**: `kAXPressAction` only. If AX error, propagate `ACTION_FAILED` (no CGEvent fallback) +- **`Action::RightClick`**: `AXShowMenu` only. Remove CGEvent fallback. Already implemented correctly; just delete the fallback branch +- **`Action::DoubleClick`**: Try `AXOpen` first. If unsupported, `kAXPressAction` twice with 50ms sleep between. If still fails, return `ACTION_FAILED` +- **`Action::Toggle`**: `kAXPressAction` only after role validation (A3). Remove CGEvent fallback +- **`Action::Select`**: Role-aware AX implementation (A2). Remove CGEvent fallback +- **`Action::Scroll`**: Replace `CGEvent::new_scroll_event` with AX scroll (see A10) +- Remove `cg_mouse_click` helper function entirely from `actions.rs` +- Remove CGEvent mouse imports from `actions.rs` + +### A10. Replace CGEvent scroll with AX scroll + +**Problem:** `Action::Scroll` uses `CGEvent::new_scroll_event` (HID injection). + +**Files:** `crates/macos/src/actions.rs` + +**Fix:** +- Find the `AXScrollArea` or `AXScrollBar` associated with the target element +- For vertical scroll: find the vertical `AXScrollBar` child, then: + - Scroll down: `AXUIElementPerformAction(scrollBar, kAXIncrementAction)` repeated `amount` times + - Scroll up: `AXUIElementPerformAction(scrollBar, kAXDecrementAction)` repeated `amount` times +- For horizontal: same with horizontal `AXScrollBar` and corresponding actions +- If no scroll bar found: return `ACTION_NOT_SUPPORTED` with suggestion "Element is not scrollable" +- Remove `CGEvent::new_scroll_event` import and usage + +### A11. Replace CGEvent keyboard input with AX equivalents + +**Problem:** `Action::TypeText` uses `synthesize_text` (CGEvent keyboard per character). `Action::PressKey` uses `synthesize_key` (CGEvent keyboard). + +**Files:** `crates/macos/src/actions.rs`, `crates/macos/src/input.rs` + +**Fix for TypeText (AX-first):** +- Set `kAXFocusedAttribute = true` on target element +- Read current `kAXValueAttribute` +- `AXUIElementSetAttributeValue(kAXValueAttribute, newText)` — this replaces the full value +- For append semantics: read current value, concatenate, set combined value +- The `type` command already has this distinction: if ref provided, set value on ref. If no ref, this is a "type into focused element" which still needs kAXValueAttribute set on the focused element + +**Fix for PressKey (AX-first, multi-strategy):** +1. **Simple keys → AX actions on focused element:** + - `return`/`enter` → `kAXConfirmAction` + - `escape`/`esc` → `kAXCancelAction` + - `space` → `kAXPressAction` + - `tab` → `kAXNextContentsAction` or move focus + - Arrow keys on sliders → `kAXIncrementAction` / `kAXDecrementAction` + +2. **Modifier combos → AX menu bar traversal:** + - Get `kAXMenuBarAttribute` from app element + - Walk `AXMenuBarItem` → expand each → walk `AXMenuItem` children + - Match `kAXMenuItemCmdChar` (e.g., "C" for Cmd+C) and `kAXMenuItemCmdModifiers` + - `kAXPressAction` on matched `AXMenuItem` + +3. **Fallback for keys with no AX equivalent:** + - `AXUIElementPostKeyboardEvent(appEl, 0, keyCode, true/false)` — this is technically AX API (not CGEvent), available in accessibility-sys + - If that also fails: return `ACTION_NOT_SUPPORTED` with message: "No AX equivalent for key combo '{combo}'. This combo has no menu-bar action." + +4. **Remove `synthesize_key` and `synthesize_text` from `input.rs`** + - If `input.rs` becomes empty, delete it and remove from `lib.rs` + +### A12. Replace osascript in `press_for_app_impl` + +**Problem:** `press --app TextEdit "cmd+c"` uses osascript System Events `keystroke`/`key code`. + +**Files:** `crates/macos/src/app_ops.rs` + +**Fix (AX-first):** +- Activate target app via AX: `AXUIElementSetAttributeValue(appEl, kAXFrontmostAttribute, kCFBooleanTrue)` +- Use the same AX menu bar traversal strategy from A11 step 2 +- If menu bar match found: `kAXPressAction` on the `AXMenuItem` +- If no match: `AXUIElementPostKeyboardEvent` fallback +- Remove osascript `keystroke`/`key code` path entirely + +--- + +## Phase B: Improvements + +### B1. Fix `list-apps` data shape + bundle_id + +**Files:** `crates/core/src/commands/list_apps.rs`, `crates/macos/src/adapter.rs` + +**Fix:** +- Wrap `data` in `{"apps": [...]}` instead of bare array +- Populate `bundle_id` using `kAXBundleIdentifierAttribute` on the AXApplication element (pure AX, no osascript) +- Add `bundle_id: Option<String>` to `AppInfo` struct if not already present + +### B2. Fix `is` returning `false` vs "not applicable" + +**Files:** `crates/core/src/commands/is_check.rs` + +**Fix:** +- Return `{ "result": false, "applicable": false }` when querying a property that doesn't apply to the element's role +- Applicability rules: + - `checked` → applies to: checkbox, switch, radiobutton, menuitemcheckbox + - `expanded` → applies to: treeitem, combobox, disclosure, outlinerow + - `focused` → applies to: ALL interactive roles + - `visible` → applies to: ALL roles + - `enabled` → applies to: ALL interactive roles + +### B3. Add `--help` descriptions to all commands and flags + +**Files:** `src/cli.rs` + +**Fix:** +- Add `#[command(about = "...")]` to every `Commands` variant +- Add `#[arg(help = "...")]` to every flag +- Descriptions should be one-line, imperative mood, agent-oriented + +### B4. Improve `find` output for unnamed elements + +**Files:** `crates/core/src/commands/find.rs` + +**Fix:** +- When `name` is null, fall back to `description`, then `title`, then `"(unnamed {role})"` +- Include `description` field in find results when available + +### B5. Filter stale menu items from window snapshots + +**Files:** `crates/macos/src/adapter.rs` or snapshot engine + +**Fix:** +- In `snapshot --surface window`: filter out nodes with role `menuitem` or `menu` that are direct children of the app/window root (not nested inside actual window content) +- This handles the race condition where pressing escape doesn't immediately remove menu items from the AX tree + +--- + +## Phase C: New Commands — Idempotent State Control + +These commands address a critical gap: agents need idempotent state-setting, not just toggling. + +### C1. `check` / `uncheck` commands + +**Problem:** `toggle` is non-idempotent — calling it twice returns to original state. An agent that wants a checkbox ON must first query state, then conditionally toggle. `check` and `uncheck` are idempotent: `check` is always a no-op if already checked. + +**CLI:** +``` +agent-desktop check @e5 +agent-desktop uncheck @e5 +``` + +**Files:** `crates/core/src/commands/check.rs`, `crates/core/src/commands/uncheck.rs` + +**New Action variants:** `Action::Check`, `Action::Uncheck` + +**Implementation (macOS, AX-only):** +1. Resolve element, verify role is checkbox/switch/radiobutton/menuitemcheckbox +2. Read current `kAXValueAttribute` (0 = unchecked, 1 = checked) +3. For `check`: if already 1, return success with `"already_checked": true`. If 0, `kAXPressAction` +4. For `uncheck`: if already 0, return success with `"already_unchecked": true`. If 1, `kAXPressAction` +5. Read back value to confirm state change +6. For unsupported roles: return `ACTION_NOT_SUPPORTED` + +**Cross-platform:** All platforms support querying checkbox state and pressing — this is pure AX. + +### C2. `scroll-to` command (scroll element into view) + +**CLI:** +``` +agent-desktop scroll-to @e15 +agent-desktop scroll-to @e15 --align top +``` + +**File:** `crates/core/src/commands/scroll_to.rs` + +**Implementation (macOS, AX-only):** +1. Resolve target element +2. Query `kAXVisibleCharacterRangeAttribute` or check if element bounds are within scroll area visible rect +3. If not visible: find parent scroll area, use `AXUIElementPerformAction(kAXScrollToVisibleAction)` on the target element (this is a native AX action that scrolls the element into view) +4. If `kAXScrollToVisibleAction` not available: incrementally `kAXIncrementAction`/`kAXDecrementAction` on the scroll bar until element bounds are within viewport + +**Cross-platform:** `kAXScrollToVisibleAction` is macOS. Windows has `IUIAutomationScrollItemPattern.ScrollIntoView()`. Linux AT-SPI has `scroll_to` in Component interface. + +### C3. `wait --text` variant + +**CLI:** +``` +agent-desktop wait --text "Loading complete" --app TextEdit --timeout 5000 +``` + +**File:** `crates/core/src/commands/wait.rs` (extend existing) + +**Implementation:** +- Poll the app's AX tree every 200ms +- Search for any element whose `kAXValueAttribute` or `kAXTitleAttribute` or `kAXDescriptionAttribute` contains the target text (case-insensitive substring match) +- Return the matching element's ref when found +- Return `TIMEOUT` if not found within timeout + +--- + +## Phase D: Enhanced `find` Command + +### D1. `find --count` — return count only + +**CLI:** `agent-desktop find --app TextEdit --role button --count` + +**Output:** `{ "count": 3 }` instead of the full element list. + +**File:** `crates/core/src/commands/find.rs` + +### D2. `find --first` / `find --last` / `find --nth N` + +**CLI:** +``` +agent-desktop find --app TextEdit --role button --first +agent-desktop find --app TextEdit --role button --last +agent-desktop find --app TextEdit --role button --nth 2 +``` + +**Output:** Single element instead of array. + +**File:** `crates/core/src/commands/find.rs` + +### D3. `find --text` — search by text content + +**CLI:** `agent-desktop find --app TextEdit --text "Save"` + +**Implementation:** Match against `name`, `value`, `title`, `description` attributes (any match counts). + +**File:** `crates/core/src/commands/find.rs` + +--- + +## Phase E: New Commands — Mouse & Coordinate Control + +These commands require CGEvent synthesis because the Accessibility API has no concept of cursor movement or raw mouse events. They are the documented exceptions to the AX-first principle. + +### New Supporting Types + +```rust +pub struct Point { + pub x: f64, + pub y: f64, +} + +pub enum MouseButton { + Left, + Right, + Middle, +} + +pub struct DragParams { + pub from: Point, + pub to: Point, + pub duration_ms: Option<u64>, + pub steps: Option<u32>, +} +``` + +### E1. `drag` command + +**CLI:** +``` +agent-desktop drag --from @e5 --to @e12 +agent-desktop drag --from @e5 --to-xy 500,300 +agent-desktop drag --from-xy 100,200 --to-xy 500,300 +agent-desktop drag --from @e5 --to @e12 --duration 500 +``` + +**File:** `crates/core/src/commands/drag.rs` + +**Why CGEvent is required:** Pure AX drag is impossible on ALL three platforms. macOS has no `AXDragAction`. Windows `IDragPattern` is read-only (describes draggable state, doesn't perform drags). Linux AT-SPI has no drag interface. + +**Implementation (macOS — hybrid AX+CGEvent):** +1. **AX phase:** Resolve source/target coordinates from refs (via `get_element_bounds`) or use raw `--xy` coordinates +2. **CGEvent phase:** + - `CGEvent::new_mouse_event(LeftMouseDown, source_point)` → post + - Sleep 50ms (initial hold for drag registration) + - 10 intermediate `CGEvent::new_mouse_event(LeftMouseDragged, interpolated_point)` events, 8ms apart + - `CGEvent::new_mouse_event(LeftMouseUp, target_point)` → post + - Sleep 50ms (settle time for app to process drop) +3. **Total duration:** ~230ms default, configurable via `--duration` + +**Cross-platform mapping:** +- macOS: CGEvent `LeftMouseDown` → `LeftMouseDragged` (N steps) → `LeftMouseUp` +- Windows: `SendInput` with `MOUSEEVENTF_LEFTDOWN` → `MOUSEEVENTF_MOVE` (N steps) → `MOUSEEVENTF_LEFTUP` +- Linux: `atspi_generate_mouse_event` with `b1p` → `abs` (N steps) → `b1r` + +### E2. `hover` command + +**CLI:** +``` +agent-desktop hover @e5 +agent-desktop hover --xy 500,300 +agent-desktop hover @e5 --duration 1000 +``` + +**File:** `crates/core/src/commands/hover.rs` + +**Why CGEvent:** Tooltips and hover states require real cursor movement. `CGWarpMouseCursorPosition` is silent and won't trigger tooltips — must use `CGEvent::new_mouse_event(MouseMoved)`. + +**Implementation (macOS):** +- Resolve target point (from ref bounds center or `--xy`) +- `CGEvent::new_mouse_event(MouseMoved, target_point)` → post +- If `--duration` specified, hold position for that time (cursor stays put naturally) + +### E3. `mouse-down` / `mouse-up` commands + +**CLI:** +``` +agent-desktop mouse-down @e5 --button left +agent-desktop mouse-up --xy 500,300 --button left +``` + +**Files:** `crates/core/src/commands/mouse_down.rs`, `crates/core/src/commands/mouse_up.rs` + +**Implementation:** Individual CGEvent mouseDown/mouseUp events. Essential for custom drag sequences and long-press patterns. + +### E4. `mouse-move` command + +**CLI:** +``` +agent-desktop mouse-move --xy 500,300 +agent-desktop mouse-move --relative -10,20 +``` + +**File:** `crates/core/src/commands/mouse_move.rs` + +**Implementation:** `CGEvent::new_mouse_event(MouseMoved)`. `--relative` uses delta from current cursor position (query via `CGEvent::location()`). + +### E5. `mouse-click` command + +**CLI:** +``` +agent-desktop mouse-click --xy 500,300 +agent-desktop mouse-click --xy 500,300 --button right --count 2 +``` + +**File:** `crates/core/src/commands/mouse_click.rs` + +**Implementation:** Click at absolute coordinates, bypassing ref system. Useful when AX tree doesn't expose an element (e.g., Calculator display, game UIs, custom-rendered views). + +### E6. `triple-click` command + +**CLI:** `agent-desktop triple-click @e1` + +**File:** `crates/core/src/commands/triple_click.rs` + +**Implementation:** AX-first attempt: three rapid `kAXPressAction` calls with 30ms sleep between. If that doesn't trigger line/paragraph selection, fall back to CGEvent triple-click at element center. + +--- + +## Phase F: New Commands — Window Geometry + +All window geometry commands are **pure AX** — no CGEvent or osascript needed. + +### F1. `resize-window` + +**CLI:** `agent-desktop resize-window --app TextEdit --width 800 --height 600` + +**File:** `crates/core/src/commands/resize_window.rs` + +**Implementation (macOS, AX-only):** +- Get window element for app +- `AXUIElementSetAttributeValue(winEl, kAXSizeAttribute, AXValueCreate(kAXValueCGSizeType, &CGSize { width, height }))` + +### F2. `move-window` + +**CLI:** `agent-desktop move-window --app TextEdit --x 100 --y 100` + +**File:** `crates/core/src/commands/move_window.rs` + +**Implementation (macOS, AX-only):** +- `AXUIElementSetAttributeValue(winEl, kAXPositionAttribute, AXValueCreate(kAXValueCGPointType, &CGPoint { x, y }))` + +### F3. `minimize` / `maximize` / `restore` + +**CLI:** +``` +agent-desktop minimize --app TextEdit +agent-desktop maximize --app TextEdit +agent-desktop restore --app TextEdit +``` + +**Files:** `crates/core/src/commands/minimize.rs`, `crates/core/src/commands/maximize.rs`, `crates/core/src/commands/restore.rs` + +**Implementation (macOS, AX-only):** +- Minimize: `AXUIElementSetAttributeValue(winEl, kAXMinimizedAttribute, kCFBooleanTrue)` +- Maximize: `AXUIElementPerformAction(zoomButton, kAXPressAction)` where zoomButton is from `kAXZoomButtonAttribute` +- Restore: `AXUIElementSetAttributeValue(winEl, kAXMinimizedAttribute, kCFBooleanFalse)` for minimized windows, `AXUIElementPerformAction(zoomButton, kAXPressAction)` for fullscreen + +**Cross-platform:** +- Windows: `IUIAutomationTransformPattern.Move/Resize`, `IUIAutomationWindowPattern.SetWindowVisualState` +- Linux: `atspi_component_set_extents`, window manager D-Bus calls + +--- + +## Phase G: New Commands — Keyboard + +### G1. `key-down` / `key-up` + +**CLI:** +``` +agent-desktop key-down shift +agent-desktop key-up shift +``` + +**Files:** `crates/core/src/commands/key_down.rs`, `crates/core/src/commands/key_up.rs` + +**Implementation:** `AXUIElementPostKeyboardEvent` with only the down or up flag. If that doesn't work for the key, fall back to `CGEventCreateKeyboardEvent` with only key-down or key-up. Essential for modifier-hold sequences (hold Shift while clicking multiple items). + +### G2. `clear` command + +**CLI:** `agent-desktop clear @e1` + +**File:** `crates/core/src/commands/clear.rs` + +**Implementation (AX-only):** `AXUIElementSetAttributeValue(el, kAXValueAttribute, "")`. Simple set-value to empty string. + +### G3. `clipboard-clear` + +**CLI:** `agent-desktop clipboard-clear` + +**File:** `crates/core/src/commands/clipboard_clear.rs` + +**Implementation (macOS):** `NSPasteboard.generalPasteboard.clearContents()` via Cocoa FFI. + +--- + +## Phase H: Architecture — Adapter Extensibility + +### H1. Make Action enum `#[non_exhaustive]` + +**File:** `crates/core/src/action.rs` + +```rust +#[non_exhaustive] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Action { + Click, + DoubleClick, + RightClick, + SetValue(String), + SetFocus, + Expand, + Collapse, + Select(String), + Toggle, + Scroll(Direction, u32), + PressKey(KeyCombo), + TypeText(String), + Check, + Uncheck, + ScrollTo, + Drag(DragParams), + Hover, +} +``` + +Adding `#[non_exhaustive]` means platform adapters must have a `_ => Err(not_supported())` catch-all, which naturally handles new actions added in future phases. + +### H2. WindowOp enum for window geometry + +**File:** `crates/core/src/action.rs` (or new `crates/core/src/window_op.rs` if action.rs grows) + +```rust +pub enum WindowOp { + Resize { width: f64, height: f64 }, + Move { x: f64, y: f64 }, + Minimize, + Maximize, + Restore, +} +``` + +Single `window_op(&self, win: &WindowInfo, op: WindowOp)` method on PlatformAdapter instead of 5 separate methods. + +### H3. MouseEvent struct for raw mouse control + +**File:** `crates/core/src/action.rs` (or new `crates/core/src/mouse.rs`) + +```rust +pub struct MouseEvent { + pub kind: MouseEventKind, + pub point: Point, + pub button: MouseButton, +} + +pub enum MouseEventKind { + Move, + Down, + Up, + Click { count: u32 }, +} +``` + +Single `mouse_event(&self, event: MouseEvent)` method on PlatformAdapter. Handles `mouse-move`, `mouse-down`, `mouse-up`, `mouse-click`, and `hover` through one trait method. + +### H4. Coordinate Resolution Helper + +**File:** `crates/core/src/commands/coords.rs` + +```rust +pub enum CoordSource { + Ref(String), + Absolute(Point), +} + +pub fn resolve_coords( + source: &CoordSource, + adapter: &dyn PlatformAdapter, +) -> Result<Point, AppError> { + match source { + CoordSource::Ref(ref_id) => { + let (entry, handle) = resolve_ref(ref_id, adapter)?; + let bounds = adapter.get_element_bounds(&handle)? + .ok_or(AppError::internal("Element has no bounds"))?; + Ok(Point { + x: bounds.x + bounds.width / 2.0, + y: bounds.y + bounds.height / 2.0, + }) + } + CoordSource::Absolute(p) => Ok(p.clone()), + } +} +``` + +Used by `drag`, `hover`, `mouse-move`, `mouse-click`, `mouse-down`, `mouse-up`. All coordinate-based commands resolve through the same path. + +### H5. Action Pre-Check Pattern + +**File:** `crates/core/src/commands/action_check.rs` + +```rust +pub fn check_action_supported( + entry: &RefEntry, + required_roles: &[&str], + action_name: &str, + suggestion: &str, +) -> Result<(), AppError> { + if !required_roles.contains(&entry.role.as_str()) { + return Err(AppError::Adapter(AdapterError::new( + ErrorCode::ActionNotSupported, + format!("'{}' is not supported on role '{}'", action_name, entry.role), + ).with_suggestion(suggestion))); + } + Ok(()) +} +``` + +Used by `toggle`, `check`, `uncheck`, `expand`, `collapse`, `select` to validate role before attempting action. + +### H6. Extended PlatformAdapter Trait + +**File:** `crates/core/src/adapter.rs` + +```rust +pub trait PlatformAdapter: Send + Sync { + // --- Discovery --- + fn list_windows(&self, filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> { not_supported() } + fn list_apps(&self) -> Result<Vec<AppInfo>, AdapterError> { not_supported() } + fn get_tree(&self, win: &WindowInfo, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError> { not_supported() } + fn resolve_element(&self, entry: &RefEntry) -> Result<NativeHandle, AdapterError> { not_supported() } + fn permission_report(&self) -> PermissionReport { PermissionReport::unknown() } + + // --- AX Actions (primary execution path) --- + fn execute_action(&self, handle: &NativeHandle, request: ActionRequest) -> Result<ActionResult, AdapterError> { not_supported() } + fn get_element_bounds(&self, handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> { not_supported() } + + // --- Window Management --- + fn focus_window(&self, win: &WindowInfo) -> Result<(), AdapterError> { not_supported() } + fn window_op(&self, win: &WindowInfo, op: WindowOp) -> Result<(), AdapterError> { not_supported() } + fn launch_app(&self, id: &str, wait: bool) -> Result<WindowInfo, AdapterError> { not_supported() } + fn close_app(&self, id: &str, force: bool) -> Result<(), AdapterError> { not_supported() } + + // --- Mouse/Coordinate (CGEvent-based, documented exceptions) --- + fn mouse_event(&self, event: MouseEvent) -> Result<(), AdapterError> { not_supported() } + fn drag(&self, params: DragParams) -> Result<(), AdapterError> { not_supported() } + + // --- Media --- + fn screenshot(&self, target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> { not_supported() } + fn get_clipboard(&self) -> Result<String, AdapterError> { not_supported() } + fn set_clipboard(&self, text: &str) -> Result<(), AdapterError> { not_supported() } + fn clear_clipboard(&self) -> Result<(), AdapterError> { not_supported() } + + // --- App-Level Keyboard --- + fn press_key_for_app(&self, app: &str, combo: &KeyCombo) -> Result<ActionResult, AdapterError> { not_supported() } + fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> { not_supported() } + + // --- Surface Management --- + fn wait_for_menu(&self, pid: i32, open: bool, timeout: u64) -> Result<(), AdapterError> { not_supported() } + fn list_surfaces(&self, pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> { not_supported() } +} +``` + +All methods default to `not_supported()`. Phase 2 Windows/Linux adapters implement what they can — everything else gracefully fails with `PLATFORM_NOT_SUPPORTED`. + +--- + +## Implementation Order + +### Sprint 1: Critical Bug Fixes — AX-First Foundation (2-3 days) + +Priority: Stop commands from lying about success, fix the tree traversal. + +1. **A1** — Sub-window tree fix (ONE LINE in `list_windows_impl`) +2. **A9** — Remove CGEvent fallbacks from click/right-click/toggle/select in `actions.rs` +3. **A2** — Role-aware `select` implementation +4. **A3** — Toggle role validation +5. **A6** — Expand/collapse error codes +6. **A10** — Replace CGEvent scroll with AX scroll bar increment/decrement +7. **B3** — Help descriptions (touches `cli.rs` only, low risk, high value) + +**Verify:** `cargo clippy --all-targets -- -D warnings && cargo test --workspace` + +### Sprint 2: AX-First Keyboard & Window (2-3 days) + +Priority: Eliminate remaining osascript and CGEvent keyboard paths. + +8. **A11** — Replace CGEvent keyboard with AX (TypeText via kAXValueAttribute, PressKey via menu bar traversal + AXUIElementPostKeyboardEvent) +9. **A12** — Replace osascript in `press_for_app_impl` +10. **A8** — Replace osascript in `focus_window_impl` +11. **A5** — Fix close-app with AX menu bar "Quit" + verification +12. **A4** — Fix launch window detection polling +13. **A7** — Fix get bounds returning null + +**After this sprint:** `input.rs` functions `synthesize_key` and `synthesize_text` should be dead code. Remove them and potentially the file. + +### Sprint 3: Improvements + New Idempotent Commands (1-2 days) + +14. **B1** — list-apps data shape + bundle_id +15. **B2** — is false vs not-applicable +16. **B4** — find unnamed elements +17. **B5** — stale menu items in snapshots +18. **C1** — `check` / `uncheck` commands (AX-only) +19. **C2** — `scroll-to` command (AX-only) +20. **C3** — `wait --text` variant + +### Sprint 4: Architecture + Find Enhancements (1-2 days) + +21. **H1** — Make Action enum `#[non_exhaustive]`, add new variants +22. **H2** — WindowOp enum +23. **H3** — MouseEvent struct +24. **H4** — Coordinate resolution helper +25. **H5** — Action pre-check pattern +26. **H6** — Extend PlatformAdapter trait +27. **D1-D3** — find --count, --first/--last/--nth, --text + +### Sprint 5: Mouse/Coordinate Commands (2-3 days) + +28. **E4** — mouse-move +29. **E2** — hover +30. **E5** — mouse-click +31. **E3** — mouse-down / mouse-up +32. **E6** — triple-click +33. **E1** — drag (most complex, depends on mouse-down/up being solid) + +### Sprint 6: Window Geometry + Misc (1-2 days) + +34. **F1** — resize-window (AX-only) +35. **F2** — move-window (AX-only) +36. **F3** — minimize / maximize / restore (AX-only) +37. **G1** — key-down / key-up +38. **G2** — clear (AX-only) +39. **G3** — clipboard-clear + +### Sprint 7: Testing & Polish (1 day) + +40. Full agentic test re-run (same task as original test in `tests/agentic_test_notes.md`) +41. Verify all 7 original bugs are fixed +42. Test all new commands end-to-end +43. Test cross-app drag (Finder → TextEdit file drag) +44. Verify no CGEvent imports remain in `actions.rs` +45. Verify no osascript calls remain in `app_ops.rs` except `launch_app_impl` (`open -a`) and `close_app_impl --force` (`pkill`) + +--- + +## Command Count After Implementation + +| Category | Phase 1 (existing) | New | Total | +|----------|-------------------|-----|-------| +| App/Window | 5 | 5 (resize, move, minimize, maximize, restore) | 10 | +| Observation | 7 | 3 (find --count/--first/--nth/--text via flags) | 7* | +| Interaction | 11 | 5 (check, uncheck, triple-click, scroll-to, clear) | 16 | +| Mouse/Coord | 0 | 5 (drag, hover, mouse-move, mouse-click, mouse-down/up) | 5 | +| Keyboard | 1 | 2 (key-down, key-up) | 3 | +| Clipboard | 2 | 1 (clipboard-clear) | 3 | +| Wait | 1 | 1 (wait --text via flag) | 1* | +| System | 3 | 0 | 3 | +| Batch | 1 | 0 | 1 | +| **Total** | **31** | **19** | **49** | + +*find enhancements and wait --text are flags on existing commands, not new command files + +--- + +## Cross-Platform Adapter Contract + +| Pattern | Commands | macOS | Windows | Linux | +|---------|----------|-------|---------|-------| +| **AX Action** | click, toggle, expand, collapse, select, set-value, focus, check, uncheck, scroll, scroll-to, clear | AXUIElementPerformAction / SetAttributeValue | IUIAutomationElement.Invoke/Toggle/ScrollIntoView | atspi_component_do_action | +| **AX Value Set** | type, set-value, clear | AXUIElementSetAttributeValue(kAXValueAttribute) | IUIAutomationValuePattern.SetValue | atspi_editable_text_set_text_contents | +| **AX Menu Traverse** | press (cmd+shortcuts), close-app (graceful) | kAXMenuBarAttribute → walk → kAXPressAction | IUIAutomationElement.FindAll for menu items | atspi_accessible_get_child_at_index | +| **AX Key Post** | press (fallback for non-menu keys) | AXUIElementPostKeyboardEvent | IUIAutomationElement.SendKeys | atspi_generate_keyboard_event | +| **AX Window Prop** | resize, move, minimize, maximize, restore, focus-window | AXUIElementSetAttributeValue (Position/Size/Minimized/Main) | IUIAutomationTransformPattern / WindowPattern | atspi_component_set_extents | +| **CGEvent Mouse** | drag, hover, mouse-move/click/down/up, triple-click | CGEvent sequences | SendInput sequences | atspi_generate_mouse_event | +| **CGEvent Keyboard** | key-down, key-up | CGEvent / AXUIElementPostKeyboardEvent | SendInput VK_ codes | atspi_generate_keyboard_event | +| **AX Attribute Get** | get, is, snapshot, find, wait --text | AXUIElementCopyAttributeValue | IUIAutomationElement.GetPropertyValue | atspi_accessible_get_* | +| **OS Shell** | launch, close-app --force | `open -a` / `pkill` | CreateProcess / TerminateProcess | xdg-open / kill | +| **OS Clipboard** | clipboard-* | NSPasteboard | Win32 Clipboard API | wl-copy / xclip | + +--- + +## Removed / Downgraded Items + +### Stderr Duplication (was A1 in original plan — REMOVED) + +**Research finding:** There is NO stderr duplication bug in the code. `emit_json()` writes exactly once to stdout via `BufWriter`. `Cli::try_parse()` does NOT write to stderr on its own. The reported "duplication" was most likely `cargo run` output noise (rustc warnings, build progress) or shell stream merging (`2>&1`). The release binary does NOT exhibit this behavior. + +**Action:** No code change needed. Verified by reading `main.rs` error paths — all produce exactly 1 JSON object on stdout. If this resurfaces, the diagnostic is: run `./target/release/agent-desktop click @invalid 2>/dev/null | wc -l` and verify it outputs exactly 1 line. + +### Commands NOT Added (from agent-browser gap analysis) + +These agent-browser commands were evaluated and intentionally excluded: + +| Command | Reason for exclusion | +|---------|---------------------| +| `navigate`, `go_back`, `go_forward`, `reload`, `wait_for_page` | Browser-only, no desktop equivalent | +| `evaluate`, `console`, `network` | Browser DevTools, no desktop equivalent | +| `pdf`, `snapshot_html`, `save_html` | Browser DOM, no desktop equivalent | +| `get_cookies`, `set_cookies`, `clear_cookies` | Browser storage, no desktop equivalent | +| `file_upload`, `file_download` | Browser file dialogs — desktop equivalent would be AX file picker interaction, deferred to Phase 2 | +| `dialog_accept`, `dialog_dismiss` | Browser dialogs — desktop alert/dialog handling is possible via AX but complex; deferred to Phase 2 | + +--- + +## Acceptance Criteria + +### Bug Fixes +- [ ] `agent-desktop snapshot --app "System Settings"` returns both sidebar AND content pane elements +- [ ] `agent-desktop select @popup "Courier"` actually changes the value and reports new value in response +- [ ] `agent-desktop toggle @textfield` returns `ACTION_NOT_SUPPORTED` (not false success) +- [ ] `agent-desktop launch Calculator` succeeds and returns window info (with polling) +- [ ] `agent-desktop close-app Calculator` reports actual close status, not premature success +- [ ] `agent-desktop expand @combobox` returns `ACTION_NOT_SUPPORTED` with suggestion (not raw AX error) +- [ ] `agent-desktop get @e1 --property bounds` returns actual coordinates (not null) + +### AX-First Execution +- [ ] `actions.rs` contains ZERO `cg_mouse_click` calls or CGEvent mouse imports +- [ ] `actions.rs` Click/RightClick/Toggle/Select use only AXUIElementPerformAction +- [ ] `actions.rs` Scroll uses AXScrollBar increment/decrement (no CGEvent::new_scroll_event) +- [ ] `actions.rs` TypeText uses kAXValueAttribute set (no synthesize_text) +- [ ] `app_ops.rs` focus_window uses AX (no osascript) +- [ ] `app_ops.rs` close_app graceful uses AX menu bar (no osascript quit) +- [ ] `app_ops.rs` press_for_app uses AX menu bar traversal (no osascript keystroke) +- [ ] `input.rs` synthesize_key and synthesize_text removed or only used by CGEvent-based commands (drag, hover, mouse-*, key-down/up) + +### New Commands +- [ ] `agent-desktop check @checkbox` sets checked state idempotently +- [ ] `agent-desktop uncheck @checkbox` sets unchecked state idempotently +- [ ] `agent-desktop scroll-to @e15` scrolls element into view +- [ ] `agent-desktop wait --text "Done" --app TextEdit --timeout 3000` waits for text appearance +- [ ] `agent-desktop find --app TextEdit --role button --count` returns count +- [ ] `agent-desktop find --app TextEdit --role button --first` returns single element +- [ ] `agent-desktop drag --from @e1 --to @e5` performs drag via CGEvent +- [ ] `agent-desktop hover @e5` moves cursor to element +- [ ] `agent-desktop minimize --app TextEdit` minimizes window via AX +- [ ] `agent-desktop resize-window --app TextEdit --width 800 --height 600` resizes via AX +- [ ] `agent-desktop mouse-click --xy 500,300` clicks at coordinates +- [ ] `agent-desktop key-down shift` holds shift modifier + +### Quality +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo test --workspace` passes +- [ ] No file exceeds 400 LOC +- [ ] `agent-desktop --help` shows descriptions for all commands +- [ ] Error JSON appears exactly once on stdout for all error cases +- [ ] Full agentic test re-run shows 0 bugs, 0 false successes + +--- + +## Files to Change (Summary) + +| File | Change | +|------|--------| +| `crates/macos/src/adapter.rs` | ONE-LINE fix for sub-window tree; add `get_element_bounds`; populate `bundle_id` in list_apps | +| `crates/macos/src/actions.rs` | Remove ALL CGEvent mouse fallbacks; rewrite select (role-aware), toggle (role check), scroll (AX scroll bar), type (kAXValueAttribute), press (menu traverse + AXUIElementPostKeyboardEvent) | +| `crates/macos/src/input.rs` | Remove `synthesize_key`, `synthesize_text`; keep file only if CGEvent mouse commands need keyboard helpers, otherwise delete | +| `crates/macos/src/app_ops.rs` | Replace osascript in focus_window, close_app, press_for_app with AX equivalents | +| `crates/core/src/adapter.rs` | Extend PlatformAdapter with `get_element_bounds`, `window_op`, `mouse_event`, `drag`, `clear_clipboard`; add `#[non_exhaustive]` consideration | +| `crates/core/src/action.rs` | Add `#[non_exhaustive]`, new variants: Check, Uncheck, ScrollTo, Drag, Hover; add Point, MouseButton, DragParams, WindowOp, MouseEvent types | +| `src/cli.rs` | Add `#[command(about)]` to all variants; add new command structs for all Phase C-G commands | +| `src/dispatch.rs` | Add match arms for all new commands | +| `crates/core/src/commands/` | New files: `check.rs`, `uncheck.rs`, `scroll_to.rs`, `drag.rs`, `hover.rs`, `mouse_down.rs`, `mouse_up.rs`, `mouse_move.rs`, `mouse_click.rs`, `triple_click.rs`, `resize_window.rs`, `move_window.rs`, `minimize.rs`, `maximize.rs`, `restore.rs`, `key_down.rs`, `key_up.rs`, `clear.rs`, `clipboard_clear.rs`, `coords.rs`, `action_check.rs` | +| `crates/core/src/commands/find.rs` | Add `--count`, `--first`, `--last`, `--nth`, `--text` flags | +| `crates/core/src/commands/wait.rs` | Add `--text` flag | +| `crates/core/src/commands/get.rs` | Fix bounds to query live from AX tree | +| `crates/core/src/commands/is_check.rs` | Add `applicable` field based on role | +| `crates/core/src/commands/toggle.rs` | Add role validation | +| `crates/core/src/commands/list_apps.rs` | Wrap data in `{"apps": [...]}` | + +--- + +## References + +- Agentic test notes: `tests/agentic_test_notes.md` +- PRD v2.0: `docs/agent_desktop_prd_v2.pdf` +- Architecture brainstorm: `docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md` +- Agent-browser command inventory: `github.com/vercel-labs/agent-browser` (used for gap analysis) +- macOS AX API: `AXUIElementPerformAction`, `AXUIElementSetAttributeValue`, `AXUIElementCopyActionNames`, `AXUIElementPostKeyboardEvent` +- macOS AX menu traversal: `kAXMenuBarAttribute` → `AXMenuBarItem` → `AXMenu` → `AXMenuItem` → `kAXMenuItemCmdChar` + `kAXMenuItemCmdModifiers` +- macOS AX scroll: `kAXScrollBarAttribute` → `kAXIncrementAction` / `kAXDecrementAction` +- Pure AX drag is impossible on all 3 platforms — CGEvent mouse synthesis required +- `accessibility-sys` crate: provides all kAX constants and AXUIElement FFI bindings diff --git a/docs/plans/2026-02-19-fix-subtree-traversal-error-dedup-ax-only-plan.md b/docs/plans/2026-02-19-fix-subtree-traversal-error-dedup-ax-only-plan.md new file mode 100644 index 0000000..356f454 --- /dev/null +++ b/docs/plans/2026-02-19-fix-subtree-traversal-error-dedup-ax-only-plan.md @@ -0,0 +1,178 @@ +--- +title: "fix: Sub-window tree, duplicate error output, AX-only commands" +type: fix +date: 2026-02-19 +--- + +# fix: Sub-window tree traversal, duplicate error output, and AX-only command execution + +## Overview + +Three bugs identified during live testing against macOS System Settings: + +1. **Sub-window tree missing** — `snapshot` only captures the sidebar; the content pane (a child AXWindow) is never reached because `window_element_for` queries only `kAXWindowsAttribute`, which excludes embedded sub-windows. +2. **Error emitted twice** — An invalid-argument error JSON appears twice in combined stdout+stderr output. Clap may write its own formatted output to stderr while `main.rs` independently emits the JSON to stdout. +3. **CGEvent and osascript in command paths** — `press`, `type`, `click` (fallback), `double-click`, `right-click` (fallback), `scroll`, `toggle` (fallback), `select` (fallback), `close-app` (graceful), `focus-window` all use CGEvent mouse/keyboard injection or osascript System Events. Every command must be AX-API-only; no CGEvent, no HID injection, no osascript for input delivery. + +--- + +## Problem Analysis + +### 1 — Sub-window tree (`crates/macos/src/tree.rs:53–75`) + +`window_element_for` walks `kAXWindowsAttribute` to find a matching AXWindow, falling back to the first window or the application element. On macOS, `kAXWindowsAttribute` returns only the **primary top-level windows** registered with the window server. + +Apps like System Settings expose their content pane as a **separate AXWindow** accessible via `kAXChildrenAttribute` on the app element (or on the main window). This child window never appears in `kAXWindowsAttribute`, so the tree traversal starts from the sidebar window and never crosses into the content pane. + +Concrete evidence: snapshotting System Settings shows `AXOutline (Sidebar)` with 40+ tree items but no content-pane elements. Taking a screenshot confirms the content pane (Appearance options) is present but invisible to the AX tree walk. + +**Root cause:** `window_element_for` stops after `kAXWindowsAttribute`; it does not consult `kAXChildrenAttribute` on either the app element or on the matched window. + +### 2 — Duplicate error output (`src/main.rs:10–30`) + +`emit_json` writes exactly once to stdout (`src/main.rs:122–130`). However, the Claude Bash tool (and any caller that merges stdout + stderr) sees two copies of the error. Clap v4's error type, when converted via `.to_string()` or when the process exits via `std::process::exit`, may flush a buffered internal writer to stderr that contains the same message text. Independently, `BufWriter` wrapping stdout's lock means our JSON flushes to stdout. Together both streams appear in a combined capture. + +Fix: configure the clap command to redirect its own error output to a null sink so our JSON is the sole output on any stream. + +### 3 — CGEvent and osascript usage + +| File | Location | Non-AX mechanism | +|------|----------|-----------------| +| `crates/macos/src/actions.rs` | `Action::Click` fallback | `cg_mouse_click` → `CGEvent::new_mouse_event` | +| `crates/macos/src/actions.rs` | `Action::DoubleClick` | `cg_mouse_click` → `CGEvent::new_mouse_event` | +| `crates/macos/src/actions.rs` | `Action::RightClick` fallback | `cg_mouse_click` → `CGEvent::new_mouse_event` | +| `crates/macos/src/actions.rs` | `Action::Toggle` fallback | `cg_mouse_click` → `CGEvent::new_mouse_event` | +| `crates/macos/src/actions.rs` | `Action::Select` fallback | `cg_mouse_click` → `CGEvent::new_mouse_event` | +| `crates/macos/src/actions.rs` | `Action::Scroll` | `CGEvent::new_scroll_event` → HID | +| `crates/macos/src/actions.rs` | `Action::TypeText` | `synthesize_text` → `CGEvent::new_keyboard_event` | +| `crates/macos/src/actions.rs` | `Action::PressKey` | `synthesize_key` → `CGEvent::new_keyboard_event` | +| `crates/macos/src/input.rs` | `synthesize_key` | `CGEvent::new_keyboard_event` + HID post | +| `crates/macos/src/input.rs` | `synthesize_text` | `CGEvent::new_keyboard_event` + HID post | +| `crates/macos/src/app_ops.rs:27–39` | `press_for_app_impl` | `osascript` → System Events `keystroke`/`key code` | +| `crates/macos/src/app_ops.rs:9–18` | `focus_window_impl` | `osascript` → `tell application X to activate` | +| `crates/macos/src/app_ops.rs:150–173` | `close_app_impl` | `osascript` → quit / `pkill` | + +**AX replacements available on macOS:** + +| Replaced mechanism | AX replacement | +|-------------------|----------------| +| Mouse click (CGEvent) | `AXUIElementPerformAction(kAXPressAction)` | +| Double-click (CGEvent) | `AXUIElementPerformAction("AXOpen")` or two sequential `kAXPressAction` calls | +| Right-click (CGEvent) | `AXUIElementPerformAction("AXShowMenu")` (already implemented; just remove fallback) | +| Scroll (CGEvent) | `AXUIElementPerformAction(kAXScrollDownAction / kAXScrollUpAction / kAXScrollLeftAction / kAXScrollRightAction)` | +| Text input (CGEvent keyboard) | `AXUIElementSetAttributeValue(kAXValueAttribute, newText)` after `kAXFocusedAttribute = true` | +| Key press for Return/Escape/Space | `AXUIElementPerformAction(kAXConfirmAction / kAXCancelAction / kAXPressAction)` on focused element | +| Key combo shortcuts (cmd+c etc.) | Traverse menu bar AX tree: find `AXMenuItem` where `AXMenuItemCmdChar` + `AXMenuItemCmdModifiers` match, then `kAXPressAction` | +| Window activation (osascript) | `AXUIElementSetAttributeValue(windowEl, kAXMainAttribute, kCFBooleanTrue)` | +| Window close (osascript) | `AXUIElementPerformAction(kAXCloseButtonAttribute child, kAXPressAction)` | + +**Limitation:** Arbitrary key combos with no menu-bar equivalent (e.g., `f5`, custom shortcuts not in any menu) cannot be delivered via pure AX API. These should return `ACTION_NOT_SUPPORTED` with a clear suggestion rather than silently falling back to HID injection. + +--- + +## Proposed Solution + +### Fix 1 — Expand sub-window discovery (`crates/macos/src/tree.rs`) + +After exhausting `kAXWindowsAttribute`, `window_element_for` must also: +1. Query `kAXChildrenAttribute` on the **application element** and collect any child whose role is `AXWindow`, `AXPanel`, `AXSheet`, or `AXDrawer`. +2. Attempt title-exact, then title-fuzzy match on those children. +3. If the matched window has **itself** an `AXSplitGroup` or secondary `AXWindow` child visible via `kAXChildrenAttribute`, include those in the subtree — `build_subtree` already recurses `kAXChildrenAttribute` so this comes for free once the root is correct. + +No changes to `build_subtree` are required; only `window_element_for` needs expanding. + +### Fix 2 — Silence clap's stderr (`src/main.rs`) + +Replace bare `Cli::try_parse()` with an explicit command build that redirects clap's error writer to stderr (discarded via a custom `Write` impl that no-ops), OR use clap's `override_usage` to suppress the automatic error echo. + +Concrete approach: configure the `Command` to use a null stderr writer before calling `try_get_matches_from`. In clap 4.x this is done by building the command with `.error_format(clap::ErrorFormat::Plain)` combined with `stderr_fn` override so our `emit_json` path is the only output. + +Simplest viable fix: add an unconditional `eprintln!` suppression by redirecting stderr on the command, or simply verify with `2>/dev/null` that only one copy appears on stdout (confirming stderr is the source). If confirmed, wrap the `Cli::command()` call to set the error writer to `std::io::sink()` before `try_get_matches_from`. + +### Fix 3 — AX-only command execution + +**`crates/macos/src/actions.rs`** — Remove all CGEvent imports and `cg_mouse_click`. Update each action: + +- `Action::Click` → `kAXPressAction` only; if it returns error, propagate `ACTION_FAILED` (no CGEvent fallback). +- `Action::DoubleClick` → `AXUIElementPerformAction("AXOpen")` first; if unsupported, `kAXPressAction` twice with a short sleep; if still unsupported, return `ACTION_NOT_SUPPORTED`. +- `Action::RightClick` → `AXUIElementPerformAction("AXShowMenu")` only; remove CGEvent fallback. +- `Action::Toggle` → `kAXPressAction` only; remove CGEvent fallback. +- `Action::Select` → `kAXPressAction` only; remove CGEvent fallback. +- `Action::Scroll` → Use `kAXScrollDownAction` / `kAXScrollUpAction` / `kAXScrollLeftAction` / `kAXScrollRightAction` from `accessibility_sys`; apply `amount` times. Remove `CGEvent::new_scroll_event`. +- `Action::TypeText` → `kAXFocusedAttribute = true` then `kAXValueAttribute = newText`. For append semantics, read current value first and concatenate. Remove `synthesize_text`. +- `Action::PressKey` → Implement `ax_press_key` (see below); remove `synthesize_key`. + +**`crates/macos/src/input.rs`** — Remove `synthesize_key` and `synthesize_text` entirely. File may become empty/deleted; if so, remove from `lib.rs` too. + +**`crates/macos/src/app_ops.rs`** — Replace: + +- `press_for_app_impl` → Activate target app via AX (`kAXMainAttribute`), then call `ax_press_key`. +- `focus_window_impl` → `AXUIElementSetAttributeValue(windowEl, kAXMainAttribute, kCFBooleanTrue)`. +- `close_app_impl` (graceful) → Query `kAXCloseButtonAttribute` on the main window element, then `kAXPressAction` on the close button. +- `close_app_impl --force` → Keep `pkill` (explicit process termination is not an AX concern). +- `launch_app_impl` → Keep `open -a` (launching an app has no AX equivalent). + +**New: `ax_press_key` function** (`crates/macos/src/actions.rs` or new `crates/macos/src/ax_key.rs`) + +``` +ax_press_key(app_pid: Option<i32>, combo: &KeyCombo) -> Result<ActionResult, AdapterError> +``` + +Strategy: +1. Get the focused element via `kAXFocusedUIElement` on the app (or system-wide element if no app given). +2. Map simple keys to AX actions: + - `return` / `enter` → `kAXConfirmAction` + - `escape` / `esc` → `kAXCancelAction` + - `space` → `kAXPressAction` + - `up` / `down` / `left` / `right` on sliders → `kAXDecrementAction` / `kAXIncrementAction` +3. For modifier combos (cmd+c, cmd+v, etc.) with no focused-element AX action: + - Enumerate the menu bar: `kAXMenuBarAttribute` → iterate `AXMenuBarItem` → expand each → walk `AXMenuItem` children + - Match against `AXMenuItemCmdChar` and `AXMenuItemCmdModifiers` + - `kAXPressAction` on the matched `AXMenuItem` +4. If no match in steps 2–3: return `Err(AdapterError::new(ErrorCode::ActionNotSupported, "No AX equivalent for key combo '...'; this combo has no menu-bar action and no AX direct action"))` + +--- + +## Acceptance Criteria + +- [ ] `agent-desktop snapshot --app "System Settings"` returns both sidebar **and** content pane elements in a single tree +- [ ] Any invalid-argument invocation emits exactly **one** JSON error object on stdout; stderr is silent +- [ ] `agent-desktop press return` (focused text field active) succeeds without CGEvent +- [ ] `agent-desktop press cmd+z` finds and activates the Undo menu item via AX, returns success +- [ ] `agent-desktop press f5` returns `{"ok":false,"error":{"code":"ACTION_NOT_SUPPORTED",...}}` with a descriptive message +- [ ] `agent-desktop click @e1` uses only `kAXPressAction`; no CGEvent in the code path +- [ ] `agent-desktop scroll @e1 --direction down` uses `kAXScrollDownAction` +- [ ] `agent-desktop type @e1 "hello"` sets value via `kAXValueAttribute` +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo test --workspace` passes + +--- + +## Files to Change + +| File | Change | +|------|--------| +| `crates/macos/src/tree.rs` | Expand `window_element_for` to include `kAXChildrenAttribute` discovery of sub-windows | +| `src/main.rs` | Redirect clap's stderr error writer to suppress duplicate output | +| `crates/macos/src/actions.rs` | Remove all `cg_mouse_click` / `CGEvent` usage; rewrite each action with AX equivalents | +| `crates/macos/src/input.rs` | Remove `synthesize_key`, `synthesize_text`; delete file if empty | +| `crates/macos/src/app_ops.rs` | Replace `press_for_app_impl` and `focus_window_impl` with AX; update `close_app_impl` graceful path | +| `crates/macos/src/lib.rs` | Remove `input` module export if file deleted | + +New file (optional, if `ax_press_key` grows beyond ~80 LOC and would push `actions.rs` over 400 LOC): + +| File | Purpose | +|------|---------| +| `crates/macos/src/ax_key.rs` | `ax_press_key` — focused-element action dispatch + menu-bar shortcut traversal | + +--- + +## References + +- `kAXScrollDownAction` / `kAXScrollUpAction` / `kAXScrollLeftAction` / `kAXScrollRightAction` — standard AX scroll actions in `accessibility_sys` +- `kAXConfirmAction`, `kAXCancelAction` — `accessibility_sys` constants for Return / Escape on focused elements +- `kAXMenuBarAttribute`, `AXMenuItemCmdChar`, `AXMenuItemCmdModifiers` — AX menu traversal attributes +- `kAXCloseButtonAttribute` — AX attribute for window close button element +- `kAXMainAttribute` — AX attribute to make a window the main window +- `kAXFocusedUIElementAttribute` — system-wide or app-level focused element query +- Clap v4 `Command::error_format` / null error writer — suppress stderr echo of parse errors diff --git a/docs/plans/2026-02-20-feat-smart-ax-click-chain-plan.md b/docs/plans/2026-02-20-feat-smart-ax-click-chain-plan.md new file mode 100644 index 0000000..8fa9671 --- /dev/null +++ b/docs/plans/2026-02-20-feat-smart-ax-click-chain-plan.md @@ -0,0 +1,334 @@ +--- +title: "feat: Smart AX-First Click Chain — Universal Element Activation Without CGEvent" +type: feat +date: 2026-02-20 +--- + +# Smart AX-First Click Chain + +> Current contract note (2026-05-12): this historical plan is superseded where it +> describes automatic CGEvent fallback for normal ref commands. The current +> command path is headless by default. Coordinate clicks run only through +> explicit mouse commands or an explicit FFI physical policy. + +## Overview + +Replace hardcoded `AXPress -> AXConfirm -> CGEvent` fallbacks with a **dynamic, discovery-based activation chain** that queries each element's capabilities at runtime and exhausts every pure-AX strategy before touching the cursor. The goal: any interactive element in any app activates without moving the mouse. + +## Problem Statement + +The current click implementation tries 2 hardcoded AX actions, then falls back to CGEvent mouse synthesis. CGEvent: +- **Moves the visible cursor** — disruptive to the user +- **Requires the window to be frontmost** — blocks background operation +- **Fails on multi-monitor setups** where bounds are in different coordinate spaces +- **Is non-deterministic by nature** — relies on screen geometry, not semantic interaction + +Meanwhile, macOS exposes multiple pure-AX strategies (selection attributes, hierarchy walking, focus+confirm) that would work silently in the background. We just never try them. + +## Proposed Solution + +A `smart_activate()` function that dynamically discovers and attempts every available AX strategy in priority order, falling through to CGEvent only as absolute last resort. + +### The 10-Step Activation Chain + +``` +1. AXPress on element (if in action list) +2. AXConfirm on element (if in action list) +3. AXOpen on element (if in action list — Finder tree items) +4. AXPick on element (if in action list — menu items) +5. Set AXSelected=true (if attribute is writable on element) +6. Set AXSelectedRows=[el] (on parent table/outline/list) +7. Explicit-policy only: Focus + AXConfirm/AXPress +8. Walk DOWN: try child actions (first child, then grandchild) +9. Walk UP: try parent actions (parent, then grandparent) +10. Explicit-policy only: CGEvent click at center +``` + +Steps 1-6 and 8-9 are semantic AX paths. Steps that focus or use CGEvent are not part of the default CLI ref-command path; they require an explicit focus/physical policy. The chain is **non-deterministic** — it queries `AXUIElementCopyActionNames()` and `AXUIElementIsAttributeSettable()` at runtime for each element, never assuming what an app supports. + +## Technical Approach + +### Prerequisite: Folder Restructure + +Before implementing the activation chain, the macOS crate must be restructured from flat files into the standard subfolder layout (see CLAUDE.md "Platform Crate Folder Structure"). This is a prerequisite because: +- `tree.rs` (512 LOC) and `adapter.rs` (438 LOC) exceed the 400 LOC limit +- The new `activate.rs` file belongs in `actions/`, not at the root + +The restructure splits files into `tree/`, `actions/`, `input/`, and `system/` subfolders. All `mod` paths and `crate::` imports update accordingly. + +### Architecture + +#### New file: `crates/macos/src/actions/activate.rs` (~180 LOC) + +Contains the smart activation logic. Lives in the `actions/` subfolder alongside `dispatch.rs` and `extras.rs`. + +**Public API:** + +```rust +pub fn smart_activate(el: &AXElement) -> Result<(), AdapterError> +pub fn smart_double_activate(el: &AXElement) -> Result<(), AdapterError> +pub fn smart_right_activate(el: &AXElement) -> Result<(), AdapterError> +``` + +**Internal helpers:** + +```rust +fn list_ax_actions(el: &AXElement) -> Vec<String> +fn try_action_from_list(el: &AXElement, actions: &[String], targets: &[&str]) -> bool +fn try_set_selected(el: &AXElement) -> bool +fn try_select_via_parent(el: &AXElement) -> bool +fn try_focus_then_activate(el: &AXElement) -> bool +fn try_child_activation(el: &AXElement) -> bool +fn try_parent_activation(el: &AXElement) -> bool +fn is_attr_settable(el: &AXElement, attr: &str) -> bool +``` + +#### Modified file: `crates/macos/src/actions/dispatch.rs` (~350 LOC, net decrease) + +Replace inline fallback logic with calls to `activate::smart_activate()`: + +```rust +Action::Click => { + crate::actions::activate::smart_activate(el)?; +} +Action::DoubleClick => { + crate::actions::activate::smart_double_activate(el)?; +} +Action::RightClick => { + crate::actions::activate::smart_right_activate(el)?; +} +Action::Toggle => { + crate::actions::activate::smart_activate(el)?; +} +Action::TripleClick => { + crate::actions::activate::smart_triple_activate(el)?; +} +``` + +`check_uncheck()` also delegates to `smart_activate()` after its role/value checks. + +#### Modified file: `crates/macos/src/actions/mod.rs` + +Add `pub mod activate;` registration. + +### Implementation Phases + +#### Phase 0: Folder Restructure (prerequisite, no behavior change) + +Split the flat macOS crate into the standard subfolder layout: + +``` +crates/macos/src/ +├── lib.rs # mod declarations only +├── adapter.rs # PlatformAdapter impl (~175 LOC) +├── tree/ +│ ├── mod.rs +│ ├── element.rs # AXElement struct + attribute readers (~180) +│ ├── builder.rs # build_subtree, tree traversal (~250) +│ ├── roles.rs # Role mapping (75) +│ ├── resolve.rs # Element re-identification (~100) +│ └── surfaces.rs # Surface detection (252) +├── actions/ +│ ├── mod.rs +│ ├── dispatch.rs # perform_action match arms (~350) +│ ├── activate.rs # NEW — smart AX-first chain (~180) +│ └── extras.rs # select_value, ax_scroll (182) +├── input/ +│ ├── mod.rs +│ ├── keyboard.rs # Key synthesis (281) +│ ├── mouse.rs # Mouse events (147) +│ └── clipboard.rs # Clipboard get/set (55) +└── system/ + ├── mod.rs + ├── app_ops.rs # launch, close, focus (165) + ├── window_ops.rs # window operations (113) + ├── key_dispatch.rs # app-targeted key press (105) + ├── permissions.rs # permission checks (19) + ├── screenshot.rs # screen capture (79) + └── wait.rs # wait utilities (77) +``` + +**Verification:** `cargo build && cargo test --workspace && cargo clippy --all-targets -- -D warnings` — zero behavior change, only file moves and import path updates. + +#### Phase 1: Core Smart Activate (~130 LOC in `actions/activate.rs`) + +**`list_ax_actions()`** — Query element's action list dynamically: +``` +AXUIElementCopyActionNames → iterate CFArray → collect Vec<String> +``` +This is the foundation — every decision is based on what the element actually reports. + +**`is_attr_settable()`** — Check if a writable attribute exists: +``` +AXUIElementIsAttributeSettable(el, attr) → bool +``` +Required import from `accessibility_sys`: `AXUIElementIsAttributeSettable`. + +**`smart_activate()`** — The 10-step chain: + +``` +Step 1-4: Query actions once, try matching activation actions in order +Step 5: is_attr_settable("AXSelected") → set true +Step 6: copy_element_attr(el, "AXParent") → check parent role → + is_attr_settable(parent, "AXSelectedRows") → set [el] +Step 7: explicit-policy only: set AXFocused=true → sleep(50ms) → retry AXConfirm/AXPress +Step 8: copy_ax_array(el, "AXChildren") → take(3) → try actions on each +Step 9: copy_element_attr(el, "AXParent") → try actions, then grandparent +Step 10: explicit physical policy only: click_via_bounds(el, Left, 1) +``` + +**Key design decisions:** + +- **Query actions ONCE per activate call** — store in a local Vec, don't re-query per step +- **Parent role check for step 6** — only try AXSelectedRows on AXTable/AXOutline/AXList roles + (prevents setting selection attributes on random groups) +- **Child walk limit** — try first 3 children only (avoid expensive deep traversal) +- **Parent walk limit** — 2 levels max (parent + grandparent) +- **Focus settle delay** — 50ms between AXFocused set and retry (matches existing codebase pattern) + +#### Phase 2: Specialized Variants (~50 LOC in `actions/activate.rs`) + +**`smart_double_activate()`:** +``` +1. AXOpen (if in action list) +2. smart_activate() twice with 50ms gap +3. CGEvent double-click fallback +``` + +**`smart_right_activate()`:** +``` +1. AXShowMenu (if in action list) +2. CGEvent right-click fallback (no AX alternative for context menus) +``` + +**`smart_triple_activate()`:** +``` +1. smart_activate() three times with 30ms gaps +2. CGEvent triple-click fallback +``` + +Right-click has fewer AX alternatives — `AXShowMenu` is the only pure-AX option. CGEvent is a reasonable fallback here since context menus are inherently visual. + +#### Phase 3: Wire Into actions/dispatch.rs (net LOC decrease) + +Replace the inline `try_ax_action` + `click_via_bounds` chains in `perform_action()` with single-line calls to `activate.rs`. This will **reduce** `dispatch.rs` LOC since the complex fallback logic moves out. + +The `try_ax_action()`, `click_via_bounds()`, and `has_ax_action()` functions stay in `dispatch.rs` as they're used by other code paths. But they also get `pub(crate)` visibility for `activate.rs` to call `click_via_bounds()` as the final fallback. + +#### Phase 4: Expand/Collapse AXDisclosing Fallback (~15 LOC in actions/dispatch.rs) + +For `Action::Expand` and `Action::Collapse`, add `AXDisclosing` attribute as fallback: + +``` +Expand: AXExpand action → set AXDisclosing=true → error +Collapse: AXCollapse action → set AXDisclosing=false → error +``` + +This uses `is_attr_settable()` from `actions/activate.rs`. Small addition, stays in `actions/dispatch.rs`. + +### Selection via Parent: Building the CFArray + +For step 6 (set AXSelectedRows on parent), we need to build a CFArray containing the target element. The pattern from tree.rs: + +```rust +// Retain the element, build a single-element array +unsafe { CFRetain(el.0 as CFTypeRef) }; +let el_as_cftype = unsafe { CFType::wrap_under_create_rule(el.0 as CFTypeRef) }; +let arr = CFArray::from_CFTypes(&[el_as_cftype]); +AXUIElementSetAttributeValue(parent.0, attr, arr.as_CFTypeRef()) +``` + +**Important:** The element must be CFRetained before wrapping, since `from_CFTypes` does its own retain, and `wrap_under_create_rule` takes ownership. + +## File Changes Summary + +### Phase 0: Folder Restructure (move only, no behavior change) + +| Old Path | New Path | +|----------|----------| +| `crates/macos/src/tree.rs` | Split → `tree/element.rs`, `tree/builder.rs` | +| `crates/macos/src/actions.rs` | → `actions/dispatch.rs` | +| `crates/macos/src/action_extras.rs` | → `actions/extras.rs` | +| `crates/macos/src/keyboard.rs` | → `input/keyboard.rs` | +| `crates/macos/src/mouse.rs` | → `input/mouse.rs` | +| `crates/macos/src/clipboard.rs` | → `input/clipboard.rs` | +| `crates/macos/src/roles.rs` | → `tree/roles.rs` | +| `crates/macos/src/surfaces.rs` | → `tree/surfaces.rs` | +| `crates/macos/src/app_ops.rs` | → `system/app_ops.rs` | +| `crates/macos/src/window_ops.rs` | → `system/window_ops.rs` | +| `crates/macos/src/key_dispatch.rs` | → `system/key_dispatch.rs` | +| `crates/macos/src/permissions.rs` | → `system/permissions.rs` | +| `crates/macos/src/screenshot.rs` | → `system/screenshot.rs` | +| `crates/macos/src/wait.rs` | → `system/wait.rs` | + +Plus extract `tree/resolve.rs` from `adapter.rs` (element re-identification logic). + +### Phases 1-4: Smart Activation Chain + +| File | Action | LOC Change | +|------|--------|------------| +| `crates/macos/src/actions/activate.rs` | **Create** | +180 | +| `crates/macos/src/actions/dispatch.rs` | Modify (simplify) | -30 (~350 LOC) | +| `crates/macos/src/actions/mod.rs` | Add `pub mod activate` | +1 | + +**Net:** +151 LOC. No file exceeds 400 LOC. + +## Acceptance Criteria + +### Functional Requirements + +- [ ] `click @ref` on System Settings treeitems works without moving cursor (AXSelected or AXSelectedRows) +- [ ] `click @ref` on standard buttons still works via AXPress (no regression) +- [ ] `click @ref` on Finder sidebar items works (AXOpen or AXSelected) +- [ ] `click @ref` on custom SwiftUI controls works (hierarchy walking or focus+confirm) +- [ ] `double-click @ref` works via AXOpen or repeated activation +- [ ] `right-click @ref` works via AXShowMenu where possible +- [ ] `toggle`, `check`, `uncheck` use smart activation +- [ ] `expand`/`collapse` falls back to AXDisclosing attribute + +### Non-Functional Requirements + +- [ ] No app-specific code — all behavior is discovered at runtime via `AXUIElementCopyActionNames` and `AXUIElementIsAttributeSettable` +- [ ] CGEvent is only reached when ALL pure-AX strategies fail +- [ ] No file exceeds 400 LOC +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo test --workspace` passes + +### Quality Gates + +- [ ] Smoke test on 5+ apps: System Settings, Finder, TextEdit, Safari, Xcode +- [ ] Verify cursor does NOT move during activation on at least 3 test cases + +## Verification Plan + +```bash +cargo build +cargo clippy --all-targets -- -D warnings +cargo fmt --all +cargo test --workspace + +# Smoke tests +cargo run -- launch "System Settings" +cargo run -- snapshot --app "System Settings" -i +cargo run -- click @eN # Appearance treeitem — should NOT move cursor +cargo run -- click @eN # Light button — should activate + +cargo run -- launch "Finder" +cargo run -- snapshot --app "Finder" -i +cargo run -- click @eN # Sidebar item + +cargo run -- launch "TextEdit" +cargo run -- snapshot --app "TextEdit" -i +cargo run -- click @eN # Toolbar button + +wc -l crates/macos/src/actions/activate.rs # must be <= 400 +wc -l crates/macos/src/actions/dispatch.rs # must be <= 400 +``` + +## References + +- Brainstorm research: AX action catalog (11 documented + app-specific custom actions) +- `AXUIElementIsAttributeSettable` available in `accessibility-sys 0.2.0` +- Existing parent-walking pattern: `actions/extras.rs` (`find_scroll_area`) +- Existing action list query: `actions/dispatch.rs` (`has_ax_action`) +- `copy_element_attr` supports "AXParent": `tree/element.rs` diff --git a/docs/plans/2026-02-21-fix-fallback-chains-edge-cases-plan.md b/docs/plans/2026-02-21-fix-fallback-chains-edge-cases-plan.md new file mode 100644 index 0000000..8dde844 --- /dev/null +++ b/docs/plans/2026-02-21-fix-fallback-chains-edge-cases-plan.md @@ -0,0 +1,324 @@ +--- +title: "fix: Add fallback chains and focus guards to all commands" +type: fix +date: 2026-02-21 +--- + +# Add Fallback Chains and Focus Guards to All Commands + +> Current contract note (2026-05-12): this historical plan is superseded where it +> treats focus or CGEvent paths as automatic fallbacks for normal ref commands. +> The current command path is headless by default; physical/headed behavior is +> explicit. + +## Overview + +Audit every command in agent-desktop to ensure: (1) every action has an AX-first multi-step fallback chain, (2) every CGEvent path has a focus guard, and (3) edge cases where AX fails due to missing focus/state are handled. Commands like `type`, `press`, `set-value`, `set-focus`, `expand`, `collapse`, `scroll-to`, and `select` currently have single-strategy or no fallback — they either succeed on the first try or return an error. + +## Problem Statement + +Three categories of issues: + +### 1. Commands with NO fallback chain (single AX strategy, error on failure) + +| Command | Current Implementation | Problem | +|---------|----------------------|---------| +| `set-value` | `AXUIElementSetAttributeValue(AXValue)` | Must verify app-observable state and fail clearly if unchanged | +| `set-focus` | `AXUIElementSetAttributeValue(AXFocused, true)` | Fails on some elements, no fallback to click-to-focus | +| `scroll-to` | `AXPerformAction(AXScrollToVisible)` | Fails on elements not in scroll area, no other strategies | +| `clear` | `AXUIElementSetAttributeValue(AXValue, "")` | Same as set-value — fails on read-only | +| `expand` | AXExpand → AXDisclosing=true (2 steps) | Missing: click fallback, AXPress on disclosure triangle | +| `collapse` | AXCollapse → AXDisclosing=false (2 steps) | Missing: click fallback, AXPress on disclosure triangle | +| `select` (popup) | AXPress → search children → AXPress item | Missing: retry with app focus, fallback to AXShowMenu | +| `type` | Set AXFocused → `synthesize_text()` via system-wide AX | No focus guard — text goes to whichever app is frontmost | +| `press` | `synthesize_key()` via system-wide AX | No focus guard — keystroke goes to whatever is focused | + +### 2. Edge cases where app focus is needed but not ensured + +| Command | Scenario | Impact | +|---------|----------|--------| +| `type @e5 "hello"` | Terminal is focused, TextEdit behind | Text types into Terminal instead of TextEdit | +| `press cmd+s` | Target app not frontmost | Saves wrong app's document | +| `select` (popup) | Popup opens in background app | Menu items can't be found | + +### 3. Commands that are correct (no changes needed) + +| Command | Why it's fine | +|---------|--------------| +| `click` | Semantic activation chain; CGEvent requires explicit physical policy in current code | +| `double-click` | Uses smart_activate (14-step) | +| `right-click` | 7-step chain with explicit ensure_app_focused | +| `triple-click` | Uses smart_activate (14-step) | +| `scroll` | Semantic scroll chain; wheel input requires explicit physical policy in current code | +| `toggle` / `check` / `uncheck` | Delegates to smart_activate (14-step) | +| `mouse-*` / `drag` / `hover` | Intentionally raw CGEvent (agent controls coordinates) | +| All read-only commands | No action to fail | +| `press --app` | Already has focus guard in key_dispatch.rs | + +## Proposed Solution + +### Phase 1: Focus Guards for TypeText and PressKey + +**File: `crates/macos/src/actions/dispatch.rs`** + +#### TypeText — ensure target element's app is focused before typing + +Current code (line 125-135): +```rust +Action::TypeText(text) => { + let cf_attr = CFString::new(kAXFocusedAttribute); + unsafe { AXUIElementSetAttributeValue(...) }; + crate::input::keyboard::synthesize_text(text)?; +} +``` + +Change to: +```rust +Action::TypeText(text) => { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + let cf_attr = CFString::new(kAXFocusedAttribute); + unsafe { AXUIElementSetAttributeValue(...) }; + std::thread::sleep(std::time::Duration::from_millis(30)); + crate::input::keyboard::synthesize_text(text)?; +} +``` + +#### PressKey — ensure target element's app is focused before keystroke + +Current code (line 137-139): +```rust +Action::PressKey(combo) => { + crate::input::keyboard::synthesize_key(combo)?; +} +``` + +Change to: Use the SAME strategy as `press --app` in `key_dispatch.rs` — try menu bar shortcut first, then AX keyboard event to app (not system-wide), then fall back to system-wide: +```rust +Action::PressKey(combo) => { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + let app_el = crate::tree::element_for_pid(pid); + // Try AX action on focused element first (return/escape/space) + // Then try menu bar shortcut + // Then AXUIElementPostKeyboardEvent to app + // Last: system-wide synthesize_key + } + crate::input::keyboard::synthesize_key(combo)?; +} +``` + +### Phase 2: SetValue / Clear Fallback Chain + +**File: `crates/macos/src/actions/dispatch.rs`** + +Replace `ax_set_value(el, val)` with a multi-step chain: + +``` +1. AXUIElementSetAttributeValue(AXValue, val) — direct set +2. If element is a textfield/textarea: focus → select-all → type replacement text +3. If element has AXInsertText action (parameterized): use it +4. Focus + clear via keyboard (Cmd+A, then type new value) +``` + +```rust +fn smart_set_value(el: &AXElement, val: &str) -> Result<(), AdapterError> { + // Step 1: Direct AX value set + if ax_set_value(el, val).is_ok() { + return Ok(()); + } + + // Step 2: Focus + select all + type + let role = element_role(el); + if matches!(role.as_deref(), Some("textfield" | "textarea" | "combobox")) { + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + let cf_attr = CFString::new(kAXFocusedAttribute); + let focus_err = unsafe { + AXUIElementSetAttributeValue( + el.0, cf_attr.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if focus_err == kAXErrorSuccess { + std::thread::sleep(Duration::from_millis(30)); + // Select all existing text + crate::input::keyboard::synthesize_key(&KeyCombo { + key: "a".to_string(), + modifiers: vec![Modifier::Cmd], + })?; + std::thread::sleep(Duration::from_millis(30)); + // Type replacement + crate::input::keyboard::synthesize_text(val)?; + return Ok(()); + } + } + + Err(AdapterError::new( + ErrorCode::ActionFailed, + "SetValue failed: element value not settable", + ).with_suggestion("Try 'click' to focus, then 'type' to enter text")) +} +``` + +For **Clear**: Same chain but with empty string (select-all + delete): +``` +1. AXUIElementSetAttributeValue(AXValue, "") +2. Focus + Cmd+A + Delete +``` + +### Phase 3: SetFocus Fallback Chain + +**File: `crates/macos/src/actions/dispatch.rs`** + +Current: single `AXUIElementSetAttributeValue(AXFocused, true)` call. + +Add fallback: +``` +1. Set AXFocused=true +2. If fail: click the element (smart_activate) — clicking often focuses +3. If fail: Set AXSelected=true (some list elements gain focus via selection) +``` + +### Phase 4: Expand/Collapse Enhanced Chain + +**File: `crates/macos/src/actions/dispatch.rs`** + +Current chain: AXExpand → AXDisclosing=true (2 steps). + +Enhanced chain: +``` +1. AXExpand / AXCollapse action +2. Set AXDisclosing = true/false +3. AXPress on element itself (toggles disclosure state in many apps) +4. AXPress on disclosure triangle child (look for child with subrole AXDisclosureTriangle) +5. smart_activate (click fallback) +``` + +### Phase 5: Select with Focus Guard + +**File: `crates/macos/src/actions/extras.rs`** + +For popup/menubutton: ensure app focused before opening popup. + +```rust +Some("popupbutton") | Some("menubutton") => { + // Ensure app is focused so popup menu is visible + if let Some(pid) = crate::system::app_ops::pid_from_element(el) { + let _ = crate::system::app_ops::ensure_app_focused(pid); + } + ax_press_or_fail(el, "select (open popup)")?; + std::thread::sleep(Duration::from_millis(200)); + // ... rest of menu search +} +``` + +### Phase 6: ScrollTo Fallback + +**File: `crates/macos/src/actions/dispatch.rs`** + +Current: single `AXScrollToVisible` action. + +Enhanced: +``` +1. AXScrollToVisible on element +2. If fail: scroll parent to bring element into view using ax_scroll() +3. If fail: error with suggestion +``` + +## Implementation Checklist + +### Phase 1: Focus Guards +- [ ] Add `ensure_app_focused(pid)` before `synthesize_text()` in TypeText handler +- [ ] Add 30ms sleep after focus before AXFocused set +- [ ] For PressKey: focus app, try AX action on focused element, try menu bar, then AX keyboard to app, then system-wide fallback +- [ ] Test: open TextEdit behind Terminal, `type @e1 "test"` should type into TextEdit + +### Phase 2: SetValue/Clear Chain +- [ ] Extract `smart_set_value(el, val)` function +- [ ] Step 1: Direct AXValue set +- [ ] Step 2: Focus + Cmd+A + type replacement (for text fields) +- [ ] Update Clear to use same chain with empty/delete +- [ ] Test: `set-value @textfield "new text"` on a read-only label should fail gracefully + +### Phase 3: SetFocus Chain +- [ ] Add click fallback after AXFocused=true fails +- [ ] Add AXSelected fallback for list items +- [ ] Test: `focus @e5` on an element that doesn't support AXFocused + +### Phase 4: Expand/Collapse Chain +- [ ] Add AXPress on element after AXDisclosing fails +- [ ] Add disclosure triangle child search +- [ ] Add smart_activate as last AX resort +- [ ] Test: expand/collapse on Finder sidebar disclosure triangles + +### Phase 5: Select Focus Guard +- [ ] Add ensure_app_focused before popup open in select_value +- [ ] Test: select dropdown value in background app + +### Phase 6: ScrollTo Fallback +- [ ] Add parent scroll fallback when AXScrollToVisible fails +- [ ] Test: scroll-to on element outside visible area + +## File Changes + +| File | Change | LOC Impact | +|------|--------|-----------| +| `crates/macos/src/actions/dispatch.rs` | TypeText/PressKey focus, SetValue/Clear/Focus/Expand/Collapse/ScrollTo chains | +80 (~447 → need to extract) | +| `crates/macos/src/actions/extras.rs` | Select focus guard | +5 | + +**LOC concern:** dispatch.rs is currently 367 LOC. Adding ~80 LOC pushes to ~447, over the 400 limit. Solution: extract `smart_set_value()` and the enhanced expand/collapse into a new file `crates/macos/src/actions/value_ops.rs` (~80 LOC). + +| File | Final LOC | +|------|----------| +| `dispatch.rs` | ~380 (after extracting value_ops) | +| `value_ops.rs` | ~80 (new: smart_set_value, smart_clear, smart_expand, smart_collapse) | +| `extras.rs` | ~400 (unchanged + 5 for focus guard) | + +## Verification + +```bash +cargo build && cargo clippy --all-targets -- -D warnings + +# Phase 1: TypeText focus guard +cargo run -- launch "TextEdit" +cargo run -- launch "Finder" # Finder now focused +cargo run -- snapshot --app "TextEdit" -i +cargo run -- type @e1 "hello world" # Should focus TextEdit first, then type + +# Phase 1: PressKey focus guard +cargo run -- snapshot --app "TextEdit" -i +cargo run -- press cmd+a # Should select all in TextEdit, not Finder + +# Phase 2: SetValue fallback +cargo run -- snapshot --app "TextEdit" -i +cargo run -- set-value @e1 "replaced text" + +# Phase 4: Expand/Collapse +cargo run -- snapshot --app "Finder" -i +cargo run -- expand @eN # disclosure triangle +cargo run -- collapse @eN + +# Phase 5: Select with focus +cargo run -- launch "System Settings" +cargo run -- snapshot --app "System Settings" -i +cargo run -- select @eN "Dark" # appearance dropdown + +# LOC checks +wc -l crates/macos/src/actions/dispatch.rs # <= 400 +wc -l crates/macos/src/actions/value_ops.rs # <= 400 +wc -l crates/macos/src/actions/extras.rs # <= 400 +``` + +## Execution Order + +1. Phase 1: Focus guards for TypeText and PressKey (highest impact, most common edge case) +2. Phase 2: SetValue/Clear fallback chain +3. Phase 3: SetFocus fallback chain +4. Phase 4: Expand/Collapse enhanced chain +5. Phase 5: Select focus guard +6. Phase 6: ScrollTo fallback +7. Build, lint, test everything +8. Update README if chain descriptions changed diff --git a/docs/plans/2026-02-23-feat-release-automation-npm-distribution-plan.md b/docs/plans/2026-02-23-feat-release-automation-npm-distribution-plan.md new file mode 100644 index 0000000..b789682 --- /dev/null +++ b/docs/plans/2026-02-23-feat-release-automation-npm-distribution-plan.md @@ -0,0 +1,303 @@ +--- +title: "feat: Automated releases with GitHub Releases and npm distribution" +type: feat +status: active +date: 2026-02-23 +origin: docs/brainstorms/2026-02-23-release-automation-brainstorm.md +--- + +# feat: Automated releases with GitHub Releases and npm distribution + +## Overview + +Set up a fully automated release pipeline for agent-desktop: Conventional Commits determine SemVer version bumps, release-please creates gated Release PRs with auto-generated CHANGELOGs, merging the Release PR triggers macOS binary builds, GitHub Release creation with tarballs, and npm publication of the `agent-desktop` package with a postinstall binary downloader. + +## Problem Statement + +agent-desktop is currently install-from-source only (`cargo build --release`). There is no versioning strategy, no CHANGELOG, no GitHub Releases, and no package manager distribution. Users must have the Rust toolchain installed to use the tool. This limits adoption, especially for AI agent developers who may not have Rust set up. + +## Proposed Solution + +Follow the agent-browser pattern (see brainstorm: `docs/brainstorms/2026-02-23-release-automation-brainstorm.md`): + +1. **Conventional Commits** → SemVer version bumps +2. **release-please** → Release PRs with CHANGELOG +3. **GitHub Actions** → build macOS binaries on Release PR merge +4. **GitHub Releases** → attach platform tarballs + SHA-256 checksums +5. **npm** → single `agent-desktop` package with postinstall binary downloader + +## Technical Approach + +### Architecture + +``` +Push to main (feat:/fix: commits) + │ + ▼ +release-please creates/updates Release PR + (bumps Cargo.toml version, generates CHANGELOG) + │ + ▼ +Developer merges Release PR + │ + ▼ +release-please creates GitHub Release (tag: v0.2.0) + │ + ▼ +Build job (matrix: aarch64-apple-darwin, x86_64-apple-darwin) + ├── cargo build --release --target <target> + ├── tar + gzip binary + └── generate SHA-256 checksum + │ + ▼ +Upload tarballs + checksums.txt to GitHub Release + │ + ▼ +Verify all assets present → npm publish --provenance + │ + ▼ +User: npm install -g agent-desktop + ├── postinstall: detect platform → download binary from GH Release + └── bin/agent-desktop.js: spawn native binary +``` + +### Implementation Phases + +#### Phase 1: release-please + CHANGELOG + +**Deliverables:** +- `release-please-config.json` — configuration targeting root `Cargo.toml` +- `.release-please-manifest.json` — version tracker (initial: `{"." : "0.1.0"}`) +- `.github/workflows/release.yml` — release-please job +- Cargo.lock auto-update step in Release PR + +**Files to create:** +- `release-please-config.json` +- `.release-please-manifest.json` +- `.github/workflows/release.yml` + +**Success criteria:** +- [ ] Pushing a `feat:` commit to `main` creates a Release PR +- [ ] Release PR bumps `workspace.package.version` in root `Cargo.toml` +- [ ] Release PR includes auto-generated CHANGELOG.md +- [ ] Cargo.lock is updated in the Release PR +- [ ] Merging the Release PR creates a GitHub Release with tag `v{version}` + +**Key decisions (see brainstorm):** +- `feat:` → minor, `fix:` → patch, `feat!:` → major +- `style:`, `docs:`, `refactor:`, `chore:`, `ci:` → excluded from CHANGELOG, no release + +#### Phase 2: Binary builds + GitHub Release assets + +**Deliverables:** +- Build matrix in `release.yml` for macOS targets +- Tarball creation + SHA-256 checksums +- Asset upload to GitHub Release +- Verification step before npm publish + +**Files to modify:** +- `.github/workflows/release.yml` (add build + upload jobs) + +**Build matrix:** + +| Target | Runner | Notes | +|--------|--------|-------| +| `aarch64-apple-darwin` | `macos-latest` (ARM) | Native build | +| `x86_64-apple-darwin` | `macos-13` (Intel) | Native build on Intel runner | + +**Tarball naming:** `agent-desktop-v{version}-{target}.tar.gz` +- e.g., `agent-desktop-v0.2.0-aarch64-apple-darwin.tar.gz` +- Binary inside tarball is just `agent-desktop` (no platform suffix) + +**Checksum file:** `checksums.txt` attached to the release, containing SHA-256 hashes for all tarballs. + +**Success criteria:** +- [ ] Release creates tarballs for both macOS architectures +- [ ] `checksums.txt` is attached to the GitHub Release +- [ ] Binary size is under 15MB (enforced) +- [ ] Both binaries execute correctly on their respective platforms + +**CI runner note:** Use `macos-13` for x86_64 (Intel runner) and `macos-latest` for aarch64 (ARM runner) to avoid cross-compilation issues. + +#### Phase 3: npm package + postinstall + +**Deliverables:** +- `npm/package.json` — package metadata, bin entry, postinstall script +- `npm/bin/agent-desktop.js` — JS wrapper that spawns native binary +- `npm/scripts/postinstall.js` — downloads platform binary from GitHub Release +- npm publish step in `release.yml` + +**Files to create:** +- `npm/package.json` +- `npm/bin/agent-desktop.js` +- `npm/scripts/postinstall.js` + +**npm package structure:** +``` +npm/ +├── package.json # name: "agent-desktop", bin, postinstall +├── bin/ +│ └── agent-desktop.js # JS wrapper: platform detect → spawn binary +└── scripts/ + └── postinstall.js # Download binary from GitHub Release +``` + +**package.json key fields:** +```json +{ + "name": "agent-desktop", + "bin": { "agent-desktop": "./bin/agent-desktop.js" }, + "scripts": { "postinstall": "node scripts/postinstall.js" }, + "files": ["bin", "scripts"], + "os": ["darwin"], + "engines": { "node": ">=18" } +} +``` + +**postinstall.js behavior:** +1. Detect platform via `process.platform` + `process.arch` +2. Map to Rust target triple (`darwin`+`arm64` → `aarch64-apple-darwin`) +3. If unsupported platform → print friendly message, exit 0 (not error) +4. Download tarball from `https://github.com/lahfir/agent-desktop/releases/download/v{version}/agent-desktop-v{version}-{target}.tar.gz` +5. Download `checksums.txt`, verify SHA-256 +6. Extract binary to `bin/agent-desktop-{platform-arch}` +7. `chmod 755` the binary +8. On global installs: attempt to patch npm's symlink for zero-overhead execution + +**postinstall robustness:** +- Respect `HTTPS_PROXY` / `HTTP_PROXY` env vars +- Support `AGENT_DESKTOP_BINARY_PATH` override for pre-placed binaries +- Support `AGENT_DESKTOP_SKIP_DOWNLOAD=1` for offline environments +- 3 retries with exponential backoff (2s, 4s, 8s) +- 60-second timeout per download attempt +- Atomic file writes (download to `.tmp`, then `rename()`) +- Detect Bun (`process.versions.bun`) and suggest `--trust` if binary missing +- Print progress to stderr (not stdout, to avoid interfering with JSON output) + +**bin/agent-desktop.js behavior:** +1. Detect platform, resolve binary path +2. If binary missing → clear error with recovery instructions +3. Ensure binary is executable (`chmod 755` if needed) +4. `spawn(binaryPath, process.argv.slice(2), { stdio: 'inherit' })` +5. Forward exit code + +**Success criteria:** +- [ ] `npm install -g agent-desktop` works on macOS (ARM + Intel) +- [ ] `npx agent-desktop version` returns correct version +- [ ] `npm install -g agent-desktop` on Linux/Windows prints "macOS only" message and exits cleanly +- [ ] Binary integrity verified via SHA-256 checksum +- [ ] Postinstall handles network failures gracefully with retry +- [ ] Global install symlink optimization works (best-effort) + +## Alternative Approaches Considered + +1. **Per-platform npm packages** (`@agent-desktop/darwin-arm64`, etc.) — like esbuild/turbo. More complex: multiple packages to publish, npm org required, optionalDependencies wiring. Rejected in favor of the simpler single-package approach (see brainstorm). + +2. **semantic-release** (fully automatic) — every merge to main auto-releases. No gate. Rejected because release-please's Release PR provides a natural review point (see brainstorm). + +3. **cargo-release** — Rust-native tooling. More manual, requires running commands locally. Rejected for being less automated (see brainstorm). + +## System-Wide Impact + +### Interaction Graph + +- Push to `main` → `release.yml` triggers → release-please creates/updates Release PR +- Release PR merge → release-please creates GitHub Release → build job → publish-github job → publish-npm job +- `ci.yml` (existing) is NOT modified — continues running on all pushes/PRs independently +- npm postinstall → HTTPS request to GitHub Releases CDN → binary placed in package + +### Error Propagation + +- Build failure → no tarballs uploaded → asset verification fails → npm publish blocked (safe) +- Postinstall download failure → JS wrapper detects missing binary → clear error message (graceful degradation) +- Version sync failure (Cargo.toml vs package.json) → CI script handles sync before `npm publish` (single source of truth is Cargo.toml) + +### State Lifecycle Risks + +- **Partial release:** GitHub Release created but binaries not yet uploaded. Mitigated by strict `needs:` job dependencies — npm publish waits for all assets. +- **npm publish without assets:** Mitigated by explicit asset verification step before publish. +- **Stale npm cache:** `npx` may cache old versions. Documented: use `npx agent-desktop@latest`. + +### API Surface Parity + +- No API changes. The binary's CLI interface, JSON output contract, and exit codes are unchanged. +- The `version` command output will reflect the new version number automatically (uses `env!("CARGO_PKG_VERSION")`). + +## Acceptance Criteria + +### Functional Requirements + +- [ ] Conventional commits (`feat:`, `fix:`, `feat!:`) correctly determine version bumps +- [ ] release-please creates a Release PR with CHANGELOG on releasable commits +- [ ] Merging the Release PR creates a GitHub Release with tag `v{version}` +- [ ] macOS binaries (ARM + Intel) are built and attached as tarballs +- [ ] SHA-256 checksums are attached to the release +- [ ] `npm install -g agent-desktop` downloads and installs the correct binary +- [ ] `npx agent-desktop version` works +- [ ] `agent-desktop` (after global install) runs with zero Node.js overhead +- [ ] Non-macOS platforms get a friendly "not supported yet" message on npm install + +### Non-Functional Requirements + +- [ ] Binary size under 15MB (existing CI check) +- [ ] Postinstall download completes within 60 seconds on reasonable connections +- [ ] Postinstall respects proxy environment variables +- [ ] npm publish includes Sigstore provenance attestation (`--provenance`) +- [ ] GitHub Actions permissions are minimal per job + +### Quality Gates + +- [ ] Existing CI (`ci.yml`) continues to pass — no modifications to it +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo test --lib --workspace` passes +- [ ] Manual end-to-end test: merge a Release PR → verify npm install works + +## Dependencies & Prerequisites + +- **npm account** with publish access to the `agent-desktop` package name (must be available) +- **NPM_TOKEN** GitHub secret configured for the repository +- **Conventional commit discipline** — all future commits to `main` should follow the convention +- No new Rust dependencies required +- Node.js 18+ for the postinstall/wrapper scripts + +## Risk Analysis & Mitigation + +| Risk | Severity | Mitigation | +|------|----------|------------| +| npm package name `agent-desktop` taken | Critical | Check availability first. Fallback: `@agent-desktop/cli` | +| Binary download blocked by firewall | Important | Proxy support, `AGENT_DESKTOP_BINARY_PATH` override, manual download docs | +| release-please workspace version handling | Important | Test with dry run, use `extra-files` config if needed | +| Race: Release exists before binaries uploaded | Important | Strict `needs:` job deps + asset verification before npm publish | +| Broken version published to npm | Important | `npm deprecate` + quick patch release procedure | +| macOS Gatekeeper warnings on unsigned binaries | Important | Document `xattr -cr` workaround; defer code signing to later | +| Cargo.lock out of sync in Release PR | Important | Auto-update step: `cargo update --workspace` in Release PR | +| CI token cannot trigger workflows on Release PR | Important | Use PAT or GitHub App token for release-please | + +## Future Considerations + +- **Phase 2 platforms:** Add Linux and Windows targets to the build matrix. Extend postinstall platform mapping. Consider musl vs glibc for Linux. Add Windows `.exe` handling. +- **macOS code signing + notarization:** Apple Developer ID for signed binaries. Eliminates Gatekeeper warnings. +- **Homebrew tap:** Leverage GitHub Release tarballs to create a Homebrew formula. +- **Prerelease channel:** `0.3.0-rc.1` releases via release-please prerelease config. +- **crates.io publishing:** Publish `agent-desktop-core` to crates.io for downstream Rust consumers. + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-02-23-release-automation-brainstorm.md](docs/brainstorms/2026-02-23-release-automation-brainstorm.md) — Key decisions carried forward: SemVer + Conventional Commits, release-please for gated releases, single npm package with postinstall download (agent-browser pattern), GitHub Release tarballs for non-npm distribution. + +### Internal References + +- Existing CI: `.github/workflows/ci.yml` +- Version source: `Cargo.toml:10` (`workspace.package.version`) +- Version command: `crates/core/src/commands/version.rs` +- Rust targets: `rust-toolchain.toml` +- Release build profile: `Cargo.toml:22-28` + +### External References + +- [release-please documentation](https://github.com/googleapis/release-please) +- [release-please Cargo/Rust support](https://github.com/googleapis/release-please/blob/main/docs/customizing.md) +- [agent-browser npm distribution pattern](https://github.com/vercel-labs/agent-browser) — reference implementation for postinstall binary download +- [npm provenance attestations](https://docs.npmjs.com/generating-provenance-statements) diff --git a/docs/plans/2026-02-23-refactor-centralized-ax-chain-executor-plan.md b/docs/plans/2026-02-23-refactor-centralized-ax-chain-executor-plan.md new file mode 100644 index 0000000..7dc2b51 --- /dev/null +++ b/docs/plans/2026-02-23-refactor-centralized-ax-chain-executor-plan.md @@ -0,0 +1,812 @@ +--- +title: "refactor: centralize macOS action dispatch via chain executor" +type: refactor +status: active +date: 2026-02-23 +origin: docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md +--- + +# refactor: centralize macOS action dispatch via chain executor + +> Current contract note (2026-05-12): this historical plan is superseded where it +> implies automatic focus, cursor, keyboard, pasteboard, or CGEvent fallback from +> normal ref commands. The current command path is headless by default. Physical +> or headed behavior is available only through explicit mouse/focus/keyboard +> commands or an explicit FFI policy. + +## Overview + +Replace per-command fallback functions (`smart_activate`, `smart_right_activate`, etc.) with a centralized chain executor where commands declare AX fallback steps as data. Also add error suggestions to all commands, AX messaging timeout, transient error retry, non-ASCII text support, NSPasteboard FFI clipboard, and post-action state hints. + +This is a pure macOS adapter refactoring. Core crate and PlatformAdapter trait are unchanged. + +## Problem Statement + +The current macOS action dispatch has three problems: + +1. **Duplicated patterns**: `activate.rs` (400 LOC), `dispatch.rs` (367 LOC), and `extras.rs` (396 LOC) share ~100-150 LOC of duplicated code: attribute setting, action enumeration, child/parent traversal, sleep timing. All three files are at the 400 LOC ceiling — any additions require splitting. + +2. **Inconsistent robustness**: `click` has a 14-step chain, `scroll` has 8 steps, but `set-value`, `type`, `clear`, `focus`, `scroll-to`, and `press` are single-shot with no fallback. This breaks headless/background operation when the single AX call fails. + +3. **Missing error guidance**: Several commands (`set-value`, `focus`, `type`, `press`, `clear`) return raw AX error codes with no suggestions for the calling agent. + +(see brainstorm: `docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md`) + +## Proposed Solution + +### Architecture: 3-Layer Chain System + +**Layer 1 — Shared Utilities (`ax_helpers.rs`)** +Extract all duplicated AX patterns into reusable functions: `try_ax_action()`, `set_ax_bool()`, `set_ax_string()`, `try_each_child()`, `try_each_ancestor()`, `with_retry()`, `ensure_visible()`, `set_messaging_timeout()`. + +**Layer 2 — Element Discovery (`discovery.rs`)** +One-time `ElementCaps` struct queried once per action: available actions, settable attributes, role, PID. Eliminates redundant AX IPC calls across chain steps. + +**Layer 3 — Chain Executor + Definitions (`chain.rs` + `chain_defs.rs`)** +`ChainStep` enum defines what to try. `execute_chain()` walks steps with pre-action setup (scroll-to-visible, messaging timeout) and post-action enrichment (state hints). Each command is a static chain definition — data, not code. + +### Concrete Data Model + +```rust +pub enum ChainStep { + /// Try AXUIElementPerformAction with the given action name. + Action(&'static str), + + /// Try AXUIElementSetAttributeValue with given attribute and value. + SetAttr { + attr: &'static str, + value: AttrValue, + }, + + /// Focus the element first (AXFocused=true), then retry the inner step. + WithFocus(Box<ChainStep>), + + /// Try the inner step on up to `limit` child elements. + OnChildren { + step: Box<ChainStep>, + limit: usize, + }, + + /// Try the inner step on up to `limit` ancestor elements. + OnAncestors { + step: Box<ChainStep>, + limit: usize, + }, + + /// Custom logic that cannot be expressed declaratively. + /// Receives element + caps, returns true if the step succeeded. + Custom { + label: &'static str, + func: fn(&AXElement, &ElementCaps) -> bool, + }, + + /// CGEvent coordinate-based fallback. Automatically focuses the window. + CGEvent(CGFallback), +} + +pub enum AttrValue { + Bool(bool), + StaticStr(&'static str), + Dynamic, // Placeholder — actual value passed via ChainContext at runtime +} + +pub enum CGFallback { + Click(MouseButton, u32), + None, +} + +pub struct ChainDef { + pub steps: &'static [ChainStep], + pub suggestion: &'static str, +} +``` + +**Why `Custom` variant**: Steps like `try_show_alternate_ui` (AXShowAlternateUI → sleep → child AXPress), `try_select_via_parent` (walk parent, check role, set AXSelectedRows), and `try_keyboard_activate` (focus + space key) involve multi-step conditional logic that cannot be expressed as flat declarative steps without over-engineering. `Custom { label, func }` is the escape hatch — the label enables tracing, the func contains the logic. + +**Why `AttrValue::Dynamic`**: `set-value` and `type` chains need runtime text values. The executor checks for `Dynamic` and substitutes from a `ChainContext` struct passed alongside. + +### File Layout + +``` +crates/macos/src/actions/ +├── mod.rs # mod declarations + pub use perform_action +├── ax_helpers.rs # NEW: shared AX utilities (~200 LOC) +├── discovery.rs # NEW: ElementCaps one-time query (~120 LOC) +├── chain.rs # NEW: ChainStep enum + execute_chain() (~250 LOC) +├── chain_defs.rs # NEW: all chain definitions (~300 LOC) +├── dispatch.rs # SIMPLIFIED: perform_action delegates to chains (~200 LOC) +├── activate.rs # DELETED: absorbed into chain_defs.rs + ax_helpers.rs +└── extras.rs # REFACTORED: select_value + ax_scroll use ax_helpers (~300 LOC) +``` + +LOC budget per file stays under 400. + +## Technical Considerations + +### Retry Policy + +- **Transient errors**: Only `kAXErrorCannotComplete` (-25204). Apple docs classify this as "target app temporarily unable to process request." +- **Terminal errors**: `kAXErrorInvalidUIElement`, `kAXErrorNotImplemented`, `kAXErrorAPIDisabled`, `kAXErrorNoValue`. +- **Max retries**: 1 retry per step (not per chain). +- **Delay**: 100ms fixed between retry attempts. +- **Idempotency guard**: For stateful actions (`Toggle`, `Check`, `Uncheck`), read element value before retry. If value already changed from pre-action state, skip retry (action succeeded despite error code). +- **Retry scope**: Inside the chain executor's step loop. A step that fails after retry simply falls through to the next step. + +### Side Effect Safety + +``` +Before chain: read element value/state (if action is stateful) + Step N: try AX action → returns error + Retry: read element value/state again + if changed → action succeeded, do NOT retry + if unchanged → retry the step +``` + +This handles the "retried AXPress toggles checkbox back" problem. + +### Window Focus Policy + +(from brainstorm: Window Focus Policy section) + +| Step Type | Focuses Window | Reason | +|-----------|---------------|--------| +| `Action(name)` | No | AX IPC targets element directly | +| `SetAttr { .. }` | No | AX IPC targets element directly | +| `WithFocus(step)` | No | Sets AXFocused on element, not window | +| `OnChildren/OnAncestors` | No | AX IPC | +| `Custom { .. }` | Depends on impl | Custom decides | +| `CGEvent(..)` | **Yes** | CGEvent is coordinate-based | + +The executor only calls `ensure_app_focused()` inside the `CGEvent` step handler. + +### Non-ASCII Text Input + +Strategy: per-segment splitting. + +1. Scan input text, split into runs of ASCII and non-ASCII characters. +2. ASCII runs: keyboard synthesis via `AXUIElementPostKeyboardEvent`. +3. Non-ASCII runs: clipboard paste strategy: + a. Save current clipboard text (best-effort — non-text clipboard data types are lost on restore). + b. Set clipboard to the non-ASCII segment. + c. Synthesize Cmd+V via app-targeted keyboard event. + d. Restore clipboard (log warning via `tracing::warn!` if restore fails, but report action as success). + +**Prerequisite**: NSPasteboard FFI must be implemented first to avoid ~200ms subprocess overhead per clipboard operation. + +### NSPasteboard FFI + +Replace `pbcopy`/`pbpaste` subprocess calls with direct Cocoa FFI. Use raw `objc_msgSend` from `core-foundation` to avoid adding new dependencies: + +```rust +// Pseudocode — actual impl uses core-foundation-sys + manual objc runtime calls +fn clipboard_get() -> Result<String, AdapterError> { + let pasteboard = msg_send![class!(NSPasteboard), generalPasteboard]; + let string = msg_send![pasteboard, stringForType: NSPasteboardTypeString]; + // Convert NSString to Rust String +} +``` + +No new crate dependency needed — `core-foundation-sys` already provides the FFI primitives. The `objc_msgSend` calls are ~10 LOC per function. + +### Stale Ref Auto-Recovery + +Lives inside the macOS adapter's `resolve_element` implementation (not in core). PlatformAdapter trait stays unchanged. + +``` +resolve_element(entry) called + → normal resolution: match (pid, role, name, bounds_hash) + → if ELEMENT_NOT_FOUND: + → relaxed resolution: match (pid, role, name) only + → if found: check bounds proximity (within 50px any edge) + → if bounds acceptable: return handle + → if multiple matches: return first in DFS order (same as snapshot) + → if still not found: return STALE_REF with existing suggestion +``` + +**Not applied during batch**: The `--stop-on-error` flag already provides batch failure handling. Auto-recovery in batch could interact with wrong elements. Only single-command execution triggers auto-recovery. + +### Post-Action State Hints + +After a successful chain execution, the executor reads element state for these action types: + +| Action | What to read | Delay | +|--------|-------------|-------| +| Click, Toggle, Check, Uncheck | value + states | 50ms | +| SetValue, Clear | value | 0ms (AX set is synchronous) | +| TypeText | value | 50ms | +| Expand, Collapse | states (expanded/collapsed) | 0ms | +| All others | nothing | - | + +If the state read fails (element disappeared), return `ActionResult` with `post_state: None`. The action itself is still reported as success. + +### AX Messaging Timeout + +Current: `element_for_pid()` in `element.rs` already sets 2.0s timeout on app-level elements. + +Change: In `execute_chain()`, before walking steps, call `AXUIElementSetMessagingTimeout(el, 3.0)` on the target element. This is separate from the app-level timeout — it applies to the specific element being acted upon. + +### Global Chain Timeout + +10 seconds max for entire chain execution. Tracked via `Instant::now()` at chain start. Before each step, check elapsed time. If exceeded, return `AdapterError::timeout("Chain execution exceeded 10s")`. + +### Error Suggestions (Complete List) + +| Command | On Failure | Suggestion | +|---------|-----------|------------| +| set-value | Chain exhausted | "Try 'clear' then 'type', or check element is a text field." | +| type | Chain exhausted | "Try 'set-value' for direct text insertion." | +| clear | Chain exhausted | "Try 'press cmd+a' then 'press delete'." | +| focus | Chain exhausted | "Try 'click' to focus the element instead." | +| scroll-to | Chain exhausted | "Element may not be in a scrollable container." (existing) | +| press | Key synth failed | "Ensure the target app is active, or use 'press' with --app flag." | +| click | Chain exhausted | "Element may not be interactable. Try 'mouse-click --xy X,Y'." | +| right-click | Chain exhausted | "Try 'mouse-click --button right --xy X,Y'." | +| expand | Chain exhausted | "Try 'click' to open it instead." (existing) | +| collapse | Chain exhausted | "Try 'click' to close it instead." (existing) | +| toggle | Wrong role | "Toggle works on checkboxes, switches, and radio buttons. Use 'click' for other elements." (existing) | +| hover | Element-level call | "Use 'hover @ref' which resolves ref to coordinates automatically." | +| drag | Element-level call | "Use 'drag --from @ref --to @ref' for ref-based drag." | +| key-down/key-up | Element-level call | "Use 'key-down'/'key-up' commands directly." | +| set-value (AX err) | Specific AX error | Include `platform_detail` with AX error code for debugging. | + +## System-Wide Impact + +### Interaction Graph + +``` +CLI command → core command handler → adapter.execute_action() + → dispatch.rs perform_action() → chain executor + → discovery.rs (one AX IPC batch) + → chain.rs execute_chain() (walks ChainStep list) + → ax_helpers.rs (try_ax_action, set_ax_bool, etc.) + → explicit physical policy path (only when the caller selected it) + → post-state read (optional) + → JSON ActionResult response +``` + +No callbacks, middleware, or observers. Pure function call chain. No new event system. + +### Error Propagation + +``` +AX API error code → with_retry() classifies as transient/terminal + → transient: retry once, then fall through to next chain step + → terminal: fall through immediately + → all steps exhausted: AdapterError with suggestion + → wrapped in AppError → JSON error envelope +``` + +### State Lifecycle Risks + +- **Clipboard corruption during non-ASCII**: Save/restore is best-effort. Non-text clipboard data (images, files) cannot be preserved across the save/restore cycle. Documented limitation. +- **Element disappears mid-chain**: Each step independently handles AX errors. If element disappears, remaining steps fail fast (kAXErrorInvalidUIElement is terminal). Final error includes "Run 'snapshot' to refresh" suggestion. +- **Stale ref auto-recovery false positive**: Mitigated by 50px bounds proximity check. In worst case, agent acts on wrong element — same risk as manually re-snapshotting and re-resolving. + +### API Surface Parity + +- `PlatformAdapter` trait: **No changes**. All refactoring is internal to macOS crate. +- `ActionResult` struct: Already has `post_state: Option<ElementState>`. No schema change. +- JSON output format: `post_state` field may now be populated (was always None). Agents that don't read it are unaffected. +- CLI interface: **No changes**. Same commands, same flags, same output format. + +## Acceptance Criteria + +### Functional Requirements + +- [ ] All 50 existing commands produce identical JSON output for the happy path +- [ ] `click` chain matches current `smart_activate` 14-step behavior +- [ ] `right-click` chain matches current `smart_right_activate` 7-step behavior +- [ ] `set-value`, `type`, `clear`, `focus`, `scroll-to` have multi-step fallback chains +- [ ] Non-ASCII text input works via clipboard paste (e.g., `type @e1 "Hola"`) +- [ ] Clipboard operations use NSPasteboard FFI (no subprocess) +- [ ] All error responses include `suggestion` field +- [ ] `post_state` populated for stateful actions +- [ ] Transient AX errors retried once with 100ms delay +- [ ] Chain execution times out at 10 seconds +- [ ] AX messaging timeout set to 3s on target elements + +### Non-Functional Requirements + +- [ ] No new crate dependencies added +- [ ] All files under 400 LOC +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo test --lib --workspace` passes +- [ ] `cargo tree -p agent-desktop-core` shows zero platform crate names +- [ ] Release binary stays under 15MB + +### Quality Gates + +- [ ] `activate.rs` fully absorbed — file deleted +- [ ] No duplicated AX patterns across files (attribute setting, action enumeration, tree traversal) +- [ ] Every `ChainStep::Custom` has a descriptive label for tracing +- [ ] Every error path has a suggestion + +## Implementation Approach: Two Stages + +### Why Two Stages + +The existing click chain (14-step `smart_activate`) and scroll chain (8-step `ax_scroll`) were built through **manual trial and error** — testing AX APIs directly against real macOS apps (TextEdit, Finder, System Settings, Safari, Xcode) to discover what works, then coding the proven sequences. The same approach is needed for the new chains. + +We cannot guess what AX actions or attributes work for `set-value`, `type`, `clear`, `focus`, or `scroll-to` across different app frameworks (AppKit, SwiftUI, Electron, Chromium). We must test first. + +**Stage A (Code Directly):** Build the chain executor infrastructure + migrate existing proven chains. This is pure refactoring of known-working code. + +**Stage B (Explore Then Code):** For each command that needs a NEW chain, manually probe real apps using the built binary, record what works, then write the chain definition based on observations. + +### Stage B Exploration Protocol + +For each command needing a new chain (set-value, type, clear, focus, scroll-to): + +1. **Build and run the binary** with the chain executor from Stage A +2. **Open target apps**: TextEdit, System Settings, Safari, Finder, Notes, Calculator +3. **Snapshot and identify elements** of the relevant type: + ```bash + agent-desktop snapshot --app "TextEdit" -i + # Find text fields, buttons, checkboxes, sliders, etc. + ``` +4. **Test AX actions directly** against each element using the existing commands: + ```bash + # For set-value: does direct AXValue set work on this element? + agent-desktop set-value @e3 "test" + # For clear: does AXValue="" work? Does Cmd+A then Delete work? + agent-desktop clear @e3 + # For focus: does AXFocused=true work? Does AXPress set focus? + agent-desktop focus @e3 + ``` +5. **Record results** in a discovery log per command: + - Which elements/roles support the primary AX approach? + - Which need focus first? + - Which need an alternative approach entirely? + - Which apps behave differently? +6. **Write the chain definition** based on the actual observed behavior +7. **Test the chain** against all target apps to verify it works end-to-end + +This ensures chains are built on empirical evidence, not assumptions. + +## Implementation Phases + +### Stage A: Infrastructure + Known Chain Migration (Code Directly) + +These phases involve pure refactoring of existing, proven code. No AX exploration needed. + +--- + +### Phase 1: Foundation (`ax_helpers.rs` + `discovery.rs`) + +Extract shared utilities from duplicated code across activate.rs, dispatch.rs, extras.rs. + +**Files created:** +- `crates/macos/src/actions/ax_helpers.rs` (~200 LOC) +- `crates/macos/src/actions/discovery.rs` (~120 LOC) + +**ax_helpers.rs contents:** +```rust +pub fn try_ax_action(el: &AXElement, name: &str) -> bool +pub fn set_ax_bool(el: &AXElement, attr: &str, value: bool) -> bool +pub fn set_ax_string(el: &AXElement, attr: &str, value: &str) -> bool +pub fn try_each_child(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool +pub fn try_each_ancestor(el: &AXElement, f: impl Fn(&AXElement) -> bool, limit: usize) -> bool +pub fn list_ax_actions(el: &AXElement) -> Vec<String> +pub fn ensure_visible(el: &AXElement) +pub fn set_messaging_timeout(el: &AXElement, seconds: f32) +pub fn with_retry<F>(f: F, max_retries: u32, delay_ms: u64) -> Result<bool, i32> +``` + +**discovery.rs contents:** +```rust +pub struct ElementCaps { + pub actions: Vec<String>, + pub settable_value: bool, + pub settable_focus: bool, + pub settable_selected: bool, + pub settable_disclosing: bool, + pub role: Option<String>, + pub has_children: bool, + pub pid: Option<i32>, +} + +pub fn discover(el: &AXElement) -> ElementCaps +``` + +**Tasks:** +- [ ] Create `ax_helpers.rs` with all shared functions +- [ ] Create `discovery.rs` with `ElementCaps` and `discover()` +- [ ] Register both modules in `actions/mod.rs` +- [ ] Update `extras.rs` to use `ax_helpers` functions (remove duplicates) +- [ ] Verify `cargo test --lib --workspace && cargo clippy` + +### Phase 2: Chain Executor (`chain.rs`) + +Build the executor that walks a `ChainDef` against an element. + +**Files created:** +- `crates/macos/src/actions/chain.rs` (~250 LOC) + +**Contents:** +- `ChainStep` enum (Action, SetAttr, WithFocus, OnChildren, OnAncestors, Custom, CGEvent) +- `AttrValue` enum (Bool, StaticStr, Dynamic) +- `CGFallback` enum (Click, None) +- `ChainDef` struct (steps, suggestion) +- `ChainContext` struct (dynamic_value: Option<String>, pre_state: Option<String>) +- `execute_chain(el: &AXElement, def: &ChainDef, ctx: &ChainContext) -> Result<(), AdapterError>` + +**Executor logic:** +1. `ensure_visible(el)` — best-effort AXScrollToVisible +2. `set_messaging_timeout(el, 3.0)` — prevent hangs +3. `discover(el)` — one-time capability query +4. Walk steps: for each step, try with retry wrapper +5. For stateful actions: read pre-state before first step, check before retry +6. Track elapsed time — abort at 10 seconds +7. All steps failed → `AdapterError::new(ActionFailed, "...").with_suggestion(def.suggestion)` +8. `tracing::debug!` for each step attempt + +**Tasks:** +- [ ] Create `chain.rs` with all types and executor +- [ ] Add unit test: executor with mock steps (Custom steps that return true/false) +- [ ] Add unit test: timeout behavior +- [ ] Add unit test: retry on transient error +- [ ] Register in `actions/mod.rs` + +### Phase 3: Migrate Known Chains (`chain_defs.rs`) + +Migrate ONLY the existing, proven chains into the new system. New chains come later in Stage B. + +**Files created:** +- `crates/macos/src/actions/chain_defs.rs` (~200 LOC initially, grows in Stage B) + +**Files modified:** +- `crates/macos/src/actions/dispatch.rs` — simplified to delegate to chains +- `crates/macos/src/actions/activate.rs` — **deleted** + +**Migrated chains (proven, existing logic):** + +| Chain | Steps | Origin | +|-------|-------|--------| +| `click_chain` | 14 steps (migrated from `smart_activate`) | activate.rs | +| `double_click_chain` | AXOpen → 2x click → CGEvent double | activate.rs | +| `right_click_chain` | 7 steps (migrated from `smart_right_activate`) | activate.rs | +| `triple_click_chain` | 3x click → CGEvent triple | activate.rs | +| `expand_chain` | Action(AXExpand) → SetAttr(AXDisclosing=true) | dispatch.rs inline | +| `collapse_chain` | Action(AXCollapse) → SetAttr(AXDisclosing=false) | dispatch.rs inline | + +**NOT migrated yet** (need AX exploration in Stage B): set-value, focus, clear, scroll-to, type. +These stay as their current single-shot implementation until Stage B discovers proper chains. + +**Custom step functions** (migrated from activate.rs): +- `try_show_alternate_ui(el, caps)` — from activate.rs +- `try_keyboard_activate(el, caps)` — from activate.rs +- `try_select_via_parent(el, caps)` — from activate.rs +- `try_select_then_show_menu(el, caps)` — from activate.rs + +**dispatch.rs after Phase 3 (~250 LOC):** +```rust +pub fn perform_action(el: &AXElement, action: &Action) -> Result<ActionResult, AdapterError> { + let label = action_label(action); + match action { + // Migrated to chain executor + Action::Click => execute_chain(el, &CLICK_CHAIN, &ChainContext::default())?, + Action::DoubleClick => execute_chain(el, &DOUBLE_CLICK_CHAIN, &ChainContext::default())?, + Action::RightClick => execute_chain(el, &RIGHT_CLICK_CHAIN, &ChainContext::default())?, + Action::TripleClick => execute_chain(el, &TRIPLE_CLICK_CHAIN, &ChainContext::default())?, + Action::Expand => execute_chain(el, &EXPAND_CHAIN, &ChainContext::default())?, + Action::Collapse => execute_chain(el, &COLLAPSE_CHAIN, &ChainContext::default())?, + + // Kept as-is until Stage B exploration + Action::SetValue(val) => ax_set_value(el, val)?, + Action::TypeText(text) => { /* current impl */ }, + Action::Clear => ax_set_value(el, "")?, + Action::SetFocus => { /* current impl */ }, + Action::ScrollTo => { /* current impl */ }, + + // Stays custom (not chain-based) + Action::Toggle => { /* role check + click_chain */ }, + Action::Check => { /* state check + click_chain */ }, + Action::Uncheck => { /* state check + click_chain */ }, + Action::Select(value) => extras::select_value(el, value)?, + Action::Scroll(d, a) => extras::ax_scroll(el, d, *a)?, + Action::PressKey(combo) => crate::input::keyboard::synthesize_key(combo)?, + + // Adapter-level (coordinate-based) + Action::Hover | Action::Drag(_) | Action::KeyDown(_) | Action::KeyUp(_) => { + return Err(adapter_level_error(action, &label)); + } + _ => return Err(AdapterError::not_supported(&label)), + } + Ok(ActionResult::new(label)) +} +``` + +**Tasks:** +- [ ] Create `chain_defs.rs` with migrated chain definitions and custom step functions +- [ ] Migrate `smart_activate` → `click_chain` (14 steps) +- [ ] Migrate `smart_right_activate` → `right_click_chain` (7 steps) +- [ ] Migrate expand/collapse inline logic → `expand_chain`/`collapse_chain` +- [ ] Simplify `dispatch.rs` to delegate migrated chains, keep others as-is +- [ ] Delete `activate.rs` +- [ ] Verify all existing behavior preserved: `cargo test --lib --workspace` +- [ ] **Manual verification**: test click, right-click, double-click on TextEdit, Finder, System Settings +- [ ] **Manual verification**: test expand/collapse on Finder sidebar, System Settings disclosure + +### Phase 4: NSPasteboard FFI + +Replace `pbcopy`/`pbpaste` subprocess calls with direct Cocoa FFI. Done early because it's a prerequisite for the non-ASCII type chain in Stage B. + +**Files modified:** +- `crates/macos/src/input/clipboard.rs` — rewrite implementation (~80 LOC) + +**Implementation approach:** +Use `objc_msgSend` via the `core-foundation-sys` crate (already a dependency). No new crate needed. + +```rust +extern "C" { + fn objc_msgSend(receiver: *const Object, sel: Sel, ...) -> *const Object; + fn objc_getClass(name: *const c_char) -> *const Object; + fn sel_registerName(name: *const c_char) -> Sel; +} + +pub fn get() -> Result<String, AdapterError> { + let pasteboard_class = objc_getClass("NSPasteboard"); + let pasteboard = objc_msgSend(pasteboard_class, sel_registerName("generalPasteboard")); + let nsstring = objc_msgSend(pasteboard, sel_registerName("stringForType:"), ns_pasteboard_type_string()); + // Convert NSString → Rust String via CFString bridge +} + +pub fn set(text: &str) -> Result<(), AdapterError> { + let pasteboard = /* generalPasteboard */; + objc_msgSend(pasteboard, sel_registerName("clearContents")); + let ns_string = CFString::new(text); + objc_msgSend(pasteboard, sel_registerName("setString:forType:"), ns_string, ns_pasteboard_type_string()); +} +``` + +**Tasks:** +- [ ] Rewrite `clipboard.rs` using objc_msgSend FFI +- [ ] Keep same public API: `get()`, `set()`, `clear()` +- [ ] Test clipboard roundtrip: set → get → verify +- [ ] Test with non-ASCII content +- [ ] Test with empty string +- [ ] Benchmark: verify < 1ms per operation (vs ~50ms with subprocess) + +### Phase 5: Error Suggestions + +Add suggestions to every command that's missing them. Can be done independently of Stage B. + +**Files modified:** +- `crates/macos/src/actions/dispatch.rs` — add suggestions to match arms for hover/drag/key-down/key-up and current single-shot commands +- `crates/macos/src/actions/chain_defs.rs` — each chain's `suggestion` field (already there for migrated chains) +- `crates/macos/src/input/keyboard.rs` — suggestion on unknown key error + +All suggestions listed in the "Error Suggestions (Complete List)" table above. + +**Tasks:** +- [ ] Add suggestions to hover/drag/key-down/key-up element-level errors in dispatch.rs +- [ ] Add suggestions to set-value, focus, clear, type, press, scroll-to errors in dispatch.rs +- [ ] Verify all chain definitions have suggestion fields +- [ ] Add suggestion to keyboard.rs unknown key error: "Valid keys: a-z, 0-9, return, escape, tab, space, delete, arrow keys, f1-f12." +- [ ] Add `platform_detail` with AX error codes to all AX-originated errors +- [ ] Audit: grep for `AdapterError::new` without `.with_suggestion` — all should have one + +### Phase 6: Resilience (Timeout + Retry + Stale Recovery) + +These are integrated into the chain executor and ax_helpers built in Phases 1-2. + +**AX Messaging Timeout:** +- In `chain.rs` `execute_chain()`: call `set_messaging_timeout(el, 3.0)` before step loop +- Already in `ax_helpers.rs` from Phase 1 + +**Retry Logic:** +- Already in `ax_helpers.rs` `with_retry()` from Phase 1 +- In `chain.rs`: wrap each step execution with retry + +**Stale Ref Auto-Recovery:** +- In `crates/macos/src/tree/resolve.rs`: add relaxed resolution fallback +- Normal resolve fails → try again without bounds_hash → check bounds proximity (50px) +- Only for single-command execution (add flag to distinguish from batch) + +**Global Chain Timeout:** +- In `chain.rs`: `Instant::now()` at start, check before each step + +**Tasks:** +- [ ] Verify `set_messaging_timeout()` is called in execute_chain (should be from Phase 2) +- [ ] Verify `with_retry()` is integrated into step execution (should be from Phase 2) +- [ ] Add global 10s timeout to chain executor if not already present +- [ ] Add relaxed resolution to resolve.rs +- [ ] Add 50px bounds proximity check +- [ ] Skip auto-recovery when called from batch dispatch +- [ ] Test: verify retry does not double-toggle a checkbox + +**Stage A complete.** At this point the binary is functional with the chain executor, migrated click/right-click/expand/collapse chains, NSPasteboard FFI, error suggestions, and resilience. The remaining single-shot commands still work as before. + +--- + +### Stage B: AX Exploration + New Chain Definitions (Explore Then Code) + +These phases require **manual AX testing against real apps** to discover proper fallback sequences. Each phase follows the exploration protocol defined above. + +--- + +### Phase 7: Explore + Define set-value Chain + +**Apps to test:** TextEdit (text field), System Settings (text inputs, sliders), Safari (URL bar), Notes (text area), Calculator (display) + +**Questions to answer through exploration:** +- Does `AXUIElementSetAttributeValue(kAXValueAttribute, value)` work on all text field roles? +- Which elements need `AXFocused=true` set before `AXSetValue` succeeds? +- Do sliders support `AXSetValue`? What format (string number)? +- Do combo boxes support `AXSetValue` directly? +- What happens when `AXSetValue` is called on a read-only element? + +**Expected chain shape** (hypothesis — to be validated): +``` +SetAttr(AXValue, value) → WithFocus(SetAttr(AXValue, value)) → error +``` + +**Tasks:** +- [ ] Run exploration protocol on 5+ apps for set-value +- [ ] Record discovery log: which roles/apps work with which approach +- [ ] Write `set_value_chain` definition in chain_defs.rs based on findings +- [ ] Wire into dispatch.rs +- [ ] Test chain against all explored apps + +### Phase 8: Explore + Define clear Chain + +**Apps to test:** TextEdit, System Settings, Safari URL bar, Notes, any search fields + +**Questions to answer:** +- Does `AXSetValue("")` clear all text field types? +- For elements where `AXSetValue` doesn't work, does focus + Cmd+A + Delete work? +- Are there elements where neither approach works? +- Does app-targeted keyboard (Cmd+A to app PID) work for select-all, or must it be system-wide? + +**Expected chain shape** (hypothesis): +``` +SetAttr(AXValue, "") → WithFocus(SetAttr(AXValue, "")) → Custom(select_all_then_delete) → error +``` + +**Tasks:** +- [ ] Run exploration protocol on 5+ apps for clear +- [ ] Record discovery log +- [ ] Write `clear_chain` definition based on findings +- [ ] Implement `select_all_then_delete` custom step if needed +- [ ] Wire into dispatch.rs and test + +### Phase 9: Explore + Define focus Chain + +**Apps to test:** TextEdit, Finder (sidebar items, file list), Safari (tabs, URL bar), System Settings + +**Questions to answer:** +- Does `AXUIElementSetAttributeValue(kAXFocusedAttribute, true)` work universally? +- Which elements don't support `AXFocused` but can be focused via `AXPress`? +- Does `AXSelected=true` function as a focus mechanism for list/table items? +- Are there elements that cannot be focused at all? + +**Expected chain shape** (hypothesis): +``` +SetAttr(AXFocused, true) → Action(AXPress) → SetAttr(AXSelected, true) → error +``` + +**Tasks:** +- [ ] Run exploration protocol on 5+ apps for focus +- [ ] Record discovery log +- [ ] Write `focus_chain` definition based on findings +- [ ] Wire into dispatch.rs and test + +### Phase 10: Explore + Define scroll-to Chain + +**Apps to test:** Finder (long file lists), Safari (long pages), System Settings (scrollable panels), Xcode (code editor) + +**Questions to answer:** +- Does `AXScrollToVisible` work reliably across apps? +- When it fails, can we walk parents to find a scroll area and verify visibility without sending physical wheel input? +- How do we verify the element is actually visible after scrolling? +- Does `AXScrollToVisible` work on elements inside nested scroll views? + +**Expected chain shape** (hypothesis): +``` +Action(AXScrollToVisible) → Custom(visible_in_scroll_context) → error +``` + +**Tasks:** +- [ ] Run exploration protocol on 5+ apps for scroll-to +- [ ] Record discovery log +- [ ] Write `scroll_to_chain` definition based on findings +- [ ] Implement a visibility-check custom step with no cursor or wheel input +- [ ] Wire into dispatch.rs and test + +### Phase 11: Explore + Define type Chain + +This is the most complex chain — it mixes AX attribute setting with keyboard synthesis and non-ASCII clipboard paste. Requires NSPasteboard FFI from Phase 4. + +**Apps to test:** TextEdit, Safari URL bar, Notes, System Settings search, Spotlight, Terminal + +**Questions to answer:** +- Does `AXSetValue(current + new_text)` (append) work on text fields? Or does it replace? +- Does `AXUIElementPostKeyboardEvent` to the app PID (not system-wide) deliver keys to the focused element within that app? +- For non-ASCII: does clipboard paste (Cmd+V) via app-targeted keyboard reliably insert text? +- What is the timing needed between focus + keyboard delivery? +- Do some apps intercept Cmd+V differently (e.g., Terminal)? + +**Expected chain shape** (hypothesis): +``` +Custom(ax_append_value) → Custom(focus_then_app_keyboard) → Custom(focus_then_system_keyboard_with_non_ascii) → error +``` + +**Tasks:** +- [ ] Run exploration protocol on 5+ apps for type +- [ ] Test ASCII typing via app-targeted keyboard events +- [ ] Test non-ASCII typing via clipboard paste strategy +- [ ] Test mixed ASCII + non-ASCII text +- [ ] Record discovery log +- [ ] Implement `execute_type()` in dispatch.rs based on findings +- [ ] Implement `try_ax_append_value()`, `type_via_app_keyboard()`, `type_with_non_ascii_support()` +- [ ] Test against all explored apps + +### Phase 12: Post-Action State Hints + +Can be done after Stage B chains are finalized — it's an enrichment layer on top of the chain executor. + +**Files modified:** +- `crates/macos/src/actions/dispatch.rs` — add `read_post_state()` after chain execution + +**Implementation:** +```rust +fn read_post_state(el: &AXElement, action: &Action) -> Option<ElementState> { + let delay = match action { + Action::Click | Action::Toggle | Action::Check + | Action::Uncheck | Action::TypeText(_) => 50, + Action::SetValue(_) | Action::Clear + | Action::Expand | Action::Collapse => 0, + _ => return None, + }; + if delay > 0 { + std::thread::sleep(std::time::Duration::from_millis(delay)); + } + let value = copy_string_attr(el, "AXValue"); + let states = read_element_states(el); + Some(ElementState { role: None, value, states }) +} +``` + +**Tasks:** +- [ ] Implement `read_post_state()` in dispatch.rs +- [ ] Wire into `perform_action()` return path +- [ ] Test: click a checkbox → verify post_state shows checked +- [ ] Test: set-value on text field → verify post_state shows new value +- [ ] Test: action on disappearing element → verify post_state is None (not an error) + +## Dependencies & Prerequisites + +- No new crate dependencies +- Requires Rust stable (already pinned via `rust-toolchain.toml`) +- NSPasteboard FFI uses `core-foundation-sys` (already in deps) +- `objc_msgSend` FFI declarations need `extern "C"` blocks — no crate needed + +## Risk Analysis & Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|------|-----------|--------|------------| +| Chain executor doesn't capture all activate.rs nuances | Medium | High | Migrate click_chain first, test manually against TextEdit/Finder/System Settings before proceeding | +| Explicit physical path regression | Low | High | Existing integration tests catch this; add new test for physical-policy path | +| Non-ASCII clipboard restore fails silently | Medium | Low | Log via tracing::warn; agent can clipboard-get to verify | +| Stale ref auto-recovery picks wrong element | Low | Medium | 50px bounds check; only for single commands | +| NSPasteboard FFI crashes on edge cases | Low | High | Test with empty clipboard, non-text content, locked pasteboard | +| Global 10s timeout too aggressive for slow apps | Medium | Medium | Make timeout configurable via env var AGENT_DESKTOP_CHAIN_TIMEOUT_MS | + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md](docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md) — Key decisions: AX-first architecture, centralized chain executor, window focus only for CGEvent, per-segment non-ASCII handling. + +### Internal References + +- Current action dispatch: `crates/macos/src/actions/dispatch.rs` +- Current activation chains: `crates/macos/src/actions/activate.rs` +- Current scroll chain: `crates/macos/src/actions/extras.rs` +- Error types: `crates/core/src/error.rs` +- ActionResult/ElementState: `crates/core/src/action.rs` +- Current clipboard: `crates/macos/src/input/clipboard.rs` +- Current keyboard: `crates/macos/src/input/keyboard.rs` +- Element resolution: `crates/macos/src/tree/resolve.rs` +- Prior plan (smart click chain): `docs/plans/2026-02-20-feat-smart-ax-click-chain-plan.md` +- Prior plan (fallback chains): `docs/plans/2026-02-21-fix-fallback-chains-edge-cases-plan.md` diff --git a/docs/plans/2026-02-25-feat-windows-adapter-phase2-plan.md b/docs/plans/2026-02-25-feat-windows-adapter-phase2-plan.md new file mode 100644 index 0000000..1149941 --- /dev/null +++ b/docs/plans/2026-02-25-feat-windows-adapter-phase2-plan.md @@ -0,0 +1,1308 @@ +--- +title: "feat: implement Windows adapter via UI Automation" +type: feat +status: active +date: 2026-02-25 +deepened: 2026-02-25 +origin: docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md +--- + +# feat: Implement Windows Adapter via UI Automation + +## Enhancement Summary + +**Deepened on:** 2026-02-25 (two rounds) +**Sections enhanced:** 14 +**Research agents used:** Architecture Strategist, Performance Oracle, Security Sentinel, Pattern Recognition Specialist, Code Simplicity Reviewer, Agent-Native Reviewer, Spec Flow Analyzer, Best Practices Researcher, Framework Docs Researcher, Repo Research Analyst, CacheRequest Deep Dive (uiautomation-rs source), App Lifecycle Security (MSDN + Raymond Chen), COM/SendInput/DPI Specialist, Phase 0 Exact Analysis + +### Key Improvements +1. **CacheRequest optimization** — Concrete `UIProperty` enum, `get_cached_*` methods, `UITreeWalker::*_build_cache` pattern for 5-10x tree traversal speedup. Corrected to `ElementMode::Full` (not `None` — `None` cannot receive actions) +2. **COM pointer lifecycle contract** — Explicit `AddRef`/`ManuallyDrop` pattern for `NativeHandle` on Windows, preventing double-free and use-after-free +3. **Security readiness** — `CreateProcessW` primary (safe for untrusted input), `ShellExecuteW` fallback with metacharacter rejection (not regex), clipboard RAII guard with 10x/100ms retry +4. **Chromium detection update** — Chrome 138+ has native UIA by default (since July 2025); version detection via `GetFileVersionInfoW`; sparse trees returned immediately with warning (no hidden latency) +5. **Complete role mapping** — Added 8 missing UIA control types (RadioButton, Spinner, ProgressBar, ScrollBar, StatusBar, Thumb, SplitButton, HeaderItem) +6. **SendInput batching** — 10-50x performance improvement for `type` command; double/triple click as 4/6 INPUT events in single call +7. **DPI awareness** — Corrected multi-monitor formula: `(pixel - virt_origin) * 65535 / (virt_size - 1)` with `MOUSEEVENTF_VIRTUALDESK` +8. **HRESULT→ErrorCode mapping table** — Complete mapping moved to Phase 0 `error_mapping.rs` +9. **PID type fix** — `i32` → `u64` promoted to Phase 0 (two-line change now vs painful migration later) +10. **Defense-in-depth timeout** — 4-layer strategy: `IsHungAppWindow` → `IUIAutomation2` timeouts → thread timeout → graceful degradation +11. **Two-tier acceptance criteria** — "core" (must-pass for merge) vs "complete" (must-pass for sign-off) + +### New Considerations Discovered +- `WindowInfo.pid` was `i32` but Windows PIDs are `u32` (DWORD) — fixed in Phase 0 (§0.7) by changing to `u64` everywhere +- `SnapshotSurface::Sheet` and `SnapshotSurface::Popover` have no Windows equivalent — must return `ElementNotFound` +- `AppInfo.bundle_id` has no Windows analog — use `None` (field is already `Option<String>`) +- `key_down`/`key_up` commands skip blocked combo check on macOS too — safety bug to fix in Phase 0 +- `wait --gone` referenced in plan but does not exist as a command variant — remove reference +- `pinch` command does not exist in the codebase — remove from acceptance criteria +- `PrintWindow` with `PW_RENDERFULLCONTENT` flag needed for modern DWM-composited apps (plain `BitBlt` returns black) +- Cloaked windows (virtual desktops) returned by `EnumWindows` but invisible — filter via `DwmGetWindowAttribute(DWMWA_CLOAKED)` +- ApplicationFrameHost.exe hosts UWP windows with PID mismatch from actual app — needs special handling in `list_windows` +- Hung/frozen app UIA calls block indefinitely — need timeout wrapper via thread + channel + +--- + +## Overview + +Implement the Windows `PlatformAdapter` for agent-desktop, enabling all 50 CLI commands to work on Windows via the UI Automation COM API. Uses `uiautomation` (v0.24+) for tree traversal and pattern-based actions, complemented by the `windows` crate for Win32 APIs (input synthesis, clipboard, screenshot, process lifecycle). Zero changes to core, macOS, or Linux crates. + +## Problem Statement + +agent-desktop currently only runs on macOS. The Phase 1 architecture was explicitly designed for additive platform expansion (see brainstorm: `docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md`). The Windows adapter is the first test of this architecture — it must prove the trait-based isolation model works without touching any existing code. + +## Proposed Solution + +Implement `WindowsAdapter` in `crates/windows/` following the exact same delegation pattern as `MacOSAdapter`. The adapter is a stateless unit struct that creates `UIAutomation` instances per-call (COM threading safety). Platform-specific dependencies are target-gated. A Windows CI job validates real UIA integration. + +### Research Insights: Architecture Validation + +**Trait Friction Points:** +- `SnapshotSurface::Sheet` and `SnapshotSurface::Popover` are macOS concepts with no Windows equivalent. The Windows adapter must return `Err(AdapterError::element_not_found("No sheet/popover on Windows"))` for these variants. +- `AppInfo.bundle_id` is macOS-centric. Windows adapter returns `None` (the field is already `Option<String>`). No Windows equivalent needed for Phase 2. +- `WindowInfo.pid` is `i32` but Windows PIDs are `u32` (DWORD). Fixed in Phase 0 (see §0.7) — two-line change now avoids a painful migration later. + +**Per-Call vs Cached UIAutomation:** +- Per-call `UIAutomation::new()` costs ~0.5-2ms COM init overhead. Keep per-call as default. +- **Concrete benchmark threshold:** If a 10-command batch exceeds 500ms cumulative COM init overhead, introduce `thread_local!` caching. Measure during Phase 1 integration tests. + +**References:** +- Microsoft UIA threading docs: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-threading +- `uiautomation::UIAutomation::new()` internally calls `CoInitializeEx(COINIT_MULTITHREADED)` — MTA is correct for CLI + +## Technical Approach + +### Architecture + +``` + ┌─────────────────────────────┐ + │ agent-desktop (binary) │ + │ src/main.rs │ + │ build_adapter() │ + │ #[cfg(target_os="windows")]│ + └──────────┬──────────────────┘ + │ &dyn PlatformAdapter + ┌──────────▼──────────────────┐ + │ agent-desktop-core │ + │ PlatformAdapter trait (22) │ + │ SnapshotEngine, RefMap │ + │ Commands, Action enum │ + └──────────┬──────────────────┘ + │ impl PlatformAdapter + ┌───────────────────▼───────────────────┐ + │ agent-desktop-windows │ + │ ┌─────────┐ ┌──────────┐ ┌─────────┐│ + │ │ tree/ │ │ actions/ │ │ input/ ││ + │ │UITreeWalk│ │UIInvoke │ │SendInput││ + │ │UICacheReq│ │UIValue │ │Clipboard││ + │ │UIMatcher │ │UIToggle │ │Mouse ││ + │ └─────────┘ └──────────┘ └─────────┘│ + │ ┌──────────────────────────────────┐ │ + │ │ system/ │ │ + │ │ CreateProcess, ShowWindow, BitBlt │ │ + │ │ EnumWindows, WaitForInputIdle │ │ + │ └──────────────────────────────────┘ │ + │ │ + │ Dependencies: │ + │ - uiautomation 0.24+ (UIA wrapper) │ + │ - windows 0.62+ (Win32 APIs) │ + └────────────────────────────────────────┘ +``` + +### Implementation Phases + +#### Phase 0: Pre-Work (Foundation Fixes) + +Minimal, targeted changes that benefit all platforms and unblock Windows work. This is the **only phase that touches files outside `crates/windows/`**. + +**0.1. Refactor BLOCKED_COMBOS out of core** (`crates/core/src/commands/press.rs:8-14`) + +Current state: macOS-specific `BLOCKED_COMBOS` const in core. `key_down`/`key_up` skip the check entirely. + +Approach: Add a `blocked_combos(&self) -> &[&str]` method to `PlatformAdapter` with an empty default. The `press`, `key_down`, and `key_up` commands call `adapter.blocked_combos()` to validate. macOS adapter returns its current list. Windows adapter returns Windows-specific list. + +Files changed: +- `crates/core/src/adapter.rs` — add `blocked_combos` method with empty default +- `crates/core/src/commands/press.rs` — replace const with `adapter.blocked_combos()` call +- `crates/core/src/commands/key_down.rs` — add blocked combo check (safety fix) +- `crates/core/src/commands/key_up.rs` — add blocked combo check (safety fix) +- `crates/macos/src/adapter.rs` — override `blocked_combos` returning current macOS list + +### Research Insights: Phase 0 + +**Security: `key_down`/`key_up` bypass is a real bug.** Currently on macOS, `press ctrl+alt+delete` is blocked but `key-down ctrl` → `key-down alt` → `key-down delete` → `key-up delete` → `key-up alt` → `key-up ctrl` bypasses the check entirely. The Phase 0 fix must apply blocked combo validation to individual key events by tracking modifier state, not just checking the full combo string. + +**Pattern: Phase 0 must be an atomic PR.** Ship all Phase 0 changes in a single PR, reviewed and merged before any Windows work begins. This prevents merge conflicts and ensures core changes are validated by macOS CI first. + +--- + +**0.2. Remove dead `permission_denied()` method** (`crates/core/src/error.rs:107-115`) + +Zero callers in the entire codebase. The macOS adapter constructs permission errors via `PermissionReport::Denied { suggestion }` directly. Remove the dead method. + +Files changed: +- `crates/core/src/error.rs` — remove `permission_denied()` method + +**0.3. Add Windows CI job** (`.github/workflows/ci.yml`) + +Add `test-windows` job on `windows-latest`. Use PowerShell for binary size check (`(Get-Item target/release/agent-desktop.exe).Length`). Initially runs only `cargo check` and `cargo test --lib -p agent-desktop-windows` until real implementation exists. + +Files changed: +- `.github/workflows/ci.yml` — add `test-windows` job + +### Research Insights: Windows CI + +**Best Practices:** +- GitHub Actions `windows-latest` runners have a real desktop session — UIA works, GUI apps render +- Use `--test-threads=1` for integration tests — concurrent tests fighting over foreground windows causes flakiness +- Scope clippy to `cargo clippy -p agent-desktop-windows --all-targets -- -D warnings` — avoid false positives from platform-gated code in other crates +- `notepad.exe` is the ideal CI test target — guaranteed available, simple UI, has textfield +- `calc.exe` is a Store/UWP app that may NOT be pre-installed on CI images — do not rely on it + +**CI Workflow Pattern:** +```yaml +test-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo clippy -p agent-desktop-windows --all-targets -- -D warnings + - run: cargo test --lib -p agent-desktop-windows + - name: Integration tests + run: | + Start-Process notepad.exe + Start-Sleep -Seconds 2 + cargo test --test windows_integration -- --test-threads=1 + Stop-Process -Name notepad -ErrorAction SilentlyContinue +``` + +--- + +**0.4. Create folder skeleton in `crates/windows/`** + +Create the full subfolder structure with `mod.rs` files and empty stubs. Each stub module re-exports nothing initially. `adapter.rs` has an empty `impl PlatformAdapter for WindowsAdapter {}` that delegates nothing (same as current). + +### Research Insights: cfg-gate Pattern + +**Every `.rs` file in `crates/windows/src/` must use the cfg-gate pattern:** + +```rust +// Every file follows this pattern: +#[cfg(target_os = "windows")] +mod imp { + // Real implementation using uiautomation, windows crates +} + +#[cfg(not(target_os = "windows"))] +mod imp { + // Stub that returns AdapterError::not_supported() +} + +// Re-export from imp +pub use imp::*; +// OR: pub(crate) fn some_function(...) { imp::some_function(...) } +``` + +This ensures `cargo check` and `cargo clippy` pass on macOS/Linux during development. Without these stubs, CI on non-Windows platforms will fail. Budget ~15-20 additional LOC per file for stubs. + +**Add `rustc-hash` to dependencies.** The macOS adapter uses `FxHashSet` from `rustc-hash` for visited-set tracking. The Windows adapter needs the same for cycle prevention in tree traversal. Currently missing from the planned Cargo.toml. + +--- + +Files created: +``` +crates/windows/src/ +├── lib.rs # mod declarations + re-export WindowsAdapter +├── adapter.rs # PlatformAdapter impl (empty initially) +├── tree/ +│ ├── mod.rs +│ ├── element.rs +│ ├── builder.rs +│ ├── roles.rs +│ ├── resolve.rs +│ └── surfaces.rs +├── actions/ +│ ├── mod.rs +│ ├── dispatch.rs +│ ├── activate.rs +│ └── extras.rs +├── input/ +│ ├── mod.rs +│ ├── keyboard.rs +│ ├── mouse.rs +│ └── clipboard.rs +└── system/ + ├── mod.rs + ├── app_ops.rs + ├── window_ops.rs + ├── key_dispatch.rs + ├── permissions.rs + ├── screenshot.rs + └── wait.rs +``` + +**0.5. Create `error_mapping.rs`** (`crates/windows/src/error_mapping.rs`) + +Consolidate all HRESULT→ErrorCode and UIA error→AdapterError conversion in one place. This unblocks all subsequent phases and prevents duplicating error mapping logic across modules. + +```rust +// crates/windows/src/error_mapping.rs (~60 LOC) +fn hresult_to_error_code(hr: i32) -> ErrorCode { + match hr { + 0x80040201 => ErrorCode::StaleRef, // UIA_E_ELEMENTNOTAVAILABLE + 0x80040200 => ErrorCode::ActionFailed, // UIA_E_ELEMENTNOTENABLED + 0x80040204 => ErrorCode::ActionNotSupported, // UIA_E_NOTSUPPORTED + 0x80131509 => ErrorCode::ActionFailed, // UIA_E_INVALIDOPERATION + 0x80131505 => ErrorCode::Timeout, // UIA_E_TIMEOUT + 0x80070005 => ErrorCode::PermDenied, // E_ACCESSDENIED + 0x80070057 => ErrorCode::InvalidArgs, // E_INVALIDARG + _ => ErrorCode::Internal, // E_FAIL and others + } +} +``` + +Files created: +- `crates/windows/src/error_mapping.rs` + +--- + +**0.6. Add dependencies to `crates/windows/Cargo.toml`** + +```toml +[dependencies] +agent-desktop-core.workspace = true +thiserror.workspace = true +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +base64.workspace = true +rustc-hash.workspace = true + +[target.'cfg(target_os = "windows")'.dependencies] +uiautomation = { version = "0.24", features = ["process"] } +windows = { version = "0.62", features = [ + "Win32_Foundation", + "Win32_System_Threading", + "Win32_System_Com", + "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging", + "Win32_System_DataExchange", + "Win32_Graphics_Gdi", + "Win32_System_Memory", + "Win32_System_ProcessStatus", + "Win32_Graphics_Dwm", + "Win32_UI_Accessibility", + "Win32_Storage_FileSystem", +] } +``` + +### Research Insights: Dependencies + +**Added `Win32_Graphics_Dwm` feature** — needed for `DwmGetWindowAttribute(DWMWA_CLOAKED)` to filter cloaked windows (virtual desktop windows that `EnumWindows` returns but aren't visible). + +**Added `Win32_UI_Accessibility` feature** — needed for `IUIAutomation2` (timeout configuration: `SetConnectionTimeout`, `SetTransactionTimeout`) and `IsHungAppWindow` pre-check. + +**Added `Win32_Storage_FileSystem` feature** — needed for `GetFileVersionInfoW` / `VerQueryValueW` (Chromium version detection from exe). + +**Removed `uiautomation` `clipboard` feature** — clipboard operations should use the `windows` crate directly for consistency with other Win32 operations and to avoid duplicating clipboard logic across two crate APIs. + +**Added `rustc-hash` workspace dependency** — same `FxHashSet` used by macOS adapter for cycle prevention. + +--- + +**0.7. Fix PID type: `i32` → `u64`** (`crates/core/src/node.rs`) + +`WindowInfo.pid` and `AppInfo.pid` are `i32`, but Windows PIDs are `u32` (DWORD) and macOS PIDs are `pid_t` (signed 32-bit). Using `u64` accommodates both platforms without truncation risk. This is a two-line type change in core with mechanical updates to callers — trivial now, painful migration later when Windows snapshot tests depend on PID handling. + +Files changed: +- `crates/core/src/node.rs` — change `pid: i32` → `pid: u64` in `WindowInfo` and `AppInfo` +- All callers (mechanical `as u64` casts in macOS adapter, direct use in Windows adapter) + +--- + +#### Phase 1: Observation Tier + +Implement the commands that let agents SEE the Windows desktop. This is the most critical tier — without observation, nothing else works. + +**1.1. Tree traversal + role mapping** (`tree/builder.rs`, `tree/roles.rs`, `tree/element.rs`) + +Implement `get_tree` using `UITreeWalker` with `UICacheRequest` for batch attribute retrieval. Map Windows UIA `ControlType` values to agent-desktop's unified role strings. + +``` +tree/element.rs — Wrapper around UIElement, attribute readers (name, role, value, states, bounds) +tree/builder.rs — build_subtree() using UITreeWalker, depth-first traversal, ancestor-set cycle prevention +tree/roles.rs — ControlType → role string mapping (Button→button, Edit→textfield, CheckBox→checkbox, etc.) +``` + +Key UIA control type mappings: +| UIA ControlType | agent-desktop role | +|---|---| +| UIA_ButtonControlTypeId | button | +| UIA_EditControlTypeId | textfield | +| UIA_CheckBoxControlTypeId | checkbox | +| UIA_HyperlinkControlTypeId | link | +| UIA_MenuItemControlTypeId | menuitem | +| UIA_TabItemControlTypeId | tab | +| UIA_SliderControlTypeId | slider | +| UIA_ComboBoxControlTypeId | combobox | +| UIA_TreeItemControlTypeId | treeitem | +| UIA_DataItemControlTypeId | cell | +| UIA_TextControlTypeId | statictext | +| UIA_GroupControlTypeId | group | +| UIA_WindowControlTypeId | window | +| UIA_ToolBarControlTypeId | toolbar | +| UIA_MenuBarControlTypeId | menubar | +| UIA_ListItemControlTypeId | listitem | +| UIA_RadioButtonControlTypeId | radiobutton | +| UIA_SpinnerControlTypeId | spinbutton | +| UIA_ProgressBarControlTypeId | progressbar | +| UIA_ScrollBarControlTypeId | scrollbar | +| UIA_StatusBarControlTypeId | statusbar | +| UIA_ThumbControlTypeId | thumb | +| UIA_SplitButtonControlTypeId | splitbutton | +| UIA_HeaderItemControlTypeId | columnheader | + +### Research Insights: CacheRequest Optimization (Critical Performance) + +**This is THE most impactful performance decision.** Without CacheRequest, every property access on every element triggers a separate cross-process COM call. Benchmarks from research: + +| App | Without CacheRequest | With CacheRequest | Improvement | +|-----|---------------------|-------------------|-------------| +| Explorer | ~1.5s | ~200ms | 7.5x | +| VS Code | ~6s+ | ~500ms | 12x | +| Chrome | ~15s+ | ~1.2s | 12.5x | + +**Properties to cache (batch in a single cross-process call):** +``` +UIA_NamePropertyId → AccessibilityNode.name +UIA_ControlTypePropertyId → role (via roles.rs mapping) +UIA_BoundingRectanglePropertyId → bounds +UIA_ValueValuePropertyId → value +UIA_HelpTextPropertyId → description +UIA_IsEnabledPropertyId → states.enabled +UIA_HasKeyboardFocusPropertyId → states.focused +UIA_IsKeyboardFocusablePropertyId → states.focusable +UIA_IsOffscreenPropertyId → (skip offscreen elements) +UIA_RuntimeIdPropertyId → element re-identification +UIA_ProcessIdPropertyId → RefEntry.pid +``` + +**Patterns to pre-fetch:** +``` +InvokePattern → determines available_actions: ["click"] +ValuePattern → determines available_actions: ["set-value", "clear"] +TogglePattern → determines available_actions: ["toggle", "check", "uncheck"] +ExpandCollapsePattern → determines available_actions: ["expand", "collapse"] +SelectionItemPattern → determines available_actions: ["select"] +ScrollPattern → determines available_actions: ["scroll", "scroll-to"] +RangeValuePattern → determines available_actions: ["set-value"] (for sliders) +``` + +**Use `AutomationElementMode::Full` (not `None`).** Research confirmed that `None` mode elements cannot receive actions, cannot call `build_updated_cache()`, and **cannot be upgraded to Full**. Since agent-desktop's RefMap stores elements that may later receive actions (click, set-value), Full mode is required. The cost is minimal — cached properties are still read locally. + +**Use the control view walker** (not raw view). This eliminates internal framework elements: +- WPF: removes layout containers, adorner decorators +- Win32: removes internal child windows of common controls +- Typically reduces element count by 40-60% + +**Implementation pattern:** +```rust +let automation = UIAutomation::new()?; +let cache = automation.create_cache_request()?; +cache.add_property(UIProperty::Name)?; +cache.add_property(UIProperty::ControlType)?; +cache.add_property(UIProperty::BoundingRectangle)?; +// ... add all properties above +cache.add_pattern(UIPatternType::Invoke)?; +cache.add_pattern(UIPatternType::Value)?; +// ... add all patterns above +cache.set_tree_scope(TreeScope::Element)?; +cache.set_element_mode(ElementMode::Full)?; // Full mode: elements can receive actions later + +let walker = automation.get_control_view_walker()?; +// Use walker.get_first_child_build_cache(&root, &cache) for traversal +``` + +**Defense-in-depth timeout protection for hung apps:** +1. **Layer 1 — Pre-check:** Call `IsHungAppWindow(hwnd)` before UIA traversal. Return `ACTION_FAILED` immediately if hung. +2. **Layer 2 — UIA timeouts:** Cast to `IUIAutomation2` and set `ConnectionTimeout` to 1000ms (default 2s) and `TransactionTimeout` to 5000ms (default 20s). +3. **Layer 3 — Thread timeout:** Wrap entire traversal in `mpsc::channel` + `recv_timeout(Duration::from_secs(2))`. +4. **Layer 4 — Graceful degradation:** If a subtree times out, return partial tree with timed-out branch pruned. + +```rust +// Layer 1: pre-check +if unsafe { IsHungAppWindow(hwnd) }.as_bool() { + return Err(AdapterError::action_failed("Target window is not responding")); +} +// Layer 2: configure UIA timeouts via IUIAutomation2 +let automation2: IUIAutomation2 = raw_automation.cast()?; +unsafe { automation2.SetConnectionTimeout(1000)?; automation2.SetTransactionTimeout(5000)?; } +// Layer 3: thread timeout +let (tx, rx) = std::sync::mpsc::channel(); +std::thread::spawn(move || { tx.send(build_subtree(...)).ok(); }); +rx.recv_timeout(Duration::from_secs(2)) + .map_err(|_| AdapterError::timeout("Tree traversal timed out"))? +``` + +**References:** +- UIA caching: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-cachingforclients +- `uiautomation::UICacheRequest`: https://docs.rs/uiautomation/latest/uiautomation/core/struct.UICacheRequest.html + +--- + +Trait methods implemented: `get_tree` + +**1.2. Window and app enumeration** (`system/window_ops.rs`, `system/app_ops.rs`) + +``` +system/window_ops.rs — EnumWindows + GetWindowText + GetWindowThreadProcessId for list_windows + GetForegroundWindow for focused_window +system/app_ops.rs — EnumProcesses + QueryFullProcessImageName for list_apps +``` + +### Research Insights: Window Enumeration + +**Filter cloaked windows.** `EnumWindows` returns windows on virtual desktops (cloaked). Filter via: +```rust +use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_CLOAKED}; +let mut cloaked: u32 = 0; +DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &mut cloaked as *mut _ as _, size_of::<u32>() as u32); +if cloaked != 0 { continue; } // Skip cloaked windows +``` + +**Handle ApplicationFrameHost.exe for UWP apps.** UWP windows report their PID as ApplicationFrameHost.exe, not the actual app. Use `GetApplicationUserModelId` or `GetPackageFamilyName` to resolve the real app identity. For `list_windows`, this means the PID in `WindowInfo` may not match the app that owns the content. + +**EnumWindows filters out UWP windows (Windows 8+).** Use `FindWindowEx` loop as fallback to discover `ApplicationFrameWindow` class windows, then filter cloaked ones via `DwmGetWindowAttribute(DWMWA_CLOAKED)`. + +**Main window detection algorithm** (matches Alt+Tab/taskbar visibility): +1. Must be visible (`IsWindowVisible`) +2. Must not be cloaked +3. Must not be a tool window (`WS_EX_TOOLWINDOW`) +4. Must have no owner (unless `WS_EX_APPWINDOW` is set) +5. Must not be the shell/desktop window + +**Window ID generation.** Use the same `FxHasher` pattern as macOS: `hash(pid, title)` → `w-{hex}`. + +--- + +Trait methods implemented: `list_windows`, `list_apps`, `focused_window` + +**1.3. Element finding and state queries** (`tree/resolve.rs`) + +``` +tree/resolve.rs — resolve_element via UIMatcher (pid, control_type, name, bounds_hash) + Also used by find, get, is commands (which use SnapshotEngine in core + adapter.resolve_element) +``` + +### Research Insights: Element Resolution + +**Use server-side `FindFirst`, not client-side tree walk.** For `resolve_element`, build a `UIMatcher` condition combining `ProcessId`, `ControlType`, and `Name`, then call `find_first(TreeScope::Subtree, &condition)`. This is a single cross-process call instead of walking potentially thousands of elements. + +**Store `RuntimeId` in RefEntry for fast verification.** UIA elements have a `RuntimeId` property (an `int[]` array). If the RuntimeId matches, the element is confirmed valid without fuzzy matching. Falls back to `(pid, role, name, bounds_hash)` when RuntimeId changes (UI rebuilt). + +**Virtualized control awareness.** WPF/UWP `ListView` and `DataGrid` use UI virtualization — only visible items have UIA elements. When `scroll` moves content, previously-resolved refs to list items may point to recycled containers. Return `STALE_REF` and let the agent re-snapshot. + +--- + +Trait methods implemented: `resolve_element`, `get_live_value`, `get_element_bounds` + +**1.4. Permissions check** (`system/permissions.rs`) + +UIA doesn't require TCC-like permissions for most apps. Check: +- COM initialization succeeds +- Can enumerate at least one window +- Return `PermissionReport::Granted` or `Denied` with Windows-specific guidance + +### Research Insights: Permissions + +**UIA reads work across UIPI boundaries without special permissions.** Unlike macOS TCC, no explicit user grant is needed for UIA read access. The permission check is essentially a smoke test that COM works and the desktop is accessible. + +**Report UIPI limitations proactively.** When `check_permissions` detects the process is not elevated but elevated apps exist, add a note to the permission status: "Some elevated applications may not respond to input commands. Run as administrator for full control." + +--- + +Trait methods implemented: `check_permissions` + +**1.5. Surface detection** (`tree/surfaces.rs`) + +Detect Windows surface types: +- Menu bars (UIA Menu/MenuBar control type) +- Modal dialogs (Window with IsModal=true) +- Popups/Flyouts (Pane with IsKeyboardFocusable) + +### Research Insights: Surface Mapping + +**SnapshotSurface → Windows mapping:** +| SnapshotSurface | Windows Detection | Notes | +|---|---|---| +| `Window` | `UIElement` from HWND | Direct equivalent | +| `Focused` | `get_focused_element()` → walk to nearest surface | Same as macOS | +| `Menu` | `ControlType == Menu` or `MenuBar` | Direct equivalent | +| `Menubar` | `ControlType == MenuBar` | Direct equivalent | +| `Alert` | `ControlType == Window` with `IsModal == true` | Maps to modal dialogs | +| `Sheet` | **No Windows equivalent** | Return `ElementNotFound("No sheet on Windows")` | +| `Popover` | **No Windows equivalent** | Return `ElementNotFound("No popover on Windows")` | + +--- + +Trait methods implemented: `list_surfaces` + +**1.6. Screenshot** (`system/screenshot.rs`) + +Use `PrintWindow` with `PW_RENDERFULLCONTENT` for window-specific capture, `BitBlt` for screen capture. `GetDIBits` to extract raw pixel data. Encode to PNG via the same base64 pipeline as macOS. + +### Research Insights: Screenshot + +**Use `PrintWindow` with `PW_RENDERFULLCONTENT` (flag value `2`) for per-window capture.** Plain `BitBlt` returns black pixels for DWM-composited windows (most modern apps). `PrintWindow` with this flag requests the window to render its full content to the provided DC. + +**Fallback chain:** `PrintWindow(PW_RENDERFULLCONTENT)` → `PrintWindow(0)` → `BitBlt` (last resort). + +**Multi-monitor support.** For `ScreenshotTarget::Screen(idx)`, use `EnumDisplayMonitors` to get monitor rects, then `BitBlt` with the correct source coordinates. `ScreenshotTarget::FullScreen` should capture the virtual screen (`GetSystemMetrics(SM_XVIRTUALSCREEN/SM_YVIRTUALSCREEN/SM_CXVIRTUALSCREEN/SM_CYVIRTUALSCREEN)`). + +**DPI awareness.** Call `SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)` early in adapter initialization. Without this, screenshot coordinates and dimensions may be scaled incorrectly on high-DPI displays. + +--- + +Trait methods implemented: `screenshot` + +**After Phase 1:** `snapshot --app Explorer` returns valid JSON with refs. `list-windows`, `list-apps`, `find`, `get`, `is`, `list-surfaces`, `screenshot`, `permissions` all work. + +--- + +#### Phase 2: Core Interaction Tier + +Enable agents to ACT on elements. Pattern-based UIA actions. + +**2.1. Action dispatch** (`actions/dispatch.rs`) + +Central `perform_action(handle, action)` that matches `Action` variants to UIA patterns: +``` +Action::Click → UIInvokePattern::invoke() +Action::SetValue(v) → UIValuePattern::set_value(v) +Action::SetFocus → UIElement::set_focus() +Action::Expand → UIExpandCollapsePattern::expand() +Action::Collapse → UIExpandCollapsePattern::collapse() +Action::Select(v) → UISelectionItemPattern::select() +Action::Toggle → UITogglePattern::toggle() +Action::Check → UITogglePattern (set to On) +Action::Uncheck → UITogglePattern (set to Off) +Action::Clear → UIValuePattern::set_value("") +``` + +### Research Insights: Action Dispatch + +**NativeHandle COM pointer lifecycle (Critical):** + +On Windows, `NativeHandle` wraps a `*const c_void` that points to a `UIElement` (which wraps `IUIAutomationElement` COM interface). The lifecycle contract: + +```rust +// Creating a NativeHandle from UIElement: +fn element_to_handle(el: UIElement) -> NativeHandle { + let boxed = Box::new(el); + NativeHandle::from_ptr(Box::into_raw(boxed) as *const std::ffi::c_void) +} + +// Recovering UIElement from NativeHandle (non-owning): +unsafe fn handle_to_element(handle: &NativeHandle) -> &UIElement { + &*(handle.as_raw() as *const UIElement) +} + +// The UIElement's Drop impl calls IUIAutomationElement::Release() (COM Release) +// ManuallyDrop prevents double-free when temporarily borrowing +``` + +**Important:** The macOS adapter uses `ManuallyDrop::new(AXElement(...))` to prevent double-free when temporarily borrowing the handle. The Windows adapter must use the same pattern. Never call `Box::from_raw` on a handle unless you intend to take ownership and free it. + +**Pattern fallback chain:** +When `InvokePattern` is unavailable for a Click action (some legacy controls), fall back to: +1. `LegacyIAccessiblePattern::do_default_action()` — MSAA bridge +2. Coordinate click via `SendInput` at element center — last resort + +**Check/Uncheck via TogglePattern:** +`TogglePattern::toggle()` cycles through states (On → Off → Indeterminate → On). For `Check`, call `toggle()` only if current state is not `On`. For `Uncheck`, call `toggle()` only if current state is not `Off`. Read `get_toggle_state()` first. + +**HRESULT → ErrorCode mapping:** +| HRESULT | ErrorCode | Context | +|---|---|---| +| `UIA_E_ELEMENTNOTAVAILABLE` (0x80040201) | `StaleRef` | Element no longer in UI tree | +| `UIA_E_ELEMENTNOTENABLED` (0x80040200) | `ActionFailed` | Element disabled | +| `UIA_E_NOTSUPPORTED` (0x80040204) | `ActionNotSupported` | Pattern not available | +| `UIA_E_INVALIDOPERATION` (0x80131509) | `ActionFailed` | Operation invalid in current state | +| `UIA_E_TIMEOUT` (0x80131505) | `Timeout` | Provider took too long | +| `E_ACCESSDENIED` (0x80070005) | `PermDenied` | UIPI or security restriction | +| `E_FAIL` (0x80004005) | `Internal` | Generic COM failure | +| `E_INVALIDARG` (0x80070057) | `InvalidArgs` | Bad argument to UIA call | + +--- + +**2.2. Activation chain** (`actions/activate.rs`) + +Smart activation: perform UIA patterns without foreground activation by default. Coordinate click via `SendInput` is reserved for an explicit physical policy path. + +### Research Insights: Window Activation + +**`SetForegroundWindow` restrictions.** Windows prevents background processes from stealing foreground. The call succeeds only if: +- The calling process is the foreground process +- The foreground process has called `AllowSetForegroundWindow` for the caller +- The user is not interacting with another window + +**Workaround chain:** +1. `AttachThreadInput` to the foreground thread +2. `SetForegroundWindow(target)` +3. `BringWindowToTop(target)` +4. `SetFocus` on the specific element via UIA + +--- + +**2.3. Type text** (`input/keyboard.rs`) + +`Action::TypeText(s)` → iterate characters, use `SendInput` with `KEYEVENTF_UNICODE` flag for each. Handle delay between keystrokes (`--delay` flag). + +### Research Insights: SendInput Batching (Major Performance) + +**Batch all `INPUT` structs in a single `SendInput` call for 10-50x performance improvement.** + +Instead of: +```rust +// SLOW: one cross-process call per character +for ch in text.chars() { + SendInput(&[key_down(ch)], size); // cross-process call + SendInput(&[key_up(ch)], size); // cross-process call +} +``` + +Do: +```rust +// FAST: single cross-process call for entire text +let mut inputs = Vec::with_capacity(text.len() * 2); +for ch in text.chars() { + inputs.push(key_down_unicode(ch)); + inputs.push(key_up_unicode(ch)); +} +SendInput(&inputs, size_of::<INPUT>() as i32); // ONE call +``` + +**Check `SendInput` return value.** It returns the number of events successfully inserted. If 0, input was blocked (UIPI). Detect elevation mismatch and return `PERM_DENIED` with actionable guidance. + +--- + +Trait methods implemented: `execute_action` (all interaction commands use this) + +**After Phase 2:** `click @e3`, `type @e5 "hello"`, `set-value @e5 "world"`, `toggle @e7`, `select @e8 "Option A"` all work. + +--- + +#### Phase 3: Input Synthesis Tier + +Raw keyboard and mouse input, independent of accessibility patterns. + +**3.1. Keyboard synthesis** (`input/keyboard.rs`) + +- `press` — `SendInput` with virtual key codes + modifier mapping (Cmd→Ctrl) +- `key-down` / `key-up` — individual key events +- Windows-specific blocked combos: `alt+f4`, `ctrl+alt+delete`, `win+l`, `win+r` + +### Research Insights: Keyboard Synthesis + +**Cmd→Ctrl mapping contract.** When agent sends `cmd+c`, the Windows adapter must: +1. Map `cmd` modifier → `VK_CONTROL` +2. Map the combo `cmd+c` → `ctrl+c` +3. This is a display-only mapping — the JSON output should show the actual keystroke sent (`ctrl+c`) + +**Virtual key code mapping for common keys:** +``` +VK_CONTROL (0x11), VK_SHIFT (0x10), VK_MENU (0x12=Alt) +VK_LWIN (0x5B), VK_RETURN (0x0D), VK_TAB (0x09), VK_ESCAPE (0x1B) +VK_BACK (0x08), VK_DELETE (0x2E), VK_SPACE (0x20) +VK_UP/DOWN/LEFT/RIGHT (0x26-0x28, 0x25) +VK_HOME (0x24), VK_END (0x23), VK_PRIOR (0x21=PageUp), VK_NEXT (0x22=PageDown) +VK_F1-VK_F12 (0x70-0x7B) +``` + +**DoubleClick/TripleClick/RightClick via SendInput.** These are NOT UIA pattern operations — they require raw mouse input synthesis: +- DoubleClick: two `LEFTDOWN`+`LEFTUP` pairs with <500ms gap at same coordinates +- TripleClick: three `LEFTDOWN`+`LEFTUP` pairs +- RightClick: `RIGHTDOWN`+`RIGHTUP` pair + +--- + +**3.2. Mouse synthesis** (`input/mouse.rs`) + +- `hover` — `SendInput` MOUSEEVENTF_MOVE + MOUSEEVENTF_ABSOLUTE +- `mouse-move`, `mouse-click`, `mouse-down`, `mouse-up` — direct SendInput +- `drag` — mouse down → series of moves → mouse up + +### Research Insights: Mouse Coordinate Translation + +**DPI awareness is critical.** SendInput uses normalized absolute coordinates (0-65535 range), not pixels: +```rust +let abs_x = (pixel_x * 65535) / screen_width; +let abs_y = (pixel_y * 65535) / screen_height; +``` + +**Multi-monitor.** Coordinates must account for the virtual screen origin. Use `MOUSEEVENTF_VIRTUALDESK` flag: +```rust +let virt_left = GetSystemMetrics(SM_XVIRTUALSCREEN); +let virt_top = GetSystemMetrics(SM_YVIRTUALSCREEN); +let virt_width = GetSystemMetrics(SM_CXVIRTUALSCREEN); +let virt_height = GetSystemMetrics(SM_CYVIRTUALSCREEN); +// Correct formula: divide by (size - 1), not size +let abs_x = ((pixel_x - virt_left) * 65535) / (virt_width - 1); +let abs_y = ((pixel_y - virt_top) * 65535) / (virt_height - 1); +// Set MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK on the INPUT struct +``` + +**UIA BoundingRectangle returns physical pixels** when DPI-aware (`SetProcessDpiAwarenessContext(PER_MONITOR_AWARE_V2)`). No conversion needed between UIA bounds and SendInput target coordinates. + +**Double/triple click via SendInput.** Batch all events in a single call — no `Sleep` needed between clicks: +- DoubleClick: 4 `INPUT` events (`LBUTTONDOWN`, `LBUTTONUP`, `LBUTTONDOWN`, `LBUTTONUP`) in one `SendInput` call +- TripleClick: 6 `INPUT` events in one `SendInput` call + +--- + +**3.3. Scroll** (`actions/extras.rs` + `input/mouse.rs`) + +- `scroll` — `UIScrollPattern::scroll()` for element scroll, `SendInput` MOUSEEVENTF_WHEEL for window scroll +- `scroll-to` — `UIScrollPattern::set_scroll_percent()` or repeated scroll events + +Trait methods implemented: `press_key_for_app`, `mouse_event`, `drag` + +**After Phase 3:** All keyboard, mouse, scroll, and drag commands work. + +--- + +#### Phase 4: System Operations Tier + +App lifecycle, window management, clipboard. + +**4.1. App lifecycle** (`system/app_ops.rs`) + +- `launch` — `CreateProcessW` for paths, `ShellExecuteW` for app names. `WaitForInputIdle` for `--wait` flag. Returns `WindowInfo` of first window. +- `close-app` — `WM_CLOSE` for graceful, `TerminateProcess` for `--force` + +### Research Insights: App Lifecycle Security + +**CRITICAL: Use `CreateProcessW` as primary launch method.** `CreateProcessW` with non-NULL `lpApplicationName` does NOT parse shell metacharacters — safe for untrusted input. Always pass the full resolved path as `lpApplicationName`. + +**Path resolution chain:** +1. App Paths registry (`HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\{name}.exe`) — fastest lookup for installed apps +2. `SearchPathW` API — searches system PATH, Windows directory +3. PATH environment variable — manual split and search + +**`ShellExecuteW` only as fallback** (for UWP apps, protocol handlers). When falling back: +- Always pass the app name as `lpFile` (not `lpParameters`) and leave `lpParameters` null +- Reject shell metacharacters in `lpFile`: `&`, `|`, `>`, `<`, `;`, `` ` `` +- Do NOT use the regex `[a-zA-Z0-9._-]+(.exe)?` — it rejects legitimate paths containing spaces (e.g., `C:\Program Files\App Name\app.exe`) +- Instead, reject only the 6 metacharacters that trigger `cmd.exe` interpretation when `lpFile` resolves to a `.bat`/`.cmd` file + +**UWP apps:** Use `IApplicationActivationManager::ActivateApplication` with the app's AUMID (Application User Model ID). `CreateProcessW` does not work for UWP apps. + +**Windows protected processes (PPL).** `TerminateProcess` fails with `ERROR_ACCESS_DENIED` on Protected Process Light (PPL) processes. `OpenProcess(PROCESS_TERMINATE)` itself fails — you never get a handle. Detect protection level via `GetProcessInformation(ProcessProtectionLevelInfo)`. Protected processes include: antivirus (Antimalware signer), LSASS, csrss.exe, smss.exe. `SeDebugPrivilege` does NOT bypass PPL. + +**Graceful close always works:** `WM_CLOSE` sent via `PostMessage` works even for protected processes because it's a window message, not a process operation. For `close-app --force`, try `TerminateProcess` first; if `ERROR_ACCESS_DENIED`, fall back to `WM_CLOSE` and report protection status. + +**`WaitForInputIdle` caveats (Raymond Chen).** This is a one-shot function — once a process reaches "input idle" (first `GetMessage` call), subsequent calls return immediately even if busy. Unreliable for multi-threaded apps (splash screen thread can trigger it). Combine with UIA tree polling for reliable readiness detection. + +--- + +**4.2. Window operations** (`system/window_ops.rs`) + +- `focus-window` — `SetForegroundWindow` + `BringWindowToTop` +- `resize-window` — `SetWindowPos` with new dimensions +- `move-window` — `SetWindowPos` with new coordinates +- `minimize` / `maximize` / `restore` — `ShowWindow` with `SW_MINIMIZE` / `SW_MAXIMIZE` / `SW_RESTORE` + +**4.3. Clipboard** (`input/clipboard.rs`) + +- `clipboard-get` — `OpenClipboard` + `GetClipboardData(CF_UNICODETEXT)` + `GlobalLock/Unlock` +- `clipboard-set` — `OpenClipboard` + `EmptyClipboard` + `SetClipboardData(CF_UNICODETEXT)` +- `clipboard-clear` — `OpenClipboard` + `EmptyClipboard` + +### Research Insights: Clipboard Safety + +**CRITICAL: RAII `ClipboardGuard` with retry.** + +`OpenClipboard` takes a global system lock. If the process crashes between `OpenClipboard` and `CloseClipboard`, the clipboard is locked system-wide. + +```rust +struct ClipboardGuard; + +impl ClipboardGuard { + fn open() -> Result<Self, AdapterError> { + // Retry up to 10 times with 100ms delay (matches .NET Clipboard.SetDataObject default) + // OpenClipboard returns ERROR_ACCESS_DENIED (5) when another app holds the lock + for attempt in 0..10 { + if unsafe { OpenClipboard(HWND::default()) }.is_ok() { + return Ok(Self); + } + if attempt < 9 { + std::thread::sleep(Duration::from_millis(100)); + } + } + Err(AdapterError::action_failed("Clipboard locked by another application after 10 retries") + .with_suggestion("Close clipboard-intensive applications and retry")) + } +} + +impl Drop for ClipboardGuard { + fn drop(&mut self) { + unsafe { CloseClipboard().ok() }; + } +} +``` + +**Always use UTF-16 (`CF_UNICODETEXT`).** Never use `CF_TEXT` (ANSI) — Windows auto-converts between them via synthesized formats, but `CF_UNICODETEXT` preserves full Unicode. + +**Thread safety:** `OpenClipboard` and `CloseClipboard` MUST be called from the same thread. Different threads in the same process are treated as different owners. The RAII guard must not be sent across threads. + +**`GlobalAlloc` ownership transfer:** After `SetClipboardData(CF_UNICODETEXT, hmem)` succeeds, ownership transfers to the system — do NOT call `GlobalFree`. If `SetClipboardData` fails, you still own the memory and must free it. + +--- + +**4.4. Wait** (`system/wait.rs`) + +- `wait --element @e5` — poll `resolve_element` until success or timeout +- `wait --window "Title"` — poll `list_windows` with title filter +- `wait --text @e5 "expected"` — poll `get_live_value` until match +- `wait --menu` — poll `list_surfaces` for menu surface + +### Research Insights: Wait Command Cleanup + +**`wait --gone` does not exist.** The original plan referenced `wait --gone @e5` but this is not an implemented command variant. Remove from the plan. + +**`wait --text` performance on large trees.** Each poll calls `get_live_value` which resolves the element and reads its value. This is fast for a single element but creates COM overhead per poll iteration. Use a polling interval of 200ms (not 100ms) to reduce COM pressure. + +--- + +Trait methods implemented: `launch_app`, `close_app`, `focus_window`, `window_op`, `get_clipboard`, `set_clipboard`, `clear_clipboard`, `wait_for_menu` + +**After Phase 4:** All 50 commands functional on Windows. + +--- + +#### Phase 5: Polish & Edge Cases + +**5.1. Chromium detection** (in `tree/builder.rs` or `adapter.rs`) + +When `get_tree` returns a suspiciously small tree (< 5 nodes) for a known Chromium process, add a warning to the snapshot output suggesting `--force-renderer-accessibility`. + +### Research Insights: Chromium Detection Update (Chrome 138+) + +**Chrome 138+ (shipped July 2025) enables native UIA by default.** The detection strategy needs updating: + +1. **For Chrome/Edge >= 138:** Native UIA support is on. If tree is sparse, the renderer may not have activated yet — **return the sparse tree immediately with a warning**, and let the AI agent decide whether to retry. Do not silently wait or hide latency. + +2. **For Electron apps:** Electron lags behind Chrome releases. Many Electron apps (VS Code, Slack, Discord) may still be on older Chromium versions. Keep the `--force-renderer-accessibility` guidance for Electron. + +3. **Enterprise policy override:** `UiAutomationProviderEnabled` policy can disable native UIA and revert to MSAA bridge. Supported through Chrome 146. + +4. **Version detection:** Use `GetFileVersionInfoW` + `VerQueryValueW` on the process exe path to read the Chromium major version. For WebView2, use `GetAvailableCoreWebView2BrowserVersionString`. There is no UIA property that exposes the Chromium version — file version is the only reliable method. + +**Updated detection flow (no hidden latency):** +``` +if tree_size < 5 && is_chromium_process(pid) { + let version = get_chromium_version_from_exe(pid); + if is_electron_app(process_name) { + warning = "Electron app may need --force-renderer-accessibility flag" + } else if version >= 138 { + warning = "Chromium renderer initializing. Re-run snapshot to get full tree." + } else { + warning = "Chromium < 138: launch with --force-renderer-accessibility" + } + // Return the sparse tree WITH the warning — let the agent decide to retry +} +``` + +**Known Chromium process names:** +- Browsers: `chrome.exe`, `msedge.exe`, `brave.exe`, `vivaldi.exe` +- Electron: `electron.exe`, `code.exe`, `slack.exe`, `discord.exe`, `teams.exe` + +--- + +**5.2. Adapter-level blocked combos** + +Override `blocked_combos()` returning: +```rust +&["alt+f4", "ctrl+alt+delete", "win+l", "win+r", "ctrl+shift+esc"] +``` + +**5.3. App name resolution** + +Windows apps can be identified by: +- Process name (e.g., `notepad.exe`) +- Window title (e.g., `Untitled - Notepad`) +- App user model ID for UWP/Store apps + +The `launch` and `close-app` commands need to handle all three. Use `shell:AppsFolder` for modern app resolution. + +**5.4. UWP / Windows Store app support** + +UWP apps run in AppContainer sandboxes and may have restricted UIA access. Detect and return `PERM_DENIED` with guidance when UIA calls fail for containerized apps. + +### Research Insights: UWP Considerations + +**UIA read access works fine across AppContainer boundaries** — the UIA framework handles the security boundary transparently. `SendInput` may be unreliable to sandboxed UWP windows. `SetForegroundWindow` may require `AllowSetForegroundWindow` token. + +**WinUI3 desktop apps** (Windows App SDK) run as standard Win32 processes — no sandbox, no special handling needed. + +--- + +## Alternative Approaches Considered + +| Approach | Rejected Because | +|---|---| +| Raw `windows` crate only | ~30% more boilerplate for UIA operations; BSTR handling, COM factory code, condition building all done manually (see brainstorm) | +| `uiautomation` crate only | Doesn't wrap SendInput, clipboard, screenshot, process lifecycle at the level we need | +| AccessBridge for Java apps | Out of scope for Phase 2; Java apps have their own accessibility bridge | +| MSAA (legacy API) | UIA is the modern replacement; MSAA has no CacheRequest, pattern matching, or structured tree walking | + +## System-Wide Impact + +### Interaction Graph + +When a Windows command executes: +1. `main.rs` calls `build_adapter()` → `WindowsAdapter::new()` (no-op, stateless) +2. `dispatch.rs` calls `adapter.method()` → `WindowsAdapter` creates `UIAutomation::new()` (COM init) +3. Method performs UIA/Win32 operations, returns `Result<Value, AppError>` +4. `UIAutomation` instance drops → COM cleanup (automatic via `uiautomation`) +5. JSON envelope written to stdout + +No callbacks, no observers, no event handlers. Fully synchronous, stateless per invocation. + +### Error & Failure Propagation + +| Layer | Error Type | Propagation | +|---|---|---| +| `uiautomation` | `uiautomation::Error` | Caught in adapter, mapped to `AdapterError` with HRESULT in `platform_detail` | +| `windows` crate | `windows::core::Error` | Caught in adapter, mapped to `AdapterError` with HRESULT | +| Win32 API | `GetLastError` | Retrieved via `windows::core::Error::from_win32()`, mapped to `AdapterError` | +| COM failure | HRESULT | Mapped to appropriate `ErrorCode` (see HRESULT mapping table in Phase 2 insights) | + +### Research Insights: Error Handling Pattern + +**Standardized error conversion functions:** +```rust +fn win_err_to_adapter(context: &str, e: windows::core::Error) -> AdapterError { + AdapterError::action_failed(context) + .with_platform_detail(format!("HRESULT 0x{:08X}: {}", e.code().0, e.message())) +} + +fn uia_err_to_adapter(context: &str, e: uiautomation::Error) -> AdapterError { + // Map specific HRESULT values to appropriate ErrorCode + AdapterError::action_failed(context) + .with_platform_detail(format!("UIAutomation: {}", e)) +} +``` + +Place these in a shared `crates/windows/src/error_mapping.rs` (new file, ~60 LOC). + +--- + +### State Lifecycle Risks + +Minimal. The adapter is stateless: +- No COM objects stored across calls (created per-call, dropped before return) +- No persistent handles (NativeHandle in RefMap is re-resolved on use) +- No file locks (RefMap uses atomic write via temp + rename, already Windows-compatible) + +Only risk: clipboard operations use `OpenClipboard/CloseClipboard` which is a global lock. Must always close in a `Drop` guard to prevent deadlock. + +### Research Insights: RefMap on Windows + +**File permissions.** RefMap save has `#[cfg(not(unix))]` branches but uses default permissions on Windows. Windows file ACLs are more complex than Unix modes. For Phase 2, default permissions are acceptable — the file is in `%USERPROFILE%\.agent-desktop\` which is already user-private. Document that enterprise environments may need additional ACL controls. + +--- + +### API Surface Parity + +All 50 CLI commands produce **identical JSON output** on macOS and Windows. The `version` command reports the platform. The `status` command reports platform-specific permission status. No command has different flags or arguments per platform. + +### Research Insights: Parity Gaps + +**Commands with undefined Windows behavior (must document):** +- `SnapshotSurface::Sheet` → return `ElementNotFound` (macOS-only concept) +- `SnapshotSurface::Popover` → return `ElementNotFound` (macOS-only concept) +- `AppInfo.bundle_id` → always `None` on Windows +- DoubleClick/TripleClick/RightClick → must use `SendInput` (UIA has no pattern for these) + +**JSON schema should be ~95% identical.** The 5% difference is in platform-specific fields that are already `Option<T>` (like `bundle_id`). + +--- + +## Acceptance Criteria + +### Tier 1: Windows Adapter Core (must-pass for each phase merge) + +These must pass before ANY Windows PR is merged. Validated per-phase: + +**Phase 0 (pre-work PR):** +- [ ] `cargo clippy --all-targets -- -D warnings` passes with zero warnings +- [ ] `cargo test --lib --workspace` passes (all existing tests still green) +- [ ] `cargo tree -p agent-desktop-core` contains no platform crate names +- [ ] `cargo check` passes on macOS (cfg-gate stubs work) +- [ ] Windows CI job (`cargo check -p agent-desktop-windows`) passes +- [ ] `blocked_combos` method added to trait; `key_down`/`key_up` check it (safety fix) +- [ ] `WindowInfo.pid` and `AppInfo.pid` changed to `u64` +- [ ] Dead `permission_denied()` removed +- [ ] `error_mapping.rs` created with HRESULT→ErrorCode table + +**Phase 1 (observation PR):** +- [ ] `snapshot --app Explorer` returns valid JSON with refs for all interactive elements +- [ ] `snapshot --app Notepad` returns editable textfield with ref +- [ ] `snapshot --app Settings` returns valid tree for modern Windows app +- [ ] `list-windows` returns all visible windows with titles and PIDs +- [ ] `list-apps` returns all running applications +- [ ] `find "Save" --role button` finds elements matching query +- [ ] `get text @eN` returns element's accessible name +- [ ] `is visible @eN` / `is enabled @eN` / `is checked @eN` return boolean +- [ ] `list-surfaces` detects menu bars, dialogs, and popups +- [ ] `screenshot` returns base64-encoded PNG +- [ ] `permissions` reports UIA availability status +- [ ] Explorer snapshot completes in < 2 seconds (with CacheRequest) +- [ ] No file in `crates/windows/` exceeds 400 LOC +- [ ] No `unwrap()` in non-test code + +**Phase 2-3 (interaction + input PR):** +- [ ] `click @eN` invokes the element via InvokePattern +- [ ] `type @eN "hello"` types text into focused element +- [ ] `set-value @eN "world"` sets value via ValuePattern +- [ ] `press ctrl+c` sends Ctrl+C keystroke +- [ ] `press cmd+c` maps to Ctrl+C on Windows (cross-platform parity) +- [ ] `SendInput` return value checked — `PERM_DENIED` on UIPI failure with elevation guidance + +**Phase 4-5 (system + polish PR):** +- [ ] `clipboard-get` / `clipboard-set` / `clipboard-clear` roundtrip correctly +- [ ] Clipboard operations use RAII guard — no system-wide lock on crash +- [ ] `launch notepad` opens Notepad and returns WindowInfo +- [ ] `close-app notepad` closes Notepad gracefully +- [ ] `close-app notepad --force` terminates Notepad process +- [ ] `CreateProcessW` used as primary launch method; `ShellExecuteW` fallback rejects metacharacters +- [ ] Protected processes return `PERM_DENIED` with explanation (not crash) + +### Tier 2: Windows Adapter Complete (must-pass for Phase 2 sign-off) + +These must ALL pass before Phase 2 is declared complete: + +**Functional completeness:** +- [ ] `focus-window --app notepad` brings Notepad to foreground +- [ ] `minimize`, `maximize`, `restore` operate on target window +- [ ] `resize-window` and `move-window` change window geometry +- [ ] `wait --element @eN` polls until element exists +- [ ] `wait --window "Title"` polls until window appears +- [ ] `batch` executes multiple commands in sequence +- [ ] All P2 commands from PRD (hover, drag, mouse-*, key-down/up, window geometry, advanced waits) work +- [ ] Chromium apps with sparse trees get a version-aware warning (returned immediately, no hidden wait) +- [ ] `SnapshotSurface::Sheet` returns `ElementNotFound` with clear message +- [ ] `SnapshotSurface::Popover` returns `ElementNotFound` with clear message + +**Non-functional:** +- [ ] JSON output schema identical to macOS for all commands +- [ ] Binary size < 15MB (release build with `opt-level = "z"`, LTO, strip) +- [ ] Notepad snapshot completes in < 500ms +- [ ] All errors carry `ErrorCode`, `message`, and `suggestion` +- [ ] Every `.rs` file has cfg-gate stubs for cross-platform compilation + +**Quality gates:** +- [ ] Windows CI job passes on every PR +- [ ] Release binary builds for `x86_64-pc-windows-msvc` +- [ ] Integration tests pass on `windows-latest` runner (Explorer, Notepad, Settings, VS Code) +- [ ] `cargo test --lib -p agent-desktop-windows` passes + +## Success Metrics + +| Metric | Target | +|---|---| +| P2-O1: Windows adapter | `snapshot` on Explorer, Notepad, Settings returns valid trees with refs | +| P2-O3: Cross-platform JSON parity | Identical schema output on macOS and Windows for all commands | +| P2-O4: Phase 2 commands ship | hover, drag, mouse-*, key-down/up, window geometry, advanced waits all working | +| P2-O5: Cross-platform CI | GitHub Actions matrix: macOS + Windows | +| P2-O6: Performance | Explorer snapshot < 2s, Notepad snapshot < 500ms (with CacheRequest) | + +## Dependencies & Prerequisites + +| Dependency | Version | Purpose | Risk | +|---|---|---|---| +| `uiautomation` | 0.24+ | UIA tree, patterns, element finding | Single maintainer; escape hatch via `Into<IUIAutomationElement>` | +| `windows` | 0.62+ | Win32 APIs (SendInput, clipboard, GDI, process) | Microsoft-backed, low risk | +| `rustc-hash` | workspace | FxHashSet for cycle prevention in tree traversal | Already used by macOS adapter | +| Windows 10 1809+ | N/A | Minimum OS version (per PRD §2.3) | | +| `windows-latest` CI runner | N/A | GitHub Actions Windows runner | Available, no cost concern | + +## Risk Analysis & Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|---|---|---|---| +| `uiautomation` crate abandoned | Low | Medium | Escape hatch to raw `windows` COM. Thin wrapper, migration path is incremental. | +| COM apartment conflicts | Medium | High | Stateless adapter, per-call init. No stored COM state. `new_direct()` fallback if MTA already initialized. | +| UWP/Store app sandboxing blocks UIA | Medium | Medium | UIA reads work across AppContainer. `SendInput` may fail — detect and return `PERM_DENIED`. | +| Chromium apps have empty trees | Medium | Medium | Chrome 138+ has native UIA. Electron apps need `--force-renderer-accessibility`. Version-aware detection. | +| Windows CI flakiness (GUI tests) | Medium | Low | `--test-threads=1`, use Notepad (not Calculator), explicit process cleanup. | +| Binary size exceeds 15MB with Windows deps | Low | Low | `windows` crate features are granular. Only enable needed features. | +| Cross-compile from macOS fails | Low | Medium | Use `x86_64-pc-windows-gnu` for `cargo check`. Full build on CI only. | +| UIPI blocks `SendInput` to elevated apps | High | Medium | Check return value, detect elevation, return `PERM_DENIED` with "run as administrator" guidance. | +| Hung/frozen app blocks UIA call indefinitely | Medium | High | Wrap UIA calls in thread+channel timeout (5s default). Return `TIMEOUT` error. | +| Cloaked windows pollute `list_windows` | Medium | Low | Filter via `DwmGetWindowAttribute(DWMWA_CLOAKED)`. | +| ApplicationFrameHost.exe PID mismatch | Medium | Medium | Detect UWP host process, resolve real app PID. Document limitation. | + +## Testing Strategy + +### Unit Tests (`crates/windows/`) + +All unit tests use `#[cfg(target_os = "windows")]` guards — they only run on Windows CI. + +- `tree/roles.rs` — ControlType → role mapping coverage (every mapped type, including new additions) +- `tree/builder.rs` — Tree depth limiting, cycle prevention +- `input/keyboard.rs` — Cmd→Ctrl modifier mapping, virtual key code mapping, blocked combo filtering +- `input/clipboard.rs` — Clipboard roundtrip (get/set/clear) with RAII guard verification +- `actions/dispatch.rs` — Action → Pattern mapping coverage +- `system/permissions.rs` — Permission check returns valid status +- `error_mapping.rs` — HRESULT → ErrorCode mapping for all known values (created in Phase 0, tested from Phase 0) + +### Integration Tests (`tests/integration/`) + +Run on `windows-latest` CI only: + +- Snapshot Explorer — non-empty tree with refs +- Snapshot Notepad — textfield with editable value +- Snapshot Settings — valid tree for modern Windows app +- Click button in test app — verify action succeeded +- Type text into Notepad — verify content changed +- Clipboard get/set roundtrip +- Launch/close Notepad lifecycle +- List windows — at least one window returned +- List apps — at least one app returned +- Screenshot — returns non-empty base64 PNG +- Window operations (minimize, maximize, restore) on Notepad +- Snapshot VS Code — Electron app, validates Chromium edge case detection and large tree performance + +### Research Insights: Testing + +**CI test targets:** +| App | Why | Notes | +|-----|-----|-------| +| `notepad.exe` | Guaranteed available, simple UI, has textfield | Perfect for type/set-value/click tests | +| `explorer.exe` | Always running | Good for list-windows, snapshot | +| `mspaint.exe` | Available, simple UI | Good for click, toolbar tests | +| `code.exe` (VS Code) | Electron app, most common dev tool | Validates Chromium detection, large tree handling. **Mark optional on CI** — may not be pre-installed | +| `calc.exe` | UWP app | **May not be available on CI — skip or make optional** | + +**Use `--test-threads=1`.** Concurrent integration tests fighting over foreground windows and focus cause flakiness. + +**Process cleanup.** Always `Stop-Process -Name ... -ErrorAction SilentlyContinue` after tests. Test failures that leave processes running will contaminate subsequent test runs. + +--- + +### Cross-Platform Tests + +- JSON schema validation — same golden fixtures pass on both platforms +- Error format validation — error JSON structure matches on both platforms +- Command flag parsing — CLI argument handling is platform-independent (already in core tests) + +## File Manifest + +### Files Modified (Phase 0 Only) + +| File | Change | +|---|---| +| `crates/core/src/adapter.rs` | Add `blocked_combos` method to `PlatformAdapter` | +| `crates/core/src/commands/press.rs` | Use `adapter.blocked_combos()` instead of const | +| `crates/core/src/commands/key_down.rs` | Add blocked combo check | +| `crates/core/src/commands/key_up.rs` | Add blocked combo check | +| `crates/core/src/error.rs` | Remove dead `permission_denied()` | +| `crates/core/src/node.rs` | Change `pid: i32` → `pid: u64` in `WindowInfo` and `AppInfo` | +| `crates/macos/src/adapter.rs` | Override `blocked_combos()` with macOS list; `as u64` PID casts | +| `.github/workflows/ci.yml` | Add `test-windows` job | +| `crates/windows/Cargo.toml` | Add dependencies | + +### Files Created (Phase 0) + +| File | Purpose | Est. LOC | +|---|---|---| +| `crates/windows/src/error_mapping.rs` | HRESULT/UIA error → `AdapterError` conversion | ~60 | + +### Files Created (Phases 1-5) + +| File | Purpose | Est. LOC | +|---|---|---| +| `crates/windows/src/lib.rs` | Module declarations, re-export `WindowsAdapter` | ~30 | +| `crates/windows/src/adapter.rs` | `PlatformAdapter` impl, delegation to submodules | ~200 | +| `crates/windows/src/tree/mod.rs` | Re-exports | ~10 | +| `crates/windows/src/tree/element.rs` | `UIElement` wrapper, attribute readers | ~150 | +| `crates/windows/src/tree/builder.rs` | `build_subtree()` via `UITreeWalker` + `UICacheRequest` | ~200 | +| `crates/windows/src/tree/roles.rs` | `ControlType` → role string mapping (24 types) | ~140 | +| `crates/windows/src/tree/resolve.rs` | Element re-identification via `UIMatcher` + `RuntimeId` | ~120 | +| `crates/windows/src/tree/surfaces.rs` | Surface detection (menus, dialogs; Sheet/Popover → error) | ~120 | +| `crates/windows/src/actions/mod.rs` | Re-exports | ~10 | +| `crates/windows/src/actions/dispatch.rs` | `Action` → UIA Pattern dispatch with fallback chain | ~220 | +| `crates/windows/src/actions/activate.rs` | Window/element activation chain | ~100 | +| `crates/windows/src/actions/extras.rs` | Selection, Scroll, Toggle pattern helpers | ~150 | +| `crates/windows/src/input/mod.rs` | Re-exports | ~10 | +| `crates/windows/src/input/keyboard.rs` | `SendInput` keyboard + Cmd→Ctrl + batching | ~220 | +| `crates/windows/src/input/mouse.rs` | `SendInput` mouse + DPI-aware coordinate translation | ~170 | +| `crates/windows/src/input/clipboard.rs` | Win32 clipboard with RAII `ClipboardGuard` + retry | ~140 | +| `crates/windows/src/system/mod.rs` | Re-exports | ~10 | +| `crates/windows/src/system/app_ops.rs` | `CreateProcessW`, sanitized `ShellExecuteW`, `TerminateProcess` | ~170 | +| `crates/windows/src/system/window_ops.rs` | `EnumWindows` (cloaked filter), `ShowWindow`, `SetWindowPos` | ~220 | +| `crates/windows/src/system/key_dispatch.rs` | App-targeted key press via `SendInput` | ~80 | +| `crates/windows/src/system/permissions.rs` | COM smoke test + UIPI advisory | ~70 | +| `crates/windows/src/system/screenshot.rs` | `PrintWindow(PW_RENDERFULLCONTENT)` + `BitBlt` fallback | ~220 | +| `crates/windows/src/system/wait.rs` | `WaitForInputIdle`, polling loops | ~100 | +| **Total** | | **~3,050** | + +### Research Insights: LOC Budget + +**Realistic budget: 3,500-4,000 LOC.** The per-file estimates above total ~3,050 for implementation code. Add: +- cfg-gate stubs: ~15-20 LOC per file × ~20 files = ~350 LOC +- Frozen app detection (`IsHungAppWindow`, `IUIAutomation2` timeout config): ~80 LOC +- UWP launch via `IApplicationActivationManager`: ~60 LOC +- Protected process detection: ~40 LOC +- Chromium version detection via `GetFileVersionInfoW`: ~60 LOC +- DPI coordinate helpers: ~40 LOC +- Unit tests within modules: ~200+ LOC + +**Total budget: ~3,500-4,000 LOC.** Well within the 400 LOC per file limit even with additions. + +**New file: `error_mapping.rs`.** Consolidates all HRESULT→ErrorCode and UIA error→AdapterError conversion in one place. Prevents duplicating error mapping logic across modules. + +--- + +## Documentation Plan + +- Update `CLAUDE.md` to reflect 22+ trait methods (currently says 12) +- Update `CLAUDE.md` PlatformAdapter example to show all 22 methods +- Update `docs/phases.md` to mark Phase 2 status as "In Progress" +- Add Windows-specific troubleshooting section to README +- Document Chromium `--force-renderer-accessibility` guidance (version-aware) +- Document UIPI limitations and "run as administrator" guidance +- Document `SnapshotSurface::Sheet`/`Popover` as macOS-only in command docs + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md](docs/brainstorms/2026-02-25-windows-adapter-phase2-brainstorm.md) — Key decisions carried forward: layered `uiautomation` + `windows` crate choice, Cmd→Ctrl mapping, platform-specific blocked combos, stateless per-call COM strategy + +### Internal References + +- PlatformAdapter trait: `crates/core/src/adapter.rs:114-216` +- macOS adapter pattern: `crates/macos/src/adapter.rs` +- BLOCKED_COMBOS: `crates/core/src/commands/press.rs:8-14` +- NativeHandle: `crates/core/src/adapter.rs:58-91` +- CI workflow: `.github/workflows/ci.yml` +- Dead code: `crates/core/src/error.rs:107-115` (`permission_denied()`) + +### External References + +- PRD v2.0 Section 7.2: Windows Adapter specification +- `uiautomation` crate: https://crates.io/crates/uiautomation (v0.24.3) +- `windows` crate: https://crates.io/crates/windows (v0.62+) +- Windows UIA documentation: https://learn.microsoft.com/en-us/windows/win32/winauto/entry-uiauto-win32 +- UIA Control Types: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-controltype-ids +- UIA Control Patterns: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-controlpattern-ids +- UIA Caching: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-cachingforclients +- UIA Threading: https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-threading +- Chrome 138+ Native UIA: https://developer.chrome.com/blog/windows-uia-support-update +- UIPI: https://learn.microsoft.com/en-us/troubleshoot/power-platform/power-automate/desktop-flows/ui-automation/uipi-issues +- SendInput: https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-sendinput +- UIA Security Research (Akamai, Dec 2024): https://www.akamai.com/blog/security-research/2024/dec/2024-december-windows-ui-automation-attack-technique-evades-edr +- `uiautomation-rs` GitHub: https://github.com/leexgone/uiautomation-rs +- `windows-rs` GitHub: https://github.com/microsoft/windows-rs diff --git a/docs/plans/2026-02-27-feat-notification-management-macos-plan.md b/docs/plans/2026-02-27-feat-notification-management-macos-plan.md new file mode 100644 index 0000000..24a2656 --- /dev/null +++ b/docs/plans/2026-02-27-feat-notification-management-macos-plan.md @@ -0,0 +1,991 @@ +--- +title: "feat: add notification management commands (macOS)" +type: feat +status: completed +date: 2026-02-27 +deepened: 2026-02-27 +origin: docs/brainstorms/2026-02-27-notification-management-brainstorm.md +--- + +# feat: add notification management commands (macOS) + +## Enhancement Summary + +**Deepened on:** 2026-02-27 +**Sections enhanced:** 7 (all major sections) +**Review agents used:** architecture-strategist, performance-oracle, security-sentinel, code-simplicity-reviewer, pattern-recognition-specialist, agent-native-reviewer, best-practices-researcher + +### Key Improvements + +1. **Dropped NSDistributedNotificationCenter observer** — Polling-only for v1. Observer adds thread-safety risks (HIGH), undocumented API surface, and complexity for marginal latency gain. The reliable path is AX polling. +2. **Reduced adapter surface from 5 methods to 3** — `list_notifications`, `dismiss_notification`, `notification_action`. Dismiss-all composes on `list_notifications`. Wait composes on `list_notifications` polling. +3. **Dropped `NotificationUnsupported` error code** — Reuse existing `PlatformNotSupported` (pattern-recognition, architecture-strategist). +4. **Added content-based verification for dismiss/interact** — TOCTOU mitigation: verify `app_name` + `title` match before acting on a positional index (agent-native-reviewer CRITICAL). +5. **Replaced AppleScript NC open with pure AX** — Eliminates shell injection risk from existing AppleScript pattern (security-sentinel HIGH). +6. **Added index >= 1 validation** — Index 0 with 1-based-to-0-based conversion causes `usize` underflow → panic (security-sentinel MEDIUM). +7. **Deferred v1 scope**: focus mode detection, inline action UI, auto-expansion of collapsed groups, `observer.rs` — all deferred to v2 to keep initial implementation minimal (~315 LOC reduction). + +### New Considerations Discovered + +- macOS Sequoia (15) added extra AXGroup nesting level in NC AX tree — heuristic parser must handle both Sonoma and Sequoia layouts +- NC close buttons only appear on hover in Sequoia — mouse hover synthesis required before dismiss +- Notification text may be `NSConcreteAttributedString` in Sequoia — need to handle both plain and attributed string extraction +- `typed batch path` is already at 472 LOC (over 400 limit) — adding 4 commands requires splitting first + +--- + +## Overview + +Add 4 new commands + 1 wait extension enabling AI agents to read, dismiss, and interact with macOS notifications via the Notification Center accessibility tree. This is the first cross-platform notification feature — macOS ships first, Windows/Linux implementations follow in their respective phases. + +The implementation follows the existing extensibility pattern: new domain types in core → new adapter trait methods with `not_supported` defaults → command handlers → CLI wiring → macOS-specific implementation in a new `notifications/` subfolder. + +## Problem Statement + +agent-desktop's 50-command surface covers app lifecycle, UI interaction, clipboard, screenshots, and keyboard/mouse — but has zero visibility into OS-level notifications. Agents automating desktop workflows frequently need to: + +- Detect when a notification arrives (e.g., download complete, message received) +- Read notification content to decide next steps +- Dismiss notifications to clear clutter +- Click notification action buttons (Reply, Open, Join, etc.) + +Without notification support, agents must resort to screenshot → OCR workflows or ignore notifications entirely. + +## Proposed Solution + +**Approach: AX tree traversal with polling-based wait** (see brainstorm: `docs/brainstorms/2026-02-27-notification-management-brainstorm.md`) + +- **CRUD operations** (list, dismiss, dismiss-all, notification-action): Programmatically open Notification Center, traverse its AX tree, perform actions, close NC — all within a single RAII-guarded session per command invocation +- **Wait operation** (`wait --notification`): AX polling at configurable intervals (default 3 seconds), opening/closing NC per cycle + +### Research Insights — Solution Design + +**Architecture (architecture-strategist):** +- Keep CLI wiring incremental with each phase rather than a separate Phase 6. Each phase should produce a testable `cargo build` — wire CLI as you go. +- `list_notifications` return type must be consistent between trait signature and macOS impl. V1 returns `Vec<NotificationInfo>` from the trait; focus mode metadata is macOS-specific and returned via a wrapper type in the macOS crate, not the trait. + +**Simplicity (code-simplicity-reviewer):** +- 3 adapter methods, not 5. `dismiss_all` composes on `list_notifications` in the command handler. `wait_for_notification` composes on `list_notifications` polling in the command handler. +- Merge `dismiss.rs` + `interact.rs` into a single `actions.rs` (~150 LOC combined, well under 400 limit). +- Drop `observer.rs` entirely from v1. + +### Alternative Approaches Considered + +| Approach | Why Rejected | +|----------|-------------| +| NSDistributedNotificationCenter for wait | Thread-safety risks (HIGH — security-sentinel), undocumented API surface, not all apps broadcast. Polling is the reliable path. Observer adds complexity for marginal latency gain. Deferred to v2 if polling proves insufficient. | +| Pure AX polling for wait (original brainstorm) | Adopted as the v1 approach after security/simplicity review. 3-second default interval, configurable via `--poll-interval`. | +| SQLite notification database | Requires Full Disk Access permission — unacceptable adoption barrier on top of existing Accessibility permission. Database schema is undocumented and changes across macOS versions. | +| Banner-only capture | Banners are transient (~5s display). Requires separate AX target. NC contents are sufficient for v1. | + +## Technical Approach + +### Architecture + +``` +┌──────────────────────────────────────────────────────┐ +│ Binary crate (src/) │ +│ ┌─────────┐ ┌───────────┐ ┌──────────────────┐ │ +│ │ cli.rs │→ │cli_args.rs│→ │ dispatch.rs │ │ +│ │ 4 new │ │ 4 new arg │ │ 4 new match arms │ │ +│ │ variants│ │ structs │ │ + wait extension │ │ +│ └─────────┘ └───────────┘ └────────┬─────────┘ │ +│ │ │ +├───────────────────────────────────────┼──────────────┤ +│ Core crate (crates/core/) │ │ +│ ┌──────────────┐ ┌──────────────┐ │ │ +│ │notification.rs│ │ error.rs │ │ │ +│ │NotifInfo │ │+1 error code │ │ │ +│ │NotifFilter │ └──────────────┘ │ │ +│ └──────────────┘ │ │ +│ ┌─────────────┐ ┌─────────────┐ │ │ +│ │ adapter.rs │ │ commands/ │ │ │ +│ │ +3 methods │ │ 4 new files │◄────┘ │ +│ │ (defaults) │ └─────────────┘ │ +│ └─────────────┘ │ +├──────────────────────────────────────────────────────┤ +│ macOS crate (crates/macos/) │ +│ ┌────────────┐ ┌──────────────────────────────┐ │ +│ │ adapter.rs │→ │ notifications/ (NEW) │ │ +│ │ +3 impls │ │ ├── mod.rs │ │ +│ │ │ │ ├── nc_session.rs (RAII) │ │ +│ │ │ │ ├── list.rs │ │ +│ │ │ │ └── actions.rs │ │ +│ └────────────┘ └──────────────────────────────┘ │ +└──────────────────────────────────────────────────────┘ +``` + +### Research Insights — Architecture + +**Naming conventions (pattern-recognition-specialist):** +- `dismiss-all-notifications` has 3 kebab segments — breaks the existing 2-segment convention (`list-windows`, `focus-window`). Consider `clear-notifications` instead. +- `notification-action` breaks the `{verb}-{noun}` pattern. Consider `click-notification` (verb-noun) with the action name as a required argument. +- Decision: Keep `list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action` as-is from the brainstorm. The 3-segment name for dismiss-all is acceptable because it disambiguates from dismiss-single. `notification-action` is acceptable because the action name is a positional arg. + +**macOS version compatibility (best-practices-researcher):** +- NC process name: `NotificationCenter` (not `notificationcenterui`) +- NC is opened via SystemUIServer menu bar click (AX click on clock area), NOT directly via the NC process +- Sequoia (macOS 15) added an extra AXGroup nesting level — parser must handle variable nesting depth +- Close buttons only appear on hover in Sequoia — must synthesize mouse hover before looking for dismiss button +- Notification body text may be `NSConcreteAttributedString` — use `AXValueAttribute` with string coercion fallback + +### Implementation Phases + +#### Phase 1: Core Types and Adapter Trait + CLI Wiring + +Establish the platform-agnostic foundation with CLI wiring. Produces a compilable binary where notification commands return `PLATFORM_NOT_SUPPORTED` on all platforms. + +**Tasks:** + +- [x] Create `crates/core/src/notification.rs` — `NotificationInfo` and `NotificationFilter` structs +- [x] Add 3 new methods to `PlatformAdapter` trait in `crates/core/src/adapter.rs` with `not_supported` defaults +- [x] Add 1 error code variant (`NotificationNotFound`) to `ErrorCode` enum in `crates/core/src/error.rs` + convenience constructor +- [x] Register module in `crates/core/src/lib.rs` +- [x] Create 4 command handlers in `crates/core/src/commands/` +- [x] Register 4 new modules in `crates/core/src/commands/mod.rs` +- [x] Add 4 command variants to `src/cli.rs` + `name()` arms +- [x] Add 4 arg structs to `src/cli_args.rs` +- [x] Add 4 match arms to `src/dispatch.rs` + extend Wait arm for `--notification` +- [x] Add 4 command routing arms to `src/typed batch path` +- [x] Add `--notification` flag + `--poll-interval` to `WaitArgs` + +**New types in `crates/core/src/notification.rs`:** + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotificationInfo { + pub index: usize, + pub app_name: String, + pub title: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub body: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + pub timestamp: Option<String>, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub actions: Vec<String>, +} + +#[derive(Debug, Clone, Default)] +pub struct NotificationFilter { + pub app: Option<String>, + pub text: Option<String>, + pub limit: Option<usize>, +} +``` + +Design notes: +- 6 fields on `NotificationInfo` (under 7-field limit per CLAUDE.md) +- `is_persistent` dropped — not reliably detectable from AX, no agent use case (see brainstorm) +- `timestamp` is ISO 8601 string when absolute time is extractable from AX, `None` when only relative time is available +- `NotificationFilter` derives `Default` and `Clone` (pattern-recognition fix: missing derives) +- `focus_mode` is NOT in `NotificationInfo` or the trait return type — it's macOS-specific metadata added by the macOS command layer + +**New adapter methods in `crates/core/src/adapter.rs`:** + +```rust +fn list_notifications(&self, _filter: &NotificationFilter) -> Result<Vec<NotificationInfo>, AdapterError> { + Err(AdapterError::not_supported("list_notifications")) +} + +fn dismiss_notification(&self, _index: usize, _app_filter: Option<&str>) -> Result<NotificationInfo, AdapterError> { + Err(AdapterError::not_supported("dismiss_notification")) +} + +fn notification_action(&self, _index: usize, _action_name: &str) -> Result<ActionResult, AdapterError> { + Err(AdapterError::not_supported("notification_action")) +} +``` + +Design notes: +- **3 methods** (reduced from 5). `dismiss_all_notifications` and `wait_for_notification` compose on `list_notifications` in command handlers — no separate adapter methods needed. +- `dismiss_notification` returns `NotificationInfo` of the dismissed notification for agent verification +- Reuse existing `ActionResult` for `notification_action` (code-simplicity: no new result type needed) + +**New error code in `crates/core/src/error.rs`:** + +```rust +// Add to ErrorCode enum: +NotificationNotFound, + +// Add to as_str(): +Self::NotificationNotFound => "NOTIFICATION_NOT_FOUND", + +// Add convenience constructor to AdapterError: +pub fn notification_not_found(index: usize) -> Self { + Self::new( + ErrorCode::NotificationNotFound, + format!("Notification at index {index} not found"), + ) + .with_suggestion("Notification may have been dismissed or expired. Run 'list-notifications' to see current notifications") +} +``` + +Design notes (architecture-strategist, pattern-recognition, code-simplicity): +- **Only 1 new error code**, not 2. `NotificationUnsupported` is dropped — the existing `PlatformNotSupported` (returned by default `not_supported()` implementations) already covers this case identically. +- This matches the existing pattern: `clipboard_get` doesn't have a `ClipboardUnsupported` error, it just returns `PlatformNotSupported` on unsupported platforms. + +### Research Insights — Error Handling + +**Index validation (security-sentinel MEDIUM):** +- Index 0 with 1-based-to-0-based conversion (`index - 1`) causes `usize` underflow → `panic!` (since `panic = "abort"` in release profile, this is a process kill) +- **Must validate `index >= 1`** at the argument parsing level (clap `value_parser` with range) or in the command handler before subtraction +- Recommended: Add `#[arg(value_parser = clap::value_parser!(usize).range(1..))]` to all index args + +**CLI args (`src/cli_args.rs`):** + +```rust +#[derive(Parser, Debug)] +pub struct ListNotificationsArgs { + #[arg(long, help = "Filter to notifications from this app")] + pub app: Option<String>, + #[arg(long, help = "Filter to notifications containing this text")] + pub text: Option<String>, + #[arg(long, help = "Maximum number of notifications to return")] + pub limit: Option<usize>, +} + +#[derive(Parser, Debug)] +pub struct DismissNotificationArgs { + #[arg(value_name = "INDEX", help = "1-based notification index from list-notifications", + value_parser = clap::value_parser!(usize).range(1..))] + pub index: usize, + #[arg(long, help = "Filter notifications by app before selecting index")] + pub app: Option<String>, +} + +#[derive(Parser, Debug)] +pub struct DismissAllNotificationsArgs { + #[arg(long, help = "Only dismiss notifications from this app")] + pub app: Option<String>, +} + +#[derive(Parser, Debug)] +pub struct NotificationActionArgs { + #[arg(value_name = "INDEX", help = "1-based notification index from list-notifications", + value_parser = clap::value_parser!(usize).range(1..))] + pub index: usize, + #[arg(value_name = "ACTION", help = "Name of the action button to click (e.g., Reply, Open)")] + pub action: String, +} +``` + +Design notes: +- `DismissNotificationArgs` now includes `--app` filter (pattern-recognition fix: missing from original plan) +- Both index args use `value_parser` with `range(1..)` to reject index 0 at parse time (security-sentinel fix) + +**Wait args extension (`src/cli_args.rs`):** + +Add to existing `WaitArgs`: +```rust +#[arg(long, help = "Wait for a notification to appear")] +pub notification: bool, +#[arg(long, help = "Poll interval in ms for notification wait (default: 3000)", default_value = "3000")] +pub poll_interval: Option<u64>, +``` + +**Command handlers (`crates/core/src/commands/`):** + +Each handler follows Pattern C (args struct → execute → json). Example for `list_notifications.rs`: + +```rust +pub fn execute(args: ListNotificationsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> { + let filter = NotificationFilter { + app: args.app, + text: args.text, + limit: args.limit, + }; + let notifications = adapter.list_notifications(&filter)?; + Ok(json!({ + "count": notifications.len(), + "notifications": notifications, + })) +} +``` + +**`dismiss_all_notifications.rs` composes on list:** + +```rust +pub fn execute(args: DismissAllNotificationsArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> { + // List all notifications matching filter, then dismiss each + let filter = NotificationFilter { app: args.app.clone(), ..Default::default() }; + let notifications = adapter.list_notifications(&filter)?; + let mut dismissed = 0; + // Dismiss in reverse order (highest index first) to avoid index shifting + for notif in notifications.iter().rev() { + match adapter.dismiss_notification(notif.index, args.app.as_deref()) { + Ok(_) => dismissed += 1, + Err(_) => {} // notification may have already been dismissed + } + } + Ok(json!({ + "dismissed_count": dismissed, + })) +} +``` + +**Success criteria:** +- `cargo test --lib -p agent-desktop-core` passes +- `cargo clippy --all-targets -- -D warnings` clean +- All 4 commands parse and dispatch correctly +- `wait --notification` flag is recognized +- `--notification` is mutually exclusive with `--element`/`--window` +- Windows/Linux stubs compile without changes (default `not_supported` implementations) +- Index 0 rejected at parse time with clear error message + +--- + +#### Phase 2: NC Session Guard (macOS) + +The critical safety layer. Every notification command operates within a `NcSession` that guarantees Notification Center is opened before work and closed after — even on errors or panics. + +**Tasks:** + +- [x] Create `crates/macos/src/notifications/mod.rs` — module re-exports +- [x] Create `crates/macos/src/notifications/nc_session.rs` — RAII NC lifecycle guard +- [x] Register `notifications` module in `crates/macos/src/lib.rs` + +**`NcSession` design (`crates/macos/src/notifications/nc_session.rs`):** + +```rust +pub struct NcSession { + was_already_open: bool, +} + +impl NcSession { + pub fn open() -> Result<Self, AdapterError> { + let was_already_open = is_nc_open()?; + if !was_already_open { + open_nc()?; + wait_for_nc_ready()?; // wait for animation + AX tree population + } + Ok(Self { was_already_open }) + } + + /// Explicit close with error reporting. Prefer this over relying on Drop. + pub fn close(self) -> Result<(), AdapterError> { + if !self.was_already_open { + close_nc()?; + } + std::mem::forget(self); // prevent Drop from double-closing + Ok(()) + } +} + +impl Drop for NcSession { + fn drop(&mut self) { + if !self.was_already_open { + // Fire-and-forget close — log error but don't propagate + if let Err(e) = close_nc() { + tracing::warn!("Failed to close NC in Drop: {e}"); + } + } + } +} +``` + +### Research Insights — NC Session + +**Explicit close + Drop fallback (best-practices-researcher, performance-oracle):** +- Provide `close(self)` for the happy path where callers want error feedback +- Drop handles panic/error paths as best-effort fallback +- Use `std::mem::forget(self)` in `close()` to prevent double-close from Drop +- Alternative: `scopeguard` crate — but adds a dependency for a pattern that's straightforward to implement manually + +**Pure AX for NC open (security-sentinel HIGH, performance-oracle):** + +```rust +fn open_nc() -> Result<(), AdapterError> { + // 1. Find SystemUIServer process + // 2. Get its AXApplication element + // 3. Find the menu bar (AXMenuBar) + // 4. Find the clock/date menu bar item + // 5. AXPress to toggle NC open + // NO AppleScript — eliminates shell injection attack surface +} +``` + +The existing `app_ops.rs:172-192` uses `osascript -e` with string interpolation — this is a known injection vector (security-sentinel). NC open must NOT replicate this pattern. Pure AX click on the SystemUIServer menu bar item is both safer and faster (~50ms vs ~200ms for AppleScript). + +**NC state detection (best-practices-researcher):** +- Check `com.apple.notificationcenterui` process for visible AXWindow +- Do NOT use toggle — toggling inverts state if NC is already open +- NC process name is `NotificationCenter` on modern macOS, bundle ID is `com.apple.notificationcenterui` + +**NC close mechanism (performance-oracle):** +- Send Escape key via `CGEventCreateKeyboardEvent` (fastest, most reliable) +- Fire-and-forget variant in Drop: dispatch close without waiting for confirmation +- Blocking variant in `close()`: wait up to 500ms for NC window to disappear + +**NC ready wait (performance-oracle):** +- After open, poll for AX tree population (notification children exist) +- Max wait: 2 seconds with 50ms polling interval +- NC animation is typically 200-400ms + +**Already-open handling:** If NC was already open when `NcSession::open()` is called, the session skips both the open and the close steps. The agent (or user) is responsible for NC in this case. + +**SIGKILL risk (security-sentinel MEDIUM):** +- If the process is killed during a notification operation, NC may remain open +- No mitigation possible — same risk as any system UI interaction +- NC will close when the user clicks elsewhere or presses Escape +- With `panic = "abort"`, Drop does NOT run — NC will be left open on panic +- Acceptable: NC auto-closes on user interaction, and this is an edge case + +**Success criteria:** +- `NcSession::open()` reliably detects whether NC is already open +- NC is always closed on session drop (verified by test that opens, panics, and checks NC state) +- `close(self)` provides error feedback without double-close +- Animation timing is handled (no empty traversals due to reading mid-animation) +- No AppleScript anywhere in the notification module + +--- + +#### Phase 3: List Implementation (macOS) + +The core traversal that all other notification commands depend on. + +**Tasks:** + +- [x] Create `crates/macos/src/notifications/list.rs` — AX tree traversal of NC +- [x] Wire `list_notifications` in macOS adapter + +**`list.rs` implementation:** + +The function receives an open `NcSession` reference (to ensure NC is open) and a `NotificationFilter`: + +```rust +pub fn list_notifications(filter: &NotificationFilter) -> Result<Vec<NotificationInfo>, AdapterError> { + let session = NcSession::open()?; + let notifications = list_from_nc(&session, filter)?; + session.close()?; // explicit close with error reporting + Ok(notifications) +} +``` + +### Research Insights — AX Tree Traversal + +**Heuristic matching strategy (best-practices-researcher):** + +The NC AX tree structure varies by macOS version. Use role + subrole heuristics: + +1. Find NC process by bundle ID `com.apple.notificationcenterui` +2. Get application AX element via `AXUIElementCreateApplication(pid)` +3. Find main NC window (AXWindow or AXSheet child) +4. Traverse children looking for notification elements: + - **Sonoma (14):** Notifications are AXGroup with subrole `AXNotificationCenterAlert` or within `AXNotificationCenterAlertStack` + - **Sequoia (15):** Extra AXGroup nesting level — notifications may be nested one level deeper + - Match by role pattern: AXGroup containing AXStaticText children + - Notification titles: AXStaticText with `AXValue` attribute + - Action buttons: AXButton children within the notification group +5. Build flat list ordered by tree position (newest first in NC) +6. Apply filters during traversal, not after (performance-oracle: avoid building full list then filtering) + - `--app`: case-insensitive substring match + - `--text`: substring match on title+body + - `--limit`: stop traversal early when limit reached + +**Performance (performance-oracle):** +- Use `AXUIElementCopyMultipleAttributeValues` for batch attribute fetch (3-5x faster than individual calls) +- Apply `--app` filter during traversal to skip non-matching groups entirely +- If `--limit` is set, stop traversal once limit is reached + +**Grouped notification handling — DEFERRED to v2 (code-simplicity-reviewer):** + +macOS groups notifications by app when count > 1. V1 behavior: +- If a group is collapsed, list the group header only (showing app name + count badge text) +- Do NOT auto-expand groups — this adds complexity, timing issues, and visual disruption +- Include a `"grouped": true` hint on collapsed group entries so agents know to use `--app` filter for details +- V2 can add `--expand` flag to auto-expand groups before traversal + +**Timestamp extraction:** + +macOS NC shows relative timestamps ("2m ago"). The AX tree may expose: +- `AXValue` on the timestamp element → relative string +- `AXDescription` or custom attributes → sometimes absolute time + +Strategy: extract whatever is available. If an absolute time can be parsed, return ISO 8601. If only relative, return `None` for `timestamp`. + +**Focus mode / DND detection — DEFERRED to v2 (code-simplicity-reviewer):** + +Reading focus mode requires CoreFoundation preferences API which adds complexity. V1 returns notifications only. V2 can add `focus_mode` metadata. + +**NSConcreteAttributedString handling (best-practices-researcher):** + +On Sequoia, notification body text may be `NSConcreteAttributedString` instead of plain NSString. Use `AXValueAttribute` with string coercion: +```rust +// Try AXValue as string first, then AXTitle, then AXDescription +// If the value is an attributed string, CFStringGetCString will still extract the text content +``` + +**Success criteria:** +- Returns correct flat list for NC with 0, 1, 5, 20+ notifications +- Filters work correctly (--app, --text, --limit) +- Returns empty list (not error) when NC has zero notifications +- Handles both Sonoma and Sequoia AX tree layouts +- Uses batch attribute fetch for performance +- Stops traversal early when --limit reached + +--- + +#### Phase 4: Dismiss and Interact Implementation (macOS) + +**Tasks:** + +- [x] Create `crates/macos/src/notifications/actions.rs` — dismiss + interact combined +- [x] Handle hover-to-reveal close button (Sequoia behavior) +- [x] Wire `dismiss_notification` and `notification_action` in macOS adapter + +### Research Insights — TOCTOU Mitigation + +**Content-based verification (agent-native-reviewer CRITICAL):** + +Positional indices are inherently racy — a notification may arrive or disappear between `list-notifications` and `dismiss-notification`. Within a single command invocation, the index is stable (NC is open and frozen). But across separate invocations, the index may point to a different notification. + +**Mitigation: Verify before acting.** When the agent provides an index from a previous `list-notifications` call: +1. Open NC session +2. Re-list notifications to get current state +3. Find notification at the given index +4. Verify that `app_name` and `title` match what the agent expects (passed via optional `--verify-app` and `--verify-title` args, or by the command handler comparing against what list returned) +5. If mismatch → return `NOTIFICATION_NOT_FOUND` with suggestion to re-list +6. If match → perform the action + +**V1 approach:** Within a single command (dismiss/interact), the tool lists and acts in the same NC session. The index is consistent within that session. The TOCTOU risk is only when the agent uses an index from a **previous** `list-notifications` call — but since agents should always list-then-act in quick succession, this is acceptable for v1. Content verification can be added in v2 via optional verify args. + +**`actions.rs` — single notification dismiss:** + +```rust +pub fn dismiss_notification(index: usize, app_filter: Option<&str>) -> Result<NotificationInfo, AdapterError> { + let session = NcSession::open()?; + let notifications = list_from_nc(&session, &build_filter(app_filter))?; + let target = notifications.get(index - 1) // 1-based to 0-based (index >= 1 validated by clap) + .ok_or_else(|| AdapterError::notification_not_found(index))?; + let info = target.clone(); + + // Hover over notification to reveal close button (Sequoia) + hover_over_element(&target.ax_handle)?; + std::thread::sleep(std::time::Duration::from_millis(200)); + + // Find and click the close/dismiss button + perform_dismiss_on_element(&target.ax_handle)?; + session.close()?; + Ok(info) +} +``` + +**"Clear All" hover requirement (best-practices-researcher, SpecFlow Gap 11):** + +On macOS Sonoma/Sequoia, "Clear" and "Clear All" buttons appear only on hover. The implementation must: +1. Find the notification group header AX element +2. Get its bounds via `kAXPositionAttribute` + `kAXSizeAttribute` +3. Synthesize a `CGEventCreateMouseEvent` mouseMoved to the center of those bounds +4. Wait 200ms for the button to appear +5. Re-traverse to find the now-visible "Clear" / "Clear All" button +6. AXPress the button + +**`actions.rs` — notification action:** + +```rust +pub fn notification_action(index: usize, action_name: &str) -> Result<ActionResult, AdapterError> { + let session = NcSession::open()?; + let notifications = list_from_nc(&session, &NotificationFilter::default())?; + let target = notifications.get(index - 1) + .ok_or_else(|| AdapterError::notification_not_found(index))?; + // Find action button matching action_name among the notification's AXButton children + let button = find_action_button(&target.ax_handle, action_name) + .ok_or_else(|| AdapterError::action_failed( + format!("Action '{action_name}' not found on notification {index}"), + ))?; + perform_ax_press(&button)?; + session.close()?; + Ok(ActionResult::new(action_name)) +} +``` + +**Inline action UI — DEFERRED to v2 (code-simplicity-reviewer):** + +Some actions (like "Reply") open an inline text field within NC. V1 does not detect or handle this — it just clicks the button and returns success. V2 can add `"inline_ui": true` to the result and allow the agent to type into the field. + +**Success criteria:** +- Single dismiss by index works correctly +- Dismiss-all (via command handler composing list + dismiss) works with and without `--app` +- Hover-to-reveal close button works on Sequoia +- Notification action buttons are correctly discovered and clicked +- Returns `NOTIFICATION_NOT_FOUND` for invalid index +- Returns `ACTION_FAILED` if action button not found by name +- Index 0 is impossible (validated at parse time) + +--- + +#### Phase 5: Wait Implementation (Core + macOS) + +**Tasks:** + +- [x] Extend the `wait` command handler in `crates/core/src/commands/wait.rs` for `--notification` flag +- [x] Implement polling loop using `list_notifications` +- [x] No new macOS files needed — wait composes on existing `list_notifications` + +**Wait architecture (simplified from original plan):** + +``` +wait --notification --app "Slack" --timeout 10000 + │ + ▼ + ┌───────────────────┐ + │ AX polling loop │ + │ every 3s (default) │ + │ or --poll-interval │ + │ │ + │ Each cycle: │ + │ 1. list_notifs() │ + │ 2. compare baseline│ + │ 3. if new → return │ + │ 4. sleep interval │ + └───────────────────┘ +``` + +The wait command lives entirely in core (`wait.rs`), calling `adapter.list_notifications()` in a loop. No separate adapter method needed. No observer. No macOS-specific wait code. + +**Polling strategy:** + +```rust +// In wait.rs, when --notification flag is set: +let filter = NotificationFilter { app: args.app.clone(), text: args.text.clone(), ..Default::default() }; +let baseline = adapter.list_notifications(&filter)?; +let baseline_count = baseline.len(); +let interval = Duration::from_millis(args.poll_interval.unwrap_or(3000)); +let deadline = Instant::now() + Duration::from_millis(args.timeout.unwrap_or(30000)); + +loop { + std::thread::sleep(interval); + if Instant::now() > deadline { + return Err(AppError::from(AdapterError::timeout("notification"))); + } + let current = adapter.list_notifications(&filter)?; + if current.len() > baseline_count { + // New notification arrived — return the newest one + return Ok(json!({ + "condition": "notification", + "matched": true, + "notification": current[0], // newest is index 1, which is current[0] + })); + } +} +``` + +### Research Insights — Wait + +**Performance (performance-oracle):** +- Each poll cycle opens and closes NC (visual flash). 3-second default interval minimizes visual disruption. +- `--poll-interval` allows agents to tune: faster for time-critical workflows, slower for background monitoring. +- NC open/close overhead is ~300-500ms per cycle. With 3s interval, that's ~10-15% overhead — acceptable. + +**`--notification` flag semantics (agent-native-reviewer):** +- `--notification` is mutually exclusive with `--element`, `--window`, `--menu` +- When `--notification` is set, `--app` filters by notification source app (NOT the app to snapshot — different semantics from `--element` mode) +- When `--notification` is set, `--text` filters by notification title/body content +- This semantic overload is documented in CLI help text + +**Success criteria:** +- `wait --notification` blocks until a notification arrives +- `wait --notification --app "Messages"` only matches Messages notifications +- `wait --notification --text "hello"` matches title or body content +- Timeout returns structured `TIMEOUT` error +- `--poll-interval` controls polling frequency +- NC is properly opened/closed each cycle + +--- + +#### Phase 6: Testing + +**Tasks:** + +- [x] Unit tests for `NotificationInfo` serialization (core) +- [x] Unit tests for `NotificationFilter` logic (core) +- [x] Unit tests for new error code (core) +- [ ] MockAdapter tests for notification commands (core) — deferred, no MockAdapter infra yet +- [ ] Integration tests for NC session lifecycle (macOS CI) — requires CI runner with Accessibility +- [ ] Integration tests for list/dismiss/interact (macOS CI) — requires CI runner with Accessibility +- [ ] Golden fixture for NC AX tree structure (tests/fixtures/) — deferred to v2 + +**Unit tests (core — `crates/core/`):** + +```rust +// notification.rs tests +#[test] +fn notification_info_serialization_omits_none_fields() { ... } + +#[test] +fn notification_info_serialization_omits_empty_actions() { ... } + +#[test] +fn notification_filter_default_is_unfiltered() { ... } + +// error.rs tests +#[test] +fn notification_not_found_error_serialization() { ... } + +// commands/ tests (using MockAdapter) +#[test] +fn list_notifications_returns_empty_on_mock() { ... } + +#[test] +fn dismiss_notification_returns_not_supported_on_mock() { ... } + +#[test] +fn dismiss_all_composes_list_and_dismiss() { ... } +``` + +**Integration tests (macOS CI — `tests/integration/`):** + +These require a macOS runner with Accessibility permission: + +```rust +#[test] +fn list_notifications_returns_valid_json() { + // Run: agent-desktop list-notifications + // Verify: valid JSON envelope with "ok": true + // Verify: "notifications" is an array + // Verify: each notification has required fields (index, app_name, title) +} + +#[test] +fn dismiss_notification_invalid_index_returns_error() { + // Run: agent-desktop dismiss-notification 999 + // Verify: NOTIFICATION_NOT_FOUND error +} + +#[test] +fn dismiss_notification_zero_index_rejected() { + // Run: agent-desktop dismiss-notification 0 + // Verify: parse error (exit code 2), not panic +} + +#[test] +fn list_notifications_with_app_filter() { + // Run: agent-desktop list-notifications --app "NonexistentApp" + // Verify: empty notifications array, ok: true +} +``` + +**Golden fixtures:** + +Capture real NC AX tree snapshots and commit to `tests/fixtures/`: +- `notification_center_macos14.json` — Sonoma layout +- `notification_center_macos15.json` — Sequoia layout (with extra nesting) + +### Research Insights — Testing + +**Security testing (security-sentinel):** +- Add test for index 0 rejection (parse error, not panic) +- Add tracing assertions: notification content reads are logged at `info` level for audit trail +- Test concurrent NC access: two simultaneous list calls should not corrupt each other + +**Performance benchmarks (performance-oracle):** +- NC open-to-first-result: target < 1 second +- List 20 notifications: target < 2 seconds +- Single dismiss: target < 1.5 seconds (includes hover delay) + +--- + +## System-Wide Impact + +### Interaction Graph + +1. Agent calls `list-notifications` → dispatch.rs → list_notifications::execute() → adapter.list_notifications() → macOS: NcSession::open() → AX tree traversal → NcSession::close() +2. Agent calls `dismiss-notification 3` → dispatch.rs → dismiss_notification::execute() → adapter.dismiss_notification() → macOS: NcSession::open() → list + find by index → hover → AXPress close button → NcSession::close() +3. Agent calls `wait --notification` → dispatch.rs → wait::execute() → polling loop calling adapter.list_notifications() every N seconds + +No callbacks, middleware, or observers fire beyond the AX system. The only external side effect is the NC open/close animation visible to the user. + +### Error Propagation + +``` +macOS AX API error + → AdapterError (with ErrorCode, message, suggestion, platform_detail) + → AppError::Adapter (via #[from]) + → JSON error envelope (main.rs) + → exit code 1 +``` + +NC close failure in `NcSession::drop()` is logged via `tracing::warn` but does not propagate (Drop cannot return errors). This is acceptable because: +- The primary operation already succeeded or failed with a proper error +- NC will auto-close after user interaction or on next session open + +### State Lifecycle Risks + +- **NC left open on crash:** If the process is killed (SIGKILL) or panics (panic=abort), NC may remain open. No mitigation — same risk as any system UI interaction. NC will close when the user clicks elsewhere. +- **RefMap interaction:** Notification commands do NOT interact with the RefMap. They use their own index scheme. No risk of corrupting snapshot refs. +- **Concurrent access:** Two simultaneous `list-notifications` calls will each open their own NC session. Since NC is a singleton UI, the second open attempt will find NC already open and skip the open step (via `NcSession::was_already_open`). Both traversals will read the same AX tree. This is safe but may produce interleaved results in edge cases. + +### API Surface Parity + +- Notification commands are new — no existing interfaces expose equivalent functionality +- The `snapshot` command will NOT snapshot NC content (NC is a system process, not a user app). The notification commands are the only way to access NC content. +- Batch dispatch (`batch` command) must handle all 4 new commands +- **Note:** `typed batch path` is at 472 LOC (over 400 limit) — must split before adding 4 new arms (pattern-recognition-specialist) + +### Research Insights — Security + +**Notification content sensitivity (security-sentinel CRITICAL):** + +Notifications may contain sensitive data: 2FA codes, private messages, financial alerts, medical reminders. The tool faithfully returns this content — it's the agent's responsibility to handle it appropriately. + +**Mitigations:** +- Add `tracing::info!` logging when notifications are read (audit trail, not prevention) +- Documentation should warn: "Notification content may include sensitive information (2FA codes, private messages). Agents should not log, store, or transmit notification content without user consent." +- RefMap-style file permissions (`0o600`) not needed — notification data is transient, not persisted to disk + +**AppleScript injection (security-sentinel HIGH):** +- NC open uses pure AX, not AppleScript — no injection vector +- Existing `app_ops.rs:172-192` has the vulnerability but is out of scope for this PR +- Flag for separate security fix PR + +--- + +## Acceptance Criteria + +### Functional Requirements + +- [ ] `list-notifications` returns a flat JSON array of notifications from NC +- [ ] `list-notifications --app "Messages"` filters to Messages notifications only +- [ ] `list-notifications --limit 5` caps results at 5 +- [ ] `dismiss-notification 2` dismisses the 2nd notification and returns its info +- [ ] `dismiss-all-notifications` clears all notifications from NC +- [ ] `dismiss-all-notifications --app "Mail"` clears only Mail notifications +- [ ] `notification-action 1 "Reply"` clicks the Reply button on notification 1 +- [ ] `wait --notification --timeout 5000` blocks until a notification arrives or times out +- [ ] `wait --notification --app "Slack"` only matches Slack notifications +- [ ] All commands return valid JSON envelopes matching the existing output contract +- [ ] All commands work when NC starts closed (auto-open) and when NC starts open (skip open) +- [ ] NC is always closed after command completes (verified on success and error paths) +- [ ] Index 0 is rejected at parse time with clear error (not panic) +- [ ] No AppleScript used anywhere in notification module + +### Non-Functional Requirements + +- [ ] NC open-to-first-result latency < 1 second (excluding NC animation) +- [ ] No new dependencies added to core crate (notification types are pure serde structs) +- [ ] Binary size increase < 50KB (notification code is thin AX wrappers) +- [ ] `cargo tree -p agent-desktop-core` still contains zero platform crate names + +### Quality Gates + +- [ ] `cargo clippy --all-targets -- -D warnings` — zero warnings +- [ ] `cargo test --lib --workspace` — all tests pass +- [ ] `cargo fmt --all -- --check` — formatted +- [ ] New commands follow existing patterns exactly (file structure, naming, error handling) +- [ ] No `unwrap()` in non-test code +- [ ] All files under 400 LOC +- [ ] `typed batch path` split before adding new commands (if currently over 400 LOC) + +--- + +## Success Metrics + +- Agents can detect, read, and dismiss notifications in automated workflows +- `list-notifications` on a typical NC (5-15 notifications) completes in < 2 seconds +- `wait --notification` detects new notifications within 5 seconds (one poll cycle + overhead) +- Zero regressions in existing 50-command test suite + +## Dependencies & Prerequisites + +- macOS 14+ (Sonoma) — primary target. macOS 15 (Sequoia) tested with extra nesting handling. Ventura (13) best-effort. +- Accessibility permission already granted (same as all existing commands) +- No new crate dependencies — uses existing `accessibility-sys`, `core-foundation`, `core-graphics` FFI + +## Risk Analysis & Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|------|-----------|--------|------------| +| NC AX tree structure changes across macOS versions | High | High | Heuristic matching by role/subrole patterns (`AXNotificationCenterAlert`, `AXNotificationCenterAlertStack`), not hardcoded paths. Golden fixtures for Sonoma + Sequoia. | +| Sequoia extra nesting level breaks parser | High | High | Variable-depth traversal with recursive heuristic matching. Test on both macOS 14 and 15. | +| "Clear All" / close buttons hidden until hover | High | Medium | Synthesize mouse hover event before looking for button. 200ms wait for button to appear. | +| Collapsed notification groups return headers not individual notifications | Medium | Medium | V1: return group header with `"grouped": true` hint. V2: add `--expand` flag. | +| NC animation timing causes empty/partial reads | Medium | Medium | Wait for AX tree population after open (poll for children, max 2s, 50ms interval). | +| NC left open after error/crash | Low | Medium | RAII NcSession guard with explicit close() + Drop fallback. `panic=abort` means Drop won't run on panic — acceptable risk. | +| 3-second poll interval too slow for wait | Low | Medium | `--poll-interval` flag allows agents to tune. Default is conservative. | +| typed batch path over 400 LOC | High | Low | Split typed batch path before adding notification commands. | + +## Documentation Plan + +- [ ] Update `docs/phases.md` — mark notification commands as implemented for macOS +- [ ] Update `README.md` — add notification commands to command reference table +- [ ] Update `.claude/skills/agent-desktop/` — add notification command documentation +- [ ] Add golden fixtures to `tests/fixtures/` for NC AX tree structure (Sonoma + Sequoia) +- [ ] Add security note to docs: notification content may contain sensitive data + +## V2 Backlog (Deferred from v1) + +Items explicitly deferred during deepening to keep v1 minimal: + +| Item | Source | Rationale for deferral | +|------|--------|----------------------| +| NSDistributedNotificationCenter observer | code-simplicity, security-sentinel | Thread-safety risk, undocumented API. Polling is reliable. | +| Focus mode / DND detection | code-simplicity | Requires CoreFoundation preferences API. Not essential for core notification ops. | +| Inline action UI detection | code-simplicity | "Reply" text field handling is edge case. V1 clicks button and returns success. | +| Auto-expansion of collapsed groups | code-simplicity | Adds timing complexity and visual disruption. V1 returns group headers. | +| Content-based TOCTOU verification | agent-native-reviewer | `--verify-app`/`--verify-title` args for cross-invocation index safety. V1 is safe within single invocation. | +| `total_count` + `has_more` in list response | agent-native-reviewer | Useful for pagination. V1 returns all matching (with --limit). | +| `click-notification` body action (open source app) | agent-native-reviewer | Different from action button click. Needs research on AX default action. | +| `--poll-interval` auto-tuning | performance-oracle | Adaptive interval based on NC change frequency. V1 uses fixed interval. | + +--- + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-02-27-notification-management-brainstorm.md](../brainstorms/2026-02-27-notification-management-brainstorm.md) — Key decisions carried forward: macOS first, AX-based approach, positional index identification, flat list (no grouping), NC-only (no banners), heuristic AX matching + +### Internal References + +- Adapter trait pattern: `crates/core/src/adapter.rs:114-216` +- Domain type pattern (SurfaceInfo): `crates/core/src/node.rs:84-92` +- Command handler pattern: `crates/core/src/commands/clipboard_get.rs`, `list_surfaces.rs` +- Error code pattern: `crates/core/src/error.rs:4-18`, `53-116` +- macOS adapter delegation: `crates/macos/src/adapter.rs` +- cfg-gated imp pattern: `crates/macos/src/input/clipboard.rs` +- Wait command: `crates/core/src/commands/wait.rs:12-55` +- AppleScript pattern (has injection vulnerability): `crates/macos/src/system/app_ops.rs:157-196` +- CLI registration: `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` +- AX-first activation pattern: `docs/brainstorms/2026-02-23-macos-ax-first-robustness-brainstorm.md` + +### Review Agent Sources + +- **architecture-strategist:** Incremental CLI wiring, return type consistency, method count reduction +- **performance-oracle:** Batch AX attribute fetch, early filter application, fire-and-forget close, AX timeout setting +- **security-sentinel:** Index 0 crash, AppleScript injection, notification content sensitivity, observer thread safety +- **code-simplicity-reviewer:** Observer removal, adapter method reduction, v2 backlog items +- **pattern-recognition-specialist:** Missing derives, --app on dismiss, naming conventions, typed batch path LOC +- **agent-native-reviewer:** TOCTOU verification, total_count/has_more, click-notification-body, poll-interval +- **best-practices-researcher:** NC process name, Sequoia nesting, hover-only buttons, attributed strings, RAII patterns + +### Files to Create + +| File | Purpose | Est. LOC | +|------|---------|----------| +| `crates/core/src/notification.rs` | NotificationInfo, NotificationFilter structs | ~40 | +| `crates/core/src/commands/list_notifications.rs` | list-notifications command handler | ~30 | +| `crates/core/src/commands/dismiss_notification.rs` | dismiss-notification command handler | ~30 | +| `crates/core/src/commands/dismiss_all_notifications.rs` | dismiss-all-notifications command handler | ~40 | +| `crates/core/src/commands/notification_action.rs` | notification-action command handler | ~30 | +| `crates/macos/src/notifications/mod.rs` | Module re-exports | ~15 | +| `crates/macos/src/notifications/nc_session.rs` | RAII NC lifecycle guard | ~120 | +| `crates/macos/src/notifications/list.rs` | AX tree traversal of NC | ~200 | +| `crates/macos/src/notifications/actions.rs` | Dismiss + interact combined | ~150 | + +**Total new code: ~655 LOC** (reduced from ~840 in original plan by removing observer.rs and merging dismiss+interact) + +### Files to Modify (Registration Points Only) + +| File | Change | +|------|--------| +| `crates/core/src/lib.rs` | `pub mod notification;` + re-export | +| `crates/core/src/adapter.rs` | +3 trait methods with `not_supported` defaults | +| `crates/core/src/error.rs` | +1 ErrorCode variant + constructor | +| `crates/core/src/commands/mod.rs` | +4 `pub mod` declarations | +| `crates/core/src/commands/wait.rs` | +`notification` field on WaitArgs + handler branch | +| `crates/macos/src/lib.rs` | `pub mod notifications;` | +| `crates/macos/src/adapter.rs` | +3 trait method implementations | +| `src/cli.rs` | +4 Commands variants + name() arms | +| `src/cli_args.rs` | +4 arg structs + notification/poll-interval fields on WaitArgs | +| `src/dispatch.rs` | +4 match arms + extend Wait arm | +| `src/typed batch path` | +4 command routing arms (split file first if over 400 LOC) | diff --git a/docs/plans/2026-03-01-feat-compact-tree-collapsing-plan.md b/docs/plans/2026-03-01-feat-compact-tree-collapsing-plan.md new file mode 100644 index 0000000..4c8de86 --- /dev/null +++ b/docs/plans/2026-03-01-feat-compact-tree-collapsing-plan.md @@ -0,0 +1,148 @@ +--- +title: "feat: implement --compact flag for tree chain collapsing" +type: feat +status: completed +date: 2026-03-01 +origin: docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md +--- + +# feat: implement --compact flag for tree chain collapsing + +## Overview + +The `--compact` flag is fully wired through CLI → TreeOptions → snapshot pipeline but is a no-op. Implement it to collapse single-child non-interactive pass-through nodes, reducing structural noise in the JSON tree by ~15% tokens. Primary beneficiary: Electron/web apps (Slack, VS Code) where HTML `<div>` wrappers create deep group chains. + +(see brainstorm: docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md) + +## Collapse Rule + +A node is collapsible when ALL of these are true: +- `ref_id` is `None` (not interactive) +- `name` is `None` (unnamed) +- `value` is `None` (no value) +- `description` is `None` (no description) +- `states` is empty (no semantic state like "disabled") +- Has exactly **1 child** +- Is **not the root node** + +When a node is collapsible, it is removed and its single child is hoisted to take its place. Cascading is natural: bottom-up recursion means inner collapses happen first, potentially making outer nodes collapsible too. + +**Deliberately excluded: name promotion.** The brainstorm considered promoting a wrapper's name to a nameless child, but this risks contaminating `RefEntry.name` and causing `STALE_REF` on re-identification. The 3% additional savings is not worth the correctness risk. + +## Acceptance Criteria + +- [x] `--compact` collapses single-child unnamed pass-through nodes in snapshot output +- [x] All interactive refs preserved (zero information loss) +- [x] Root node never collapsed +- [x] Nodes with `description` or non-empty `states` are never collapsed +- [x] Cascading collapse works (group > group > group > button → button) +- [x] Works correctly when combined with `--interactive-only` +- [x] Works correctly in batch mode (`{"command": "snapshot", "args": {"compact": true}}`) +- [x] `--compact` alone (without `-i`) also works +- [x] Clippy clean, fmt clean, all existing tests pass +- [x] New unit tests cover: cascading, description preservation, states preservation, compact+interactive_only + +## Implementation + +### `crates/core/src/snapshot.rs` + +**1. Add helper predicate** (~10 LOC): + +```rust +fn is_collapsible(node: &AccessibilityNode) -> bool { + node.ref_id.is_none() + && node.name.is_none() + && node.value.is_none() + && node.description.is_none() + && node.states.is_empty() + && node.children.len() == 1 +} +``` + +**2. Add `compact: bool` param to `allocate_refs`** — extend signature alongside existing `interactive_only`: + +```rust +fn allocate_refs( + mut node: AccessibilityNode, + refmap: &mut RefMap, + include_bounds: bool, + interactive_only: bool, + compact: bool, // new + window_pid: i32, + source_app: Option<&str>, +) -> AccessibilityNode +``` + +**3. Add compact logic in the existing `filter_map`** — after children are recursed, before the interactive_only check: + +```rust +node.children = node + .children + .into_iter() + .filter_map(|child| { + let child = allocate_refs(child, refmap, include_bounds, interactive_only, compact, ...); + + // Compact: hoist single child of unnamed pass-through containers + if compact && is_collapsible(&child) { + return child.children.into_iter().next(); + } + + // Interactive-only: prune non-interactive leaves + if interactive_only && child.ref_id.is_none() && child.children.is_empty() { + return None; + } + + Some(child) + }) + .collect(); +``` + +Order: compact fires first (hoists child), then interactive_only may prune the hoisted child if it's a non-interactive leaf. This is correct. + +**4. Thread `compact` through the call site** in `build()`: + +```rust +let mut tree = allocate_refs(raw_tree, &mut refmap, opts.include_bounds, opts.interactive_only, opts.compact, ...); +``` + +**5. Update tracing log** in `snapshot::execute()` to include `compact={}`. + +### `src/cli_args.rs` + +**6. Update help text** from "Omit empty structural nodes from output" to "Collapse single-child unnamed nodes to reduce tree depth": + +```rust +#[arg(long, help = "Collapse single-child unnamed nodes to reduce tree depth")] +pub compact: bool, +``` + +### Tests — `crates/core/src/snapshot.rs` (unit tests) + +**7. Add tests** (~60 LOC): + +- `test_compact_collapses_single_child_chain` — group > group > group > button → button +- `test_compact_preserves_named_containers` — group("Sidebar") > button stays +- `test_compact_preserves_description` — group(desc="toolbar") > button stays +- `test_compact_preserves_states` — group(states=["disabled"]) > button stays +- `test_compact_preserves_multi_child` — group > (button + textfield) stays +- `test_compact_with_interactive_only` — both flags together work correctly + +## Files Changed + +| File | Change | +|------|--------| +| `crates/core/src/snapshot.rs` | Add `is_collapsible`, add `compact` param to `allocate_refs`, compact logic in filter_map, thread through call site, unit tests | +| `src/cli_args.rs` | Update `--compact` help text | + +## Not Changing + +- `append_surface_refs` — already uses `interactive_only: true`, compact adds negligible value for surface overlays +- `node.rs` — no struct changes needed +- `hints.rs` — runs after compact, indexes are correct post-collapse +- macOS adapter (`builder.rs`) — compact is a core-level tree transform, not platform-level + +## Sources + +- **Origin brainstorm:** [docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md](../brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md) — key decisions: no separate bridge package, chain collapsing over flattening, universal optimization not Electron-specific +- Existing pattern: `interactive_only` filter in `snapshot.rs:191` +- Existing pattern: `add_structural_hints` in `hints.rs` (post-processing pass) diff --git a/docs/plans/2026-03-02-feat-scalable-skill-architecture-clawhub-publishing-plan.md b/docs/plans/2026-03-02-feat-scalable-skill-architecture-clawhub-publishing-plan.md new file mode 100644 index 0000000..b82084b --- /dev/null +++ b/docs/plans/2026-03-02-feat-scalable-skill-architecture-clawhub-publishing-plan.md @@ -0,0 +1,299 @@ +--- +title: "feat: scalable skill architecture with ClawHub auto-publishing" +type: feat +status: completed +date: 2026-03-02 +origin: docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md +--- + +# feat: scalable skill architecture with ClawHub auto-publishing + +## Overview + +Restructure the `skills/` directory into a nested hierarchy (core → platform → app references), sync stale skill content from `.agents/` to `skills/`, add ClawHub metadata to all SKILL.md files, create a symlink script for local development, and wire CI auto-publishing to ClawHub on every release. + +(see brainstorm: docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md) + +## Problem Statement + +1. **Skills are stale** — `skills/agent-desktop/` has 50 commands documented, but `.agents/skills/agent-desktop/` has 54 (notifications added Feb 27). Git-tracked version lags. +2. **macOS skill not publishable** — lives in `.claude/skills/agent-desktop-macos/` (gitignored), not in `skills/` +3. **No publishing pipeline** — ClawHub integration doesn't exist. Manual publish only. +4. **No scaffolding** — adding a new platform or app-specific skill has no clear pattern or automation. + +## Proposed Solution + +### Directory Structure (Target) + +``` +skills/ +├── agent-desktop/ # Core skill (platform-agnostic) +│ ├── SKILL.md # Commands, observe-act loop, ref system, JSON contract +│ └── references/ +│ ├── commands-observation.md +│ ├── commands-interaction.md +│ ├── commands-system.md +│ └── workflows.md +│ +├── agent-desktop-macos/ # macOS platform skill +│ ├── SKILL.md # TCC, AX API, smart activation chain, surfaces, NC +│ └── references/ +│ └── notifications.md # NC lifecycle, dismiss strategies (extracted from SKILL.md) +│ +scripts/ +└── link-skills.sh # Symlinks skills/ → .claude/skills/ for local dev +``` + +### CI Pipeline Addition + +New `publish-skills` job in `.github/workflows/release.yml` after `publish-npm`. + +## Acceptance Criteria + +- [x] `skills/agent-desktop/` synced to match `.agents/skills/agent-desktop/` (54 commands, notifications section) +- [x] `skills/agent-desktop-macos/` created from `.claude/skills/agent-desktop-macos/` (git-tracked) +- [x] `references/macos.md` removed from core skill (moved to platform skill) +- [x] Core skill SKILL.md reference table updated (no macos.md row) +- [x] ClawHub metadata added to all SKILL.md frontmatters (`version`, `tags`, `requirements`) +- [x] `scripts/link-skills.sh` created and working +- [x] `publish-skills` job added to `.github/workflows/release.yml` +- [x] npm postinstall prompts user to install Claude Code skills (with platform auto-detection) +- [x] All SKILL.md files reviewed using `/skill-creator` skill for best practices +- [x] All existing tests still pass +- [x] Clippy clean, fmt clean + +## Build Guidance + +**Use the `/skill-creator` skill** when writing or updating any SKILL.md file. It provides best practices for frontmatter structure, trigger keywords, reference file organization, and description quality. Invoke it before finalizing each skill to ensure the content meets Claude Code skill standards. + +## Implementation + +### Phase 1: Sync & Restructure Skills Directory + +#### 1.1 Sync core skill from `.agents/` to `skills/` + +The `.agents/skills/agent-desktop/` directory is the most up-to-date version (54 commands, includes notifications). Copy it over the stale `skills/agent-desktop/`. + +**Files to sync:** + +| Source (`.agents/skills/agent-desktop/`) | Destination (`skills/agent-desktop/`) | +|---|---| +| `SKILL.md` | `SKILL.md` (overwrite — adds notifications section) | +| `references/commands-observation.md` | `references/commands-observation.md` | +| `references/commands-interaction.md` | `references/commands-interaction.md` | +| `references/commands-system.md` | `references/commands-system.md` (adds notification commands) | +| `references/workflows.md` | `references/workflows.md` | + +**After sync, delete:** `skills/agent-desktop/references/macos.md` — this content moves to the platform skill. + +**Update:** `skills/agent-desktop/SKILL.md` reference table — remove the `macos.md` row. + +#### 1.2 Move macOS skill to `skills/` + +```bash +mkdir -p skills/agent-desktop-macos/references/ +cp .claude/skills/agent-desktop-macos/SKILL.md skills/agent-desktop-macos/SKILL.md +``` + +If the macOS SKILL.md contains a Notification Center section that's large enough to be a reference, extract it to `skills/agent-desktop-macos/references/notifications.md` and reference it from the SKILL.md table. Otherwise keep it inline. + +### Phase 2: Add ClawHub Metadata + +#### 2.1 Core skill frontmatter + +```yaml +# skills/agent-desktop/SKILL.md +--- +name: agent-desktop +version: 0.1.8 +tags: desktop-automation, accessibility, ai-agent, gui-automation, cli +requirements: + - agent-desktop +description: > + Desktop automation via native OS accessibility trees... +--- +``` + +#### 2.2 macOS skill frontmatter + +```yaml +# skills/agent-desktop-macos/SKILL.md +--- +name: agent-desktop-macos +version: 0.1.8 +tags: desktop-automation, macos, accessibility, ax-api, tcc-permissions +requirements: + - agent-desktop +description: > + macOS platform details for agent-desktop... +--- +``` + +### Phase 3: Create Scaffolding Scripts + +#### 3.1 `scripts/link-skills.sh` + +```bash +#!/usr/bin/env bash +# Links skills/ directories to .claude/skills/ for local Claude Code use. +# Run after clone or when adding new skills. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CLAUDE_SKILLS="$REPO_ROOT/.claude/skills" + +mkdir -p "$CLAUDE_SKILLS" + +for skill_dir in "$REPO_ROOT"/skills/*/; do + name=$(basename "$skill_dir") + target="../../skills/$name" + link="$CLAUDE_SKILLS/$name" + + if [ -L "$link" ]; then + rm "$link" + fi + + ln -s "$target" "$link" + echo "Linked: .claude/skills/$name → skills/$name" +done +``` + +### Phase 4: Post-Install Skill Prompt + +#### 4.1 Extend `npm/scripts/postinstall.js` + +After the binary download succeeds, detect the platform and prompt the user to install the agent-desktop Claude Code skills. This runs in the terminal so we can use stdin. + +```javascript +// npm/scripts/postinstall.js — append after binary install success + +function promptSkillInstall() { + const os = require('os'); + const { execSync } = require('child_process'); + + // Check if Claude Code CLI is available + try { + execSync('claude --version', { stdio: 'ignore' }); + } catch { + log('Tip: Install Claude Code skills for agent-desktop with:'); + log(' claude /plugin marketplace add lahfir/agent-desktop'); + return; + } + + // Detect platform for the right skill + const plat = os.platform(); + const platformSkill = { + darwin: 'agent-desktop-macos', + win32: 'agent-desktop-windows', + linux: 'agent-desktop-linux', + }[plat]; + + log(''); + log('Claude Code skills available for agent-desktop!'); + log('Install with:'); + log(' claude /plugin marketplace add lahfir/agent-desktop'); + if (platformSkill) { + log(` claude /plugin install ${platformSkill}@lahfir-agent-desktop`); + } + log(''); +} + +promptSkillInstall(); +``` + +**Design decision:** Print install instructions rather than auto-running `claude` commands. Postinstall scripts should not modify the user's Claude Code config without explicit consent. The user copies and runs the commands themselves. + +#### 4.2 Platform detection mapping + +| `os.platform()` | Core skill | Platform skill | +|---|---|---| +| `darwin` | `agent-desktop` | `agent-desktop-macos` | +| `win32` | `agent-desktop` | `agent-desktop-windows` | +| `linux` | `agent-desktop` | `agent-desktop-linux` | + +### Phase 5: CI Auto-Publishing + +#### 5.1 Add `publish-skills` job to `release.yml` + +Add after the `publish-npm` job: + +```yaml +# .github/workflows/release.yml + publish-skills: + needs: [release-please] + if: needs.release-please.outputs.release_created == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install ClawHub CLI + run: npm i -g clawhub + + - name: Publish all skills to ClawHub + run: | + clawhub sync \ + --root skills/ \ + --all \ + --bump patch \ + --changelog "Release ${{ needs.release-please.outputs.tag_name }}" + env: + CLAWHUB_TOKEN: ${{ secrets.CLAWHUB_TOKEN }} +``` + +#### 5.2 Add `CLAWHUB_TOKEN` to GitHub repo secrets + +Manual step: generate token at clawhub.ai, add to repo settings → Secrets → Actions. + +## Files Changed + +| File | Change | +|------|--------| +| `skills/agent-desktop/SKILL.md` | Sync from `.agents/`, add ClawHub metadata, remove macos.md reference | +| `skills/agent-desktop/references/commands-system.md` | Sync from `.agents/` (adds notification commands) | +| `skills/agent-desktop/references/macos.md` | **Delete** (moved to platform skill) | +| `skills/agent-desktop-macos/SKILL.md` | **New** — moved from `.claude/skills/`, add ClawHub metadata | +| `skills/agent-desktop-macos/references/notifications.md` | **New** — NC details extracted if SKILL.md is too large | +| `scripts/link-skills.sh` | **New** — symlink automation | +| `npm/scripts/postinstall.js` | Add skill install prompt after binary download | +| `.github/workflows/release.yml` | Add `publish-skills` job | + +## Ongoing Convention: Skills Updated With Every Feature + +This plan establishes a **permanent convention**: every new feature, command, or platform change must update the corresponding skill files. This is enforced via: + +1. **Memory rule** — added to `MEMORY.md` under "Skill Maintenance (MANDATORY)" so it's loaded into every conversation context. Claude will automatically update skills as part of feature work. +2. **PRD addendum** — `docs/prd-addendum-skill-maintenance.md` has detailed per-phase rules. +3. **`/skill-creator` skill** — used when writing or updating any SKILL.md to ensure quality. + +| Change Type | Skill Update Required | +|---|---| +| New CLI command | Add to `skills/agent-desktop/references/commands-*.md` | +| New platform adapter | Create `skills/agent-desktop-{platform}/SKILL.md` + `references/` | +| App-specific quirk discovered | Add `skills/agent-desktop-{platform}/references/{app}.md` | +| Changed CLI flags or JSON output | Update all affected skill files | +| New workflow pattern | Add to `skills/agent-desktop/references/workflows.md` | + +## Not Changing + +- `crates/` — no Rust code changes +- `src/` — no binary changes +- `release-please-config.json` — skill versions are managed by ClawHub CLI `--bump`, not release-please +- `.gitignore` — `skills/` is already tracked, `.claude/` is already ignored + +## Dependencies & Risks + +- **CLAWHUB_TOKEN secret required** — CI job will fail without it. Must be configured before first automated release. +- **ClawHub automated review** — published skills go through syntax validation and permission scanning (<5 min). If rejected, need to fix and re-publish. +- **First publish should be manual** — run `clawhub sync --root skills/ --all --dry-run` locally to verify before relying on CI. + +## Sources + +- **Origin brainstorm:** [docs/brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md](../brainstorms/2026-03-02-clawhub-skill-publishing-brainstorm.md) — key decisions: nested hierarchy, CI auto-publish, platform > apps scoping +- Existing CI pattern: `.github/workflows/release.yml` — 4-job pipeline (release-please → build → publish-github → publish-npm) +- Skill maintenance rules: `docs/prd-addendum-skill-maintenance.md` +- ClawHub CLI: [docs.openclaw.ai/tools/clawhub](https://docs.openclaw.ai/tools/clawhub) diff --git a/docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md b/docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md new file mode 100644 index 0000000..36183a1 --- /dev/null +++ b/docs/plans/2026-03-10-feat-progressive-skeleton-traversal-plan.md @@ -0,0 +1,480 @@ +--- +title: "feat: progressive skeleton traversal with ref-rooted drill-down" +type: feat +status: active +date: 2026-03-10 +deepened: 2026-03-10 +origin: docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md +--- + +# Progressive Skeleton Traversal + +## Enhancement Summary + +**Deepened on:** 2026-03-10 +**Agents used:** architecture-strategist, performance-oracle, security-sentinel, pattern-recognition-specialist, code-simplicity-reviewer, agent-native-reviewer, data-integrity-guardian, best-practices-researcher + +### Key Improvements from Review + +1. **Scope reduction** — Cut `epoch` (redundant with per-element staleness), `new_ref_count` (agent computes delta), `mode` response field (agent knows what it asked for). Defer `find --root` and `--skeleton --root` combo to follow-up. +2. **Naming fix** — Rename `drill_down()` to `run_from_ref()` for consistency with existing `build()`/`run()` pattern. +3. **Performance** — Add `count_children()` using raw `CFArrayGetCount` to avoid N `CFRetain`/`CFRelease` pairs at skeleton boundary. +4. **Data integrity** — Add write-side size check to `RefMap::save()` to prevent >1MB files that would fail on next load. +5. **LOC management** — `snapshot.rs` is at 354 LOC. Extract ref-rooted logic to `snapshot_ref.rs` to stay under 400. +6. **Security** — Document TOCTOU race on refmap as known limitation, defer file locking to Phase 4. + +### Simplifications Applied + +| Original | Decision | Rationale | +|----------|----------|-----------| +| `epoch: u32` on RefEntry/RefMap | **Cut** | Existing per-element re-identification via `(pid, role, name, bounds_hash)` already handles staleness. Epoch adds a second, redundant mechanism. | +| `new_ref_count` in response | **Cut** | Agent can diff `ref_count` across calls if needed. One number is sufficient. | +| `mode` field in response | **Cut** | Agent knows what it passed (`--skeleton` or `--root`). Presence of `children_count` is the structural signal. | +| `find --root` | **Deferred** | Scope creep. Agent can do `snapshot --root @e3` then `find` separately. Follow-up PR. | +| `--skeleton --root` combo | **Deferred** | Edge case for very deep subtrees. Add in follow-up if agents need it. Reduces testing matrix. | +| `increment_epoch()` | **Cut** | No epoch to increment. | + +--- + +## Overview + +Add a two-phase incremental tree traversal system that lets AI agents explore dense accessibility trees progressively — skeleton overview first, then targeted drill-downs — instead of consuming the full tree every time. This introduces two new `snapshot` flags (`--skeleton`, `--root @ref`), refmap merging with `root_ref`-tagged scoped invalidation, and a new `PlatformAdapter::get_subtree()` trait method. + +Token savings for Slack: **73-96%** vs current full snapshot (see brainstorm: `docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md`). + +## Problem Statement + +Dense Electron apps (Slack, VS Code, Discord) produce massive accessibility trees — 500+ nodes, ~12,000 tokens per full snapshot. Even with `--interactive-only --compact`, Slack still costs ~5,400 tokens (~204 nodes). Agents pay per-token and struggle to navigate large trees efficiently. + +Current optimizations (depth-skip, compact, surfaces, interactive-only) reduce output size but still require full tree traversal from the window root. There is no way to: +- Get a cheap structural overview of an app +- Drill into a specific region without re-traversing the entire tree +- Accumulate knowledge across multiple targeted observations + +## Proposed Solution + +Two new capabilities on the `snapshot` command: + +1. **`--skeleton`** — Shallow overview (default depth 3) where truncated nodes show `children_count` instead of full subtrees. Named containers at the truncation boundary receive refs as drill-down targets. + +2. **`--root @ref`** — Start traversal from a previously-discovered ref element instead of the window root. Merges new refs into the existing refmap with `root_ref`-tagged scoped invalidation. + +These compose with all existing flags (`--compact`, `--interactive-only`, `--max-depth`, `--surface`). + +## Technical Approach + +### Architecture + +The feature touches three layers: + +``` +CLI (cli_args.rs, dispatch.rs, typed batch path) + ↓ --skeleton, --root flags +Core (adapter.rs, snapshot.rs, snapshot_ref.rs [NEW], refs.rs, node.rs) + ↓ TreeOptions, skeleton logic, merge logic, children_count +Platform (macos/adapter.rs, macos/tree/builder.rs, macos/tree/element.rs) + ↓ get_subtree(), skeleton-aware build_subtree(), count_children() +``` + +One new file: `crates/core/src/snapshot_ref.rs` — extracted drill-down logic to keep `snapshot.rs` under 400 LOC. The dependency inversion principle is preserved — core defines the interface, platform implements it. + +### Research Insights: Architecture + +**From architecture-strategist:** +- The plan initially proposed `get_subtree()` as a separate trait method. An alternative is to extend `get_tree()` with an optional `&NativeHandle` parameter. However, `NativeHandle` contains a raw pointer and would break `TreeOptions`'s `Clone`/`Copy` derives. **Decision: keep `get_subtree()` as additive trait method** — both `get_tree()` and `get_subtree()` delegate to the same internal `build_subtree()` in each platform adapter, preventing behavior divergence. +- `snapshot.rs` is at 354 LOC. Adding `run_from_ref()` and merge logic would push it over 400. **Decision: extract to `snapshot_ref.rs`** with `pub(crate)` visibility, re-exported from `snapshot.rs`. + +**From pattern-recognition-specialist:** +- `drill_down()` does not follow the verb-first naming convention (existing: `build()`, `run()`, `allocate_refs()`). **Renamed to `run_from_ref()`** to match the existing `run()` pattern. +- TreeOptions grows from 5 to 7 fields (at the CLAUDE.md boundary of 7). Acceptable since both new fields are simple primitives. If more fields are needed later, extract a `SnapshotMode` enum. + +### Data Flow: Skeleton Mode + +``` +snapshot --app Slack --skeleton -i + → SnapshotArgs { skeleton: true, interactive_only: true } + → TreeOptions { skeleton: true, interactive_only: true, ... } + → snapshot::build() [existing path] + → adapter.get_tree(window, opts) [skeleton flag in opts] + → build_subtree(el, 0, 3, true, ...) [skeleton_depth=3] + → At depth limit: count_children() [O(1), no per-element CFRetain] + → Set children_count on node, return with empty children vec + → allocate_refs(tree, &mut fresh_refmap, opts) + → Interactive elements: assign refs (existing) + → Named containers at boundary with children_count > 0: assign refs (NEW) + → Pruning: skip if non-interactive AND children empty AND children_count is None + → refmap.save() [full replace] + → JSON response +``` + +### Data Flow: Drill-Down Mode + +``` +snapshot --root @e3 -i --compact + → SnapshotArgs { root_ref: Some("@e3"), interactive_only: true, compact: true } + → snapshot_ref::run_from_ref(adapter, opts, "@e3") [NEW function, NEW file] + → resolve_ref("@e3") → (RefEntry, NativeHandle) + → Derive app/window context from RefEntry.pid + → adapter.get_subtree(handle, opts) [NEW trait method] + → macOS: ManuallyDrop AXElement from handle, build_subtree() + → existing_refmap = RefMap::load() + → existing_refmap.remove_by_root_ref("@e3") [scoped invalidation] + → allocate_refs(subtree, &mut existing_refmap, opts) [counter continues] + → New refs tagged with root_ref: "@e3" + → existing_refmap.save() [merged, with write-side size check] + → JSON response with ref_count (total) +``` + +### Research Insights: Performance + +**From performance-oracle:** +- **Children count at skeleton boundary**: `copy_children()` triggers a Mach IPC call that materializes the entire CFArray of child AXUIElementRefs. Even calling `.len()` pays full IPC + CFArray allocation. **Recommendation: Add a `count_children()` function** that calls `AXUIElementCopyAttributeValue` then `CFArrayGetCount` on the raw `CFArrayRef`, then releases — without materializing per-element `AXElement` wrappers. Saves N `CFRetain`/`CFRelease` pairs per truncation boundary node. +- **RefMap file I/O**: ~3-4ms for load+save of a 500-entry refmap (50-80KB). Negligible against 50-200ms total CLI invocation time. No optimization needed. +- **Counter collision**: Safe — counter continues from loaded refmap. `allocate()` increments before formatting, so first new ref is `@e{loaded_counter+1}`. +- **Branch cost of `skeleton: bool` in recursive `build_subtree()`**: Near-zero. Branch prediction handles it (the flag is constant across the entire recursive call tree). +- **Net savings**: CLI startup is ~10ms. Even with 3 invocations (skeleton + 2 drill-downs), total overhead is ~30ms vs ~5,400 tokens saved. Token cost dominates by orders of magnitude. + +### Implementation Phases + +#### Phase 1: Data Model Foundation + +Add new fields to core types. No behavioral changes yet — all new fields are optional with serde defaults for backward compatibility. + +**`crates/core/src/node.rs`** — AccessibilityNode: +- [ ] Add `children_count: Option<u32>` with `#[serde(skip_serializing_if = "Option::is_none", default)]` +- [ ] Place before `children` field (line ~29) +- [ ] Update test helper `make_node()` in `snapshot.rs` to include field + +**`crates/core/src/refs.rs`** — RefEntry: +- [ ] Add `root_ref: Option<String>` with `#[serde(skip_serializing_if = "Option::is_none", default)]` + +**`crates/core/src/refs.rs`** — RefMap: +- [ ] Add `fn remove_by_root_ref(&mut self, root: &str)` — `self.inner.retain(|_, v| v.root_ref.as_deref() != Some(root))` +- [ ] Update `allocate()` to accept `root_ref: Option<&str>` parameter, store on each new `RefEntry` +- [ ] Add write-side size check in `save()`: if serialized JSON exceeds `MAX_REFMAP_BYTES`, return error instead of writing (preserves previous valid refmap on disk) + +**`crates/core/src/adapter.rs`** — TreeOptions: +- [ ] Add `skeleton: bool` (default `false`) +- [ ] Add `root_ref: Option<String>` (default `None`) +- [ ] Update `Default` impl + +**`crates/core/src/adapter.rs`** — PlatformAdapter trait: +- [ ] Add `fn get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError>` with default `not_supported("get_subtree")` + +### Research Insights: Data Model + +**From data-integrity-guardian:** +- **Write-side size check is critical.** Currently `MAX_REFMAP_BYTES` (1MB) is checked only on `load()`. A series of drill-downs could produce a >1MB file that then fails on every subsequent `load()`, making all refs inaccessible. The fix: check in `save()` and fail early, preserving the previous valid file (temp+rename never completes). +- **Orphaned refs from dead PIDs** accumulate but are harmless — `resolve_element()` returns `STALE_REF` naturally. No cleanup needed. + +**From best-practices-researcher (serde):** +- `#[serde(default)]` on `Option<String>` fields defaults to `None`. Correct for backward compat. +- `#[serde(skip_serializing_if = "Option::is_none")]` prevents writing `null` fields. Correct. +- Do NOT use `#[serde(deny_unknown_fields)]` — allow unknown fields for forward compatibility (newer binary writes fields that older binary doesn't know about). + +**Tests (Phase 1):** +- [ ] Backward compat: deserialize old RefEntry JSON (missing root_ref) → defaults to `root_ref: None` +- [ ] `remove_by_root_ref()`: removes only matching entries, preserves others +- [ ] `AccessibilityNode` with `children_count: Some(5)` and empty `children` serializes correctly (no `children` key, has `children_count`) +- [ ] `RefMap::save()` rejects oversized refmaps (>1MB) with actionable error +- [ ] `RefMap::save()` on oversized rejection: previous file on disk is preserved + +#### Phase 2: Skeleton Mode + +Implement the `--skeleton` flag end-to-end. + +**`crates/macos/src/tree/element.rs`** — New `count_children()` function: +- [ ] Add `pub fn count_children(el: &AXElement) -> usize` that calls `AXUIElementCopyAttributeValue` for `kAXChildrenAttribute`, then `CFArrayGetCount` on raw `CFArrayRef`, then `CFRelease` — without materializing per-element `AXElement` wrappers +- [ ] Returns 0 if attribute fetch fails + +**`crates/macos/src/tree/builder.rs`** — build_subtree(): +- [ ] Accept `skeleton: bool` parameter (add to function signature) +- [ ] When `skeleton=true` and `depth >= skeleton_depth`: call `count_children(el)` instead of `copy_children()`, set `children_count` on the node, return the node with empty children vec +- [ ] Web-wrapper depth-skip still applies during skeleton traversal (existing behavior preserved) +- [ ] Skeleton depth constant: `SKELETON_DEFAULT_DEPTH: u8 = 3` +- [ ] Effective skeleton depth: `if skeleton { min(max_depth, SKELETON_DEFAULT_DEPTH) } else { max_depth }` — `--max-depth` can make it shallower but not deeper than 3 in skeleton mode + +**`crates/core/src/snapshot.rs`** — allocate_refs(): +- [ ] Accept `skeleton: bool` parameter +- [ ] New ref assignment rule for skeleton mode: if node has `children_count.is_some()` AND node has a non-empty name → assign a ref (drill-down target), even if role is not interactive +- [ ] `available_actions` for skeleton boundary refs: empty `vec![]` +- [ ] Fix `interactive_only` pruning: change guard to `child.ref_id.is_none() && child.children.is_empty() && child.children_count.is_none()` — do NOT prune truncated nodes that have `children_count` set + +**`crates/core/src/commands/snapshot.rs`** — SnapshotArgs: +- [ ] Add `skeleton: bool` field +- [ ] Pass to TreeOptions construction + +**`src/cli_args.rs`** — SnapshotArgs: +- [ ] Add `#[arg(long, help = "Shallow overview with children_count on truncated nodes")]` `skeleton: bool` + +**`src/dispatch.rs`**: +- [ ] Pass `skeleton` through to core SnapshotArgs + +**`src/typed batch path`**: +- [ ] Parse `skeleton` from JSON in batch snapshot dispatch + +### Research Insights: Skeleton Mode + +**From performance-oracle:** +- At the skeleton truncation boundary, `count_children()` still triggers one Mach IPC call per boundary node. For a depth-3 skeleton with ~50 boundary nodes, that is 5-25ms overhead. Acceptable — the savings (avoiding 450+ deeper IPC calls) vastly outweigh it. + +**From code-simplicity-reviewer:** +- `allocate_refs()` currently has 7 parameters (already exceeds the 5-param max from CLAUDE.md). Adding `skeleton: bool` makes 8. **Recommendation: pass `skeleton` via `TreeOptions`** (already done — `opts.skeleton` is available). Do NOT add a separate parameter. Access it from the `opts` struct that's already threaded through. + +**Tests (Phase 2):** +- [ ] Skeleton of mock tree with depth 5: output has max depth 3, truncated nodes have `children_count`, no `children` array +- [ ] Named container at boundary gets a ref; anonymous wrapper at boundary does not +- [ ] `interactive_only` does NOT prune truncated nodes with `children_count` +- [ ] `--skeleton --max-depth 2` produces depth-2 output +- [ ] `--skeleton --compact` composes: compact collapses within skeleton depth +- [ ] Skeleton ref_count includes both interactive refs and drill-down target refs +- [ ] Golden fixture: skeleton of standard test tree +- [ ] `count_children()` does not materialize AXElement wrappers (unit test in macos crate) + +#### Phase 3: Drill-Down Mode + +Implement `--root @ref` with refmap merging. + +**`crates/macos/src/adapter.rs`** — MacOSAdapter: +- [ ] Implement `get_subtree(&self, handle: &NativeHandle, opts: &TreeOptions) -> Result<AccessibilityNode, AdapterError>` +- [ ] Extract AXElement from handle: `ManuallyDrop::new(AXElement(handle.as_raw() as AXUIElementRef))` (existing pattern from `get_live_value`) +- [ ] Call `build_subtree(&el, 0, opts.max_depth, opts.include_bounds, &mut FxHashSet::default())` — fresh ancestor set +- [ ] Delegate to same `build_subtree()` as `get_tree()` — prevents behavior divergence + +**`crates/core/src/snapshot_ref.rs`** — New file (~80-100 LOC): +- [ ] `pub(crate) fn run_from_ref(adapter: &dyn PlatformAdapter, opts: &TreeOptions, root_ref: &str) -> Result<RefRootResult, AppError>` +- [ ] Step 1: `resolve_ref(root_ref, adapter)` → `(entry, handle)`. On failure → `STALE_REF` with suggestion "Run 'snapshot --skeleton' to refresh" +- [ ] Step 2: Derive app context from `entry.pid` and `entry.source_app` +- [ ] Step 3: `adapter.get_subtree(&handle, opts)` → raw subtree +- [ ] Step 4: `RefMap::load()` → existing refmap +- [ ] Step 5: `existing_refmap.remove_by_root_ref(root_ref)` — scoped invalidation +- [ ] Step 6: `allocate_refs(subtree, &mut existing_refmap, opts)` with `root_ref_tag: Some(root_ref)` — counter continues from loaded refmap +- [ ] Step 7: `existing_refmap.save()` (includes write-side size check) +- [ ] Step 8: Return `RefRootResult { tree, refmap, app_name, window }` +- [ ] **Skeleton boundary ref preservation**: @e3 itself (from skeleton, with `root_ref: None`) is NOT removed by scoped invalidation. Only refs where `root_ref == "@e3"` are removed. The agent keeps using @e3 for re-drilling. + +**`crates/core/src/snapshot.rs`**: +- [ ] Add `pub mod snapshot_ref;` or re-export from the module +- [ ] In existing code, no changes needed — `build()` and `run()` are untouched + +**`crates/core/src/commands/snapshot.rs`** — execute(): +- [ ] Branch: if `args.root_ref.is_some()` → call `snapshot_ref::run_from_ref()` instead of `run()` +- [ ] Format response with `ref_count` (total) +- [ ] Derive `app` and `window` from the resolved `RefEntry.source_app` and PID window lookup + +**`src/cli_args.rs`**: +- [ ] Add `#[arg(long, value_name = "REF", help = "Start from a previously-discovered ref")]` `root: Option<String>` + +**`src/dispatch.rs`**: +- [ ] Pass `root` as `root_ref` through to core SnapshotArgs + +**`src/typed batch path`**: +- [ ] Parse `root` from JSON in batch snapshot dispatch: `root_ref: str_field(&args, "root")` + +**Argument validation:** +- [ ] `--root` + `--surface` → return `INVALID_ARGS` with message "Cannot use --root and --surface together" +- [ ] Validate ref format (`@e{N}`) before attempting resolution + +### Research Insights: Drill-Down Mode + +**From security-sentinel:** +- **TOCTOU race (HIGH):** The load → modify → save cycle on the refmap has no file locking. Concurrent `agent-desktop` invocations can silently overwrite each other's refs. The atomic rename prevents corruption but not lost updates. **Mitigation for now:** Document as known limitation. **For Phase 4:** Add advisory `flock()`/`fcntl()` around the load-modify-save cycle. +- **PID reuse (MEDIUM):** If an app restarts with a different PID, the old PID could theoretically belong to a different process. The `resolve_element()` function already matches on `(pid, role, name, bounds_hash)` which makes accidental cross-process resolution extremely unlikely. Acceptable risk. +- **Write-side size check:** Implemented in Phase 1 (RefMap::save). Prevents the scenario where drill-downs produce a >1MB file that blocks all future operations. + +**From agent-native-reviewer:** +- **Batch dispatch parity is critical.** Every new CLI field must have an explicit batch JSON parsing line in `typed batch path`. The plan now includes explicit checklist items for both `skeleton` and `root` parsing. +- **Batch sequencing works:** A batch like `[{"command": "snapshot", "skeleton": true}, {"command": "snapshot", "root": "@e3"}]` executes sequentially, and the second command reads the refmap written by the first. This is the existing batch execution model. + +**Tests (Phase 3):** +- [ ] Drill-down from a ref: subtree returned with correct depth, refs merge into existing map +- [ ] Scoped invalidation: drill into @e3, refs with `root_ref: "@e3"` removed, others preserved +- [ ] Skeleton boundary ref (@e3 with `root_ref: None`) preserved after drill-down +- [ ] Re-drill same region: old subtree refs replaced, new ones added, counter continues +- [ ] Multiple drill-downs accumulate: refs from @e3 and @e7 coexist +- [ ] Stale root ref: returns STALE_REF with skeleton suggestion +- [ ] `--root @e3 --surface menu` → INVALID_ARGS error +- [ ] Empty subtree (root has no children): valid response with ref_count of 0 or 1 +- [ ] Counter continuity: after skeleton (refs @e1-@e10), drill-down creates @e11+ +- [ ] Golden fixture: drill-down result merged refmap +- [ ] Write-side size check: rapid drill-downs approaching 1MB → error, previous refmap preserved + +#### Phase 4: Polish and Documentation + +- [ ] Update STALE_REF suggestion in `crates/core/src/commands/helpers.rs`: when `STALE_REF` occurs and the ref was from a drill-down (has `root_ref`), suggest "Run 'snapshot --skeleton' to refresh, then re-drill" +- [ ] Update skill files: `skills/agent-desktop/references/commands-observation.md` — document `--skeleton`, `--root` flags +- [ ] Add golden fixtures for skeleton and drill-down JSON output +- [ ] Run full clippy + fmt + test suite + +### Deferred to Follow-Up PRs + +These were cut from scope based on simplicity review: + +- [ ] **`find --root @ref`** — Scoped search within a ref's subtree. Agent can achieve this today with `snapshot --root @e3` then `find`. Follow-up PR. +- [ ] **`--skeleton --root @ref` combo** — Skeleton view of a subtree. Useful for very deep regions. Follow-up PR. +- [ ] **`epoch` counter on RefMap/RefEntry** — Was deemed redundant with existing per-element staleness detection. Revisit if agents report confusion about ref freshness. +- [ ] **Advisory file locking (`flock`)** — Prevents TOCTOU race on refmap during concurrent access. Deferred to Phase 4 daemon. +- [ ] **RefMap size eviction policy** — Auto-purge oldest refs when approaching 1MB. For now, agents should periodically run `--skeleton` to reset. + +## Alternative Approaches Considered + +(see brainstorm: `docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md`) + +1. **Semantic Zone Auto-Detection** — Auto-label regions ("sidebar", "toolbar") via role heuristics. Rejected: fragile across apps, platform-dependent role semantics, introduces new concept (zones) alongside refs. + +2. **Query-Driven Traversal** — Skip trees, agents search directly with `find --near "Channels"`. Rejected: requires agents to know what they're looking for, fails for the "orient myself in an unfamiliar app" use case. + +3. **Auto-skeleton default** — Make `--skeleton` the default when `--compact --interactive-only` are both set. Rejected: changes existing behavior, could break agents already using those flags. + +4. **Extend `get_tree()` signature instead of adding `get_subtree()`** — Architecture reviewer suggested adding `Option<&NativeHandle>` to `get_tree()`. Rejected: `NativeHandle` contains raw pointers, would break `TreeOptions`'s `Clone`/`Copy` derives. The additive `get_subtree()` method is cleaner and follows the project's pattern of default `not_supported` implementations. + +## System-Wide Impact + +### Interaction Graph + +``` +snapshot --skeleton → build() → adapter.get_tree() → build_subtree() [skeleton-aware] + → allocate_refs() [skeleton-aware ref assignment] + → refmap.save() [full replace] + +snapshot --root → run_from_ref() → resolve_ref() → adapter.get_subtree() + → RefMap::load() → remove_by_root_ref() + → allocate_refs() [merge into loaded refmap] + → refmap.save() [merged, size-checked] + +get @ref → RefMap::load() → resolve → [transparent, no changes needed] +click @ref → RefMap::load() → resolve → [transparent, no changes needed] +``` + +### Error Propagation + +- `--root @e3` with stale ref → `resolve_ref()` fails → `STALE_REF` (exit 1) with skeleton suggestion +- `--root @e3 --surface menu` → validation in `execute()` → `INVALID_ARGS` (exit 2) +- `adapter.get_subtree()` fails (e.g., app crashed mid-traversal) → `ACTION_FAILED` propagated +- RefMap load failure during merge → `INTERNAL` error (file permissions, corrupt JSON) +- RefMap save exceeds 1MB → `INTERNAL` error with suggestion to run full snapshot to reset + +### State Lifecycle Risks + +- **RefMap is the critical state artifact.** Skeleton replaces entirely (safe). Drill-down merges (partial update). Concurrent access (two agents) can cause lost updates — documented, deferred to Phase 4. +- **Unbounded refmap growth:** Multiple drill-downs without a skeleton reset accumulate refs. Write-side size check (1MB) prevents catastrophic failure. Agent best practice: periodically run `--skeleton` to reset. +- **TOCTOU race (documented):** The load → modify → save cycle has no file locking. Atomic rename prevents corruption but not lost updates. Phase 4 will add advisory `flock()`. + +### API Surface Parity + +| Interface | Needs update | +|-----------|-------------| +| `snapshot` CLI command | Yes — `--skeleton`, `--root` flags | +| `batch` JSON API | Yes — `skeleton`, `root` fields | +| MCP server (Phase 3) | Will inherit via dispatch | +| All action commands | No — transparent via merged refmap | +| `get`, `is` commands | No — transparent via merged refmap | +| `find` CLI command | Deferred — follow-up PR | + +### Integration Test Scenarios + +1. **End-to-end progressive workflow**: `snapshot --skeleton` → `snapshot --root @e3` → `click @e45` → verify action succeeds on element from drill-down +2. **Re-drill after UI change**: `snapshot --skeleton` → `snapshot --root @e3` → [UI changes] → `snapshot --root @e3` again → verify stale refs replaced, fresh refs work +3. **Cross-region interaction**: `snapshot --skeleton` → `snapshot --root @e3` → `snapshot --root @e7` → `click @e25` (from @e3 drill-down) → verify still works +4. **Full snapshot resets**: `snapshot --skeleton` → `snapshot --root @e3` → `snapshot --app Slack -i` (no flags) → verify refmap fully replaced, drill-down refs gone +5. **Batch with skeleton + drill-down**: batch JSON with skeleton first, drill-down second → verify sequential execution and refmap merge + +## Acceptance Criteria + +### Functional Requirements + +- [ ] `snapshot --skeleton -i` returns max depth 3 tree with `children_count` on truncated nodes +- [ ] Named containers at skeleton boundary receive refs as drill-down targets +- [ ] `snapshot --root @e3` returns subtree rooted at @e3, merges refs into existing refmap +- [ ] Scoped invalidation: re-drilling @e3 removes only @e3-rooted refs, preserves others +- [ ] `--root @ref --surface menu` returns `INVALID_ARGS` +- [ ] Stale root ref returns `STALE_REF` with skeleton suggestion +- [ ] All existing flags compose: `--skeleton --compact --interactive-only`, `--root @e3 --compact --interactive-only --max-depth 5` +- [ ] Write-side refmap size check prevents >1MB files + +### Non-Functional Requirements + +- [ ] Skeleton snapshot of Slack: < 50 nodes, < 1,000 tokens +- [ ] Drill-down of Slack sidebar: < 100 nodes, < 2,000 tokens +- [ ] No regression in full snapshot performance (same path when no new flags used) +- [ ] Backward compatibility: old refmap files (no root_ref) deserialize with defaults + +### Quality Gates + +- [ ] `cargo clippy --all-targets -- -D warnings` passes +- [ ] `cargo fmt --all -- --check` passes +- [ ] `cargo test --lib --workspace` passes (all new + existing tests) +- [ ] No file exceeds 400 LOC (snapshot_ref.rs extraction keeps snapshot.rs under limit) +- [ ] Zero `unwrap()` in non-test code +- [ ] Golden fixtures for skeleton output and merged refmap + +## Dependencies & Prerequisites + +- No new crate dependencies +- No platform-specific dependencies beyond what exists +- Windows/Linux stubs inherit `get_subtree()` default (`not_supported`) — no stub changes needed + +## Risk Analysis & Mitigation + +| Risk | Impact | Mitigation | +|------|--------|------------| +| `interactive_only` pruning removes truncated skeleton nodes | High — drill-down targets lost | Guard: only prune if `children_count.is_none()` | +| Ref ID collision during merge | High — corrupt refmap | Counter continues from loaded refmap, never resets during merge | +| Concurrent drill-down race condition | Medium — lost updates | Documented. Atomic write prevents corruption. flock in Phase 4. | +| RefMap exceeds 1MB after many drill-downs | Medium — all refs inaccessible | Write-side size check in save(). Fail early, preserve previous file. | +| `children_count` not O(1) on Linux AT-SPI | Low (Phase 3) | Verify during Linux adapter. Fallback: quick recursive count. | +| `--skeleton --max-depth 100` defeats purpose | Low — foot-gun | Cap: `min(max_depth, SKELETON_DEFAULT_DEPTH)` | +| `snapshot.rs` exceeds 400 LOC | Low — coding standard violation | Extract to `snapshot_ref.rs` | + +## File Change Summary + +| File | Change | LOC Impact | +|------|--------|------------| +| `crates/core/src/node.rs` | Add `children_count` field | +2 | +| `crates/core/src/refs.rs` | Add `root_ref` to RefEntry, `remove_by_root_ref()` + save size check to RefMap | +25 | +| `crates/core/src/adapter.rs` | Add fields to TreeOptions, `get_subtree()` to trait | +15 | +| `crates/core/src/snapshot.rs` | Skeleton logic in `allocate_refs()`, pruning fix | +20 | +| `crates/core/src/snapshot_ref.rs` | **NEW** — `run_from_ref()` drill-down logic | +80-100 | +| `crates/core/src/commands/snapshot.rs` | Branch on `root_ref`, skeleton args | +15 | +| `crates/macos/src/adapter.rs` | Implement `get_subtree()` | +15 | +| `crates/macos/src/tree/builder.rs` | Skeleton-aware truncation | +15 | +| `crates/macos/src/tree/element.rs` | `count_children()` function | +15 | +| `src/cli_args.rs` | `--skeleton`, `--root` flags | +5 | +| `src/dispatch.rs` | Pass new args | +3 | +| `src/typed batch path` | Parse new JSON fields | +5 | +| **Total** | | **~215-235 new LOC** | + +## Sources & References + +### Origin + +- **Brainstorm document:** [docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md](docs/brainstorms/2026-03-10-progressive-skeleton-traversal-brainstorm.md) — Key decisions: skeleton with children_count hint, ref-rooted drill-down with scoped merge, named containers get refs at skeleton boundary, `--skeleton` flag naming, opt-in only. + +### Internal References + +- `crates/core/src/snapshot.rs:127-149` — `append_surface_refs()` precedent for refmap merge pattern +- `crates/core/src/snapshot.rs:160-215` — `allocate_refs()` where skeleton logic and pruning fix go +- `crates/core/src/refs.rs:8-28` — RefEntry and RefMap structs to extend +- `crates/core/src/adapter.rs:27-45` — TreeOptions to extend +- `crates/core/src/adapter.rs:115-247` — PlatformAdapter trait for new method +- `crates/macos/src/adapter.rs:33-56` — `get_tree()` pattern to follow for `get_subtree()` +- `crates/macos/src/tree/builder.rs:44-119` — `build_subtree()` to make skeleton-aware +- `crates/macos/src/tree/element.rs` — Where `count_children()` goes +- `crates/core/src/commands/helpers.rs:11-34` — `resolve_ref()` used by drill-down path + +### Review Agents Applied + +| Agent | Key Finding | +|-------|-------------| +| architecture-strategist | Keep `get_subtree()` additive; extract `snapshot_ref.rs` for LOC management | +| performance-oracle | Add `count_children()` using raw CFArrayGetCount; RefMap I/O negligible | +| security-sentinel | TOCTOU race documented, defer flock to Phase 4; write-side size check critical | +| pattern-recognition | Rename `drill_down` → `run_from_ref`; TreeOptions at 7-field boundary | +| code-simplicity-reviewer | Cut epoch, new_ref_count, mode; defer find --root and skeleton+root combo | +| agent-native-reviewer | Explicit batch dispatch parsing for all new fields | +| data-integrity-guardian | Write-side size check prevents catastrophic refmap failure | + +### Related Work + +- `docs/brainstorms/2026-03-01-electron-tree-compaction-brainstorm.md` — compact mode (already implemented), composes with skeleton +- `docs/brainstorms/2026-02-19-ax-tree-accuracy-speed-brainstorm.md` — tree accuracy insights diff --git a/docs/plans/2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md b/docs/plans/2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md new file mode 100644 index 0000000..ca46166 --- /dev/null +++ b/docs/plans/2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md @@ -0,0 +1,1008 @@ +--- +title: "fix: resolve PR #22 FFI safety and ABI correctness findings" +type: fix +status: active +date: 2026-04-16 +reviewed: 2026-04-16 +origin: https://github.com/lahfir/agent-desktop/pull/22#pullrequestreview-4102879703 +--- + +> **Review status.** This plan was reviewed on 2026-04-16 by five reviewer personas (coherence, feasibility, security-lens, scope-guardian, adversarial). A P0 finding — Cargo rejecting per-package `panic` override — was verified experimentally (spike-compile) and the plan was restructured: +> - **Unit 1** now uses a dedicated `[profile.release-ffi]` inheriting from release (custom profile verified: `cargo run --profile release-ffi --example panic_spike` catches panic and returns cleanly). +> - **Unit 5** split into 5a/5b/5c; **Unit 6** split into 6a/6b/6c; **Unit 8** split into 8a/8b; **Unit 13** split into 13a/13b/13c. Each atomic-commit-sized. +> - **Unit 10** (action constructor helpers) dropped — out of review scope, surface expansion, consumer wrappers make helpers moot. +> - **New Unit 6d**: `ad_abi_version()` + compile-time `AD_ABI_VERSION` constant to prevent silent heap corruption from header/dylib mismatch throughout pre-1.0. +> - **New Unit 11b**: minimal C-side test harness exercising struct memcpy, enum fuzzing, interior-NUL, double-free, list-handle-after-free (Rust-only tests cannot exercise these bug classes). +> - **Unit 11** main-thread check elevated to unconditional release-mode enforcement (`pthread_main_np()` → `ErrInternal`); Send+Sync on `NativeHandle` removed and replaced with FFI-layer thread-ownership tracking. +> - **Unit 7** makes `AdActionResult` opaque (matches list-handle pattern) to close memcpy-copy double-free vector. +> - **Security elevation**: `AXIsProcessTrusted` inheritance when Python/Node dlopens the cdylib is documented as a privilege-escalation vector, not a threading note. Header + skill carry a P0 security callout. +> - **Unit 4** gains an Alternatives Considered section (BFS vs DFS-additive); BFS pruning parity is property-tested against current DFS output. +> - **cbindgen opaque_types** list added to Unit 5 (cbindgen silently skips non-repr(C) structs otherwise). +> - **`publish = false`** guard on `crates/ffi/Cargo.toml` added as Unit 0 (prevents accidental cargo-publish until distribution plan lands). +> - **`AdRefEntry` partial-freeze** moved to Deferred Tasks with explicit rationale and user sign-off checkpoint. +> - **tracing output** in cdylib mode: Unit 7 redacts AX string content (log field name + NUL position only) to prevent host-log data leakage until `ad_set_log_callback` lands in Phase 2. +> - **New Phase 0 / Unit R (user request, 2026-04-16)**: refactor the entire `crates/ffi/` crate to the project's modular file layout **before any safety work**. Split `types.rs` (28 packed types) into `types/` subdirectory with one struct/enum per file; mirror the platform-crate folder pattern (`convert/`, `tree/`, `actions/`, `apps/`, `windows/`, `input/`, `screenshot/`, `surfaces/` each with submodules); strip every inline `//` comment repo-wide in the crate; replace wildcard `pub use types::*` with explicit re-exports; remove the `.unwrap()` in `error.rs:56`; eliminate unjustified `#[allow(dead_code)]` annotations; keep every file under the 400 LOC hard limit. `///` doc-comments retained only where the item name isn't self-documenting. All subsequent units operate on the post-refactor paths. + +# FFI Safety and ABI Correctness + +## Overview + +PR #22 introduced `crates/ffi/` — a new `cdylib + cbindgen` C FFI layer exposing `PlatformAdapter` over the C ABI so Python / Swift / Go / Node / C++ consumers can drive agent-desktop without forking a subprocess per call. + +The PR review identified 24 findings (4 blockers, 11 important, 5 smaller, 4 nits) that must be resolved before the first `agent_desktop.h` ships. Once the header is distributed, every one of these fixes becomes a breaking change for downstream consumers. + +This plan sequences the fixes so safety primitives (panic boundary, enum validation, error lifetime) land first and are reused across later units, and so every ABI-shape change (opaque list handles, `surface` field, `focused_only`, `ad_free_handle`, `AdRefEntry` completion) lands together and before publication. + +**Target branch:** `feat/c-bindings` (PR #22 is currently open). + +## Problem Frame + +The reviewer ([lahfir](https://github.com/lahfir), CHANGES_REQUESTED on PR #22) identified that the FFI crate as submitted is architecturally correct but contains multiple soundness bugs that are either undefined behavior, heap-corruption primitives, or silent data loss: + +- Flat-tree child ranges walk into grandchildren (API contract violation, silent wrong data) +- Error-string pointers dangle after any subsequent successful FFI call (use-after-free) +- `#[repr(i32)]` enum fields are read from C-supplied structs with no range validation (instant UB on invalid discriminant) +- Workspace release profile is `panic = "abort"`, making every panic inside an `extern "C"` fn immediately `SIGABRT` the host Python / Swift / Node process + +Additionally, several ABI-shape omissions (no `surface` field on `AdTreeOptions`, no `focused_only` on `ad_list_windows`, no `ad_free_handle`, no notification FFI, no observation primitives) would require breaking changes after publication. + +## Requirements Trace + +Numbered against the 24 findings in the origin review for traceability: + +- **R1.** (Blocker #1) Direct-child iteration over `AdNode.child_start..child_start+child_count` must return only direct children, not descendants. +- **R2.** (Blocker #2) `ad_last_error_{code,message,suggestion,platform_detail}` pointers must remain valid across subsequent successful FFI calls; only the next *failing* call invalidates them. +- **R3.** (Blocker #3) Every `#[repr(i32)]` enum field read from a C-supplied struct must be validated via range check before conversion; invalid discriminants return `AD_RESULT_ERR_INVALID_ARGS`. +- **R4.** (Blocker #4) Every `extern "C"` function wraps its body in `std::panic::catch_unwind`; the cdylib's release profile is overridden to `panic = "unwind"` per-package. +- **R5.** (Important #5) Every successful `ad_resolve_element` (and future `ad_find`) has a matching `ad_free_handle` that `CFRelease`s on macOS; core trait gains `release_handle(&NativeHandle)` method with platform impls. +- **R6.** (Important #6) Every FFI function that writes an out-param zeroes it at entry, before any fallible work. +- **R7.** (Important #7) Debug builds assert `pthread_main_np()` at every macOS-sensitive entry point; the header loudly documents the main-thread requirement. +- **R8.** (Important #8) List-returning FFI functions return opaque handles (`AdWindowList`, `AdAppList`, `AdSurfaceList`, `AdNotificationList`) with `_count` / `_get` / `_free` accessors; raw `ptr + count` APIs are replaced entirely. `AdImageBuffer` tracks its allocation length internally. +- **R9.** (Important #9) `AdTreeOptions` carries a `surface` field covering all seven `SnapshotSurface` variants. +- **R10.** (Important #10) `ad_list_windows` accepts `focused_only: bool` or a dedicated `ad_focused_window` fn exists. +- **R11.** (Important #11) FFI exposes `ad_list_notifications`, `ad_dismiss_notification`, `ad_dismiss_all_notifications`, `ad_notification_action`, `ad_find`, `ad_get`, `ad_is`. Remaining gaps (`wait_*`, `batch`, `press_key_for_app`, `get_live_value`) are explicit non-goals for this plan. +- **R12.** (Important #12) `skills/agent-desktop-ffi/SKILL.md` ships with the FFI, covering memory ownership rules, error pattern, thread-safety, and a build / link example. +- **R13.** (Important #13) `ad_window_to_core` validates `id` and `title` non-null; null/invalid-UTF-8 returns `AD_RESULT_ERR_INVALID_ARGS`. +- **R14.** (Important #14) `build.rs` fails loudly on cbindgen failure; cbindgen version is pinned exactly; CI regenerates the header and diffs against the committed copy. +- **R15.** (Important #15) Interior NUL bytes in mandatory string fields are replaced (U+FFFD or `?`) instead of returning null pointers; optional fields retain null-on-NUL. +- **R16.** (Smaller) Remove wildcard `pub use types::*` in `lib.rs`; use explicit re-exports. +- **R17.** (Smaller) Strip inline comments in `tree.rs`; keep only `///` doc-comments on public items. +- **R18.** (Smaller) Every C-visible optional string or sentinel field documents nullability and meaning in the generated header via `///` doc-comments on the Rust types. +- **R19.** (Smaller) Remove all `.unwrap()` / `.expect()` from non-test code in `crates/ffi/`. +- **R20.** (Smaller) Consolidate or remove misleading `#[allow(dead_code)]` annotations; document cdylib false-positive rationale at file scope where still needed. +- **R21.** (Smaller) Re-evaluate `unsafe impl Send + Sync for NativeHandle` now that FFI materializes the cross-thread risk; either enforce single-thread discipline or remove the impl. +- **R22.** (Smaller) `AdImageBuffer.data_len` is not read back from the C-mutable struct at free time. +- **R23.** (Smaller) Compile-time assertion that `ErrorCode::VARIANTS == AdResult::VARIANTS - 1` prevents silent misses when core adds an error variant. +- **R24.** (Nit) `CLAUDE.md` reflects that `crates/ffi/` is now the second platform → core wiring point. +- **R25.** (Plan-added, user request) The FFI crate is restructured to match the project's modular standards before any fixes land: one domain type per file, mirrored subfolder layout (tree/ actions/ input/ system/ equivalents), explicit per-item re-exports, zero inline `//` commentary, `///` doc-comments only where the name is insufficient, every file under 400 LOC, zero `unwrap`/`expect` in non-test code, and no unjustified `#[allow(dead_code)]`. This is the precondition for every later unit. + +## Scope Boundaries + +- **Phase 1 macOS only.** Windows and Linux adapter stubs exist in `crates/ffi/Cargo.toml`'s `[target.'cfg(...)']` deps but are compile-time only. Main-thread enforcement is macOS-specific (`pthread_main_np`); other platforms get a no-op shim. +- **No ABI stability guarantees.** The header carries a loud comment that the ABI is unstable until v1.0. No `SONAME` versioning, no `install_name` tricks — filename versioning is deferred to the 1.0 release plan. +- **Pre-1.0 remains pre-1.0.** Workspace version stays `0.1.x`. Adding the FFI artifact to the release pipeline is explicitly deferred (see below). + +### Deferred to Separate Tasks + +- **`ad_wait_*`, `ad_batch`, `ad_press_key_for_app`, `ad_get_live_value` FFI exposure** — follow-up PR after this one lands. Each needs its own API design (wait mode enum, batch JSON schema) and none block the v0 publication. +- **Release pipeline cdylib artifacts** — `libagent_desktop_ffi.{dylib,so,dll}` and `agent_desktop.h` are not currently produced or uploaded by `.github/workflows/release.yml`; adding them is a separate distribution plan (touches `release-please-config.json`, `.github/workflows/release.yml`, `npm/` postinstall, and needs cross-platform build matrix expansion). +- **`docs/solutions/` capture of FFI learnings** — runs after this plan merges via `ce:compound`. Ten candidate topics already identified; writing them is out of scope here. +- **`AdRefEntry` expansion** — core's `RefEntry` has `value`, `states`, `bounds`, `available_actions`, `source_app` fields that the FFI currently omits. They are not required by today's resolve path (resolver only needs `pid` + `role` + `name` + optional `bounds_hash`), so the minimal-but-sufficient shape is what this plan freezes. Expansion is a separate ABI addition if future resolver versions need richer context. +- **Log callback registration** (`ad_set_log_callback`) — foreign-exception propagation hazard documented in flow analysis; deferred to Phase 2 when a callback ABI is designed end-to-end. +- **`ad_version()` / `ad_abi_version()` exports** — useful for dlopen'd consumers but non-blocking for pre-1.0 fixes; bundle with the 1.0 release plan. +- **Cross-language integration tests** — Python ctypes, Go cgo, Swift bridging, Node ffi-napi, glibc-vs-musl matrices. Each is valuable but each is its own multi-day scaffold. This plan adds Rust-side coverage for every finding; cross-language E2E is a separate QA track. + +## Context & Research + +### Relevant Code and Patterns + +FFI crate (all in `crates/ffi/src/`): +- `lib.rs` — `AdAdapter { inner: Box<dyn PlatformAdapter> }` + `build_adapter()` parallel to `src/main.rs:136-154`. Current wildcard `pub use types::*`. +- `types.rs` — 28 `#[repr(C)]` structs + `#[repr(i32)]` enums. `AdTreeOptions` missing `surface`. `AdNativeHandle { ptr }` has no destructor path. +- `error.rs` — thread-local `StoredError` with paired `CString`s. `set_last_error`, `clear_last_error`, C exports. Line 56 has `.unwrap()` on an infallible `CString::new`. +- `convert.rs` — `string_to_c` (returns null on NUL), `c_to_str` (unbound lifetime declaration), `free_*_fields` helpers. File-scoped `#![allow(dead_code)]`. +- `tree.rs` — DFS flatten with broken child-range addressing. Recursive walker at `flatten_recursive` (line 26-67). `ad_get_tree` hardcodes `SnapshotSurface::Window` at line 158. +- `actions.rs` — enum converters, `ad_resolve_element` (leaks `CFRetain`ed element), `ad_execute_action` (no out-param zeroing), `ad_free_action_result`. +- `apps.rs` / `windows.rs` / `surfaces.rs` — `Box::from_raw(slice_from_raw_parts_mut(ptr, count))` with caller-supplied count → heap corruption primitive. `ad_free_window` and `ad_free_windows` naming collision. +- `windows.rs::ad_window_to_core` (line 10-28) — silent wrong-window match via `.unwrap_or("")` coercion of null title. +- `input.rs` — clipboard + mouse + drag. Current `ad_free_string` is only safe for clipboard-returned strings. +- `screenshot.rs` — `AdImageBuffer` C-mutable `data_len` field used at free time → heap corruption if consumer mutates. +- `build.rs` — `.expect()` on cbindgen config, `.ok()` swallows generation errors, writes into `include/agent_desktop.h` on every build. +- `cbindgen.toml` — `style = "both"`, `prefix_with_name = true`, `ScreamingSnakeCase`. +- `include/agent_desktop.h` — 461 LOC, committed; missing nullability / lifetime docs. + +Core crate: +- `crates/core/src/adapter.rs:59-92` — `NativeHandle` definition. `unsafe impl Send + Sync` with "Phase 4 rework" comment. No `Drop`. +- `crates/core/src/adapter.rs:15-25` — `SnapshotSurface` (7 variants). +- `crates/core/src/adapter.rs:10-13` — `WindowFilter { focused_only, app }`. +- `crates/core/src/adapter.rs:115-247` — `PlatformAdapter` trait (28 methods). `find_element` / `batch` are NOT trait methods — live in `crates/core/src/commands/{find,batch}.rs` as tree walkers + dispatchers. +- `crates/core/src/notification.rs` — `NotificationInfo { index, app_name, title, body, actions }`, `NotificationFilter { app, text, limit }`. +- `crates/core/src/error.rs` — `ErrorCode` (12 variants, declaration order matches `AdResult = -1..=-12`). +- `crates/core/src/refs.rs` — `RefEntry { pid, role, name, bounds_hash, value, states, bounds, available_actions, source_app }`. Current FFI omits the last five. + +macOS crate: +- `crates/macos/src/tree/element.rs:24-39` — `AXElement` with `Drop`+`Clone` doing `CFRelease`/`CFRetain`. Pattern for `NativeHandle` `release_handle`. +- `crates/macos/src/tree/resolve.rs:88` — `CFRetain(el.0)` before wrapping in `NativeHandle::from_ptr`. Defines retain ownership: caller must release exactly once. +- `crates/macos/src/adapter.rs:38-52` — `get_tree` wires each `SnapshotSurface` to a `crates/macos/src/tree/surfaces.rs` function. +- `crates/macos/src/adapter.rs:131-161` — `get_live_value` / `get_element_bounds` use `ManuallyDrop<AXElement>` to borrow without double-release. Pattern to follow. + +Binary crate reference: +- `src/main.rs:136-154` — `build_adapter()` that FFI's `lib.rs` parallels; single source of truth pattern the CLAUDE.md "only binary wires platform → core" rule assumes (now violated by FFI). +- `src/batch.rs` parses batch commands through the typed CLI command path. + +CI / release: +- `.github/workflows/ci.yml` — Dependency isolation check only guards `agent-desktop-core`. No cdylib build. No header-drift check. +- `.github/workflows/release.yml` — ships CLI tarballs + npm; no cdylib artifact slot. +- `Cargo.toml` release profile — `panic = "abort"`, `lto = true`, `opt-level = "z"`. The `panic = "abort"` makes `catch_unwind` a no-op in release builds unless overridden per-package. + +### Institutional Learnings + +None. `docs/solutions/` does not yet exist in this repo. This fix cycle is the first time C-ABI concerns touch the codebase and should seed that directory via `ce:compound` after merge. + +### External References + +Not consulted (the origin review is external-quality grounding and cites every external pattern it recommends: `catch_unwind` + `panic=unwind` per-package override, errno-style thread-local error lifetimes, `pthread_main_np()` assertions, `Box::into_raw`/`Box::from_raw` ownership discipline, cbindgen version pinning for drift-free output). + +## Key Technical Decisions + +- **Panic strategy: dedicated `[profile.release-ffi]` inheriting from release.** Cargo rejects per-package `panic` overrides (`error: panic may not be specified in a package profile` — verified 2026-04-16). The workaround is a custom profile: `[profile.release-ffi] inherits = "release" panic = "unwind"`. The cdylib is built via `cargo build --profile release-ffi -p agent-desktop-ffi`; the CLI still builds via `cargo build --release` under the unchanged `panic = "abort"` workspace profile. Spike-verified: a panic inside an `extern "C"` fn wrapped in `catch_unwind` returns cleanly under release-ffi (exit 0). CI + release workflow must build the cdylib with the explicit `--profile release-ffi` flag. Size measured: 468 KB cdylib, 1.1 MB CLI — both well under limits. Alternative considered and rejected: drop workspace `panic = "abort"` entirely — the CLI would gain unwind metadata for no user-visible benefit, and the release binary budget is tight (15 MB gate). +- **Error lifetime: keep last-error intact on success paths.** Remove `clear_last_error()` calls from every success path. The last-error slot only rotates when a *new* error is set. Matches the C `errno` contract. +- **Enum validation: `TryFrom<i32>` generated by declarative macro.** A single `try_from_c_enum!(AdActionKind, 0..=20)`-style macro per enum generates the `TryFrom<i32>` impl. Each extern fn reads the field as `i32`, calls `try_into()`, returns `ErrInvalidArgs` on `Err`. +- **Panic boundary: macro, not closure wrapper.** A `#[ffi_entrypoint]` macro (or `ffi_try!` declarative macro) wraps every extern fn body in `catch_unwind` and translates unwinds to `AD_RESULT_ERR_INTERNAL`. Using a macro keeps each function's prose unchanged; a closure wrapper forces every fn to be rewritten. +- **Tree layout: BFS with contiguous sibling runs.** The flat array is emitted level-by-level (breadth-first), so `nodes[child_start..child_start+child_count]` enumerates direct children exactly. Iterative traversal with `VecDeque` (not recursion) also resolves the review's nit about stack depth on thin-stack consumers (JNI, Python ctypes). +- **Opaque list handles.** `AdWindowList`, `AdAppList`, `AdSurfaceList`, `AdNotificationList` each wrap a `Box<[T]>` inside a forward-declared struct. `_count(list)`, `_get(list, i)`, `_free(list)` accessors. Caller never sees the backing pointer or length; count mismatches become impossible by construction. +- **`ad_free_handle` + core trait addition.** `PlatformAdapter` gains `release_handle(&self, handle: &NativeHandle) -> Result<(), AdapterError>` with default `not_supported()` impl. macOS impl calls `CFRelease(ptr as CFTypeRef)`. FFI exports `ad_free_handle(adapter, handle)`. Explicit release (not `Drop`) because `AdNativeHandle { ptr }` is a C struct the consumer owns — Rust has no lifecycle hook into it. +- **Pre-freeze `AdTreeOptions.surface` + `ad_list_windows` focused_only.** Both ABI additions land together as "freeze the struct shape" work. Adding them after publication breaks every consumer compiled against the old layout. +- **Notifications: full CRUD on first release.** All four notification commands become FFI fns in this plan. The review's stance: "ship notifications in this PR" — followed. +- **`find` / `get` / `is`: FFI-side implementation, no core trait additions.** `find` is implemented as an iterative walk over the flat tree inside the FFI. `get` reuses `get_live_value` + `get_element_bounds`. `is` is a state-bitset check on the resolved element. No platform-layer additions needed. Matches existing `commands/find.rs` approach. +- **Main-thread check: macOS `pthread_main_np()` debug_assert.** No-op on other platforms (Windows UIA and Linux AT-SPI don't require main thread). Loud `///` comment in header + skill doc explains the macOS constraint to consumers. +- **Header drift detection: CI regen + `git diff --exit-code`.** `build.rs` continues to copy into `include/agent_desktop.h`. CI runs a fresh `cargo build -p agent-desktop-ffi` and fails if the working tree has any diff afterward. Cbindgen pinned exactly (`cbindgen = "= 0.27.0"`) to eliminate formatting drift across CI images. +- **`NativeHandle` `Send + Sync` stance.** Keep the `unsafe impl` but narrow its justification comment: "Phase 1: handles are single-owner and single-thread by FFI contract. Consumers violating this invoke undefined behavior." Debug assertion in `release_handle` verifies current-thread equals creation-thread. +- **Interior NUL policy.** Mandatory fields (role, name on notifications, app_name, etc.) use lossy replace — NUL byte → U+FFFD UTF-8 sequence. Optional fields (value, description, hint) keep the current null-on-NUL behavior because the header will explicitly document nullability. + +## Open Questions + +### Resolved During Planning + +- **Macro vs runtime wrapper for `catch_unwind`?** Declarative macro `ffi_try!` wrapping the fn body. Chosen over proc-macro attribute to keep build times sane and debugging straightforward. +- **Expand `AdRefEntry` to match `RefEntry`?** No. Minimal-but-sufficient set freezes today's actual resolver input (`pid`, `role`, `name`, `bounds_hash`); the other fields are resolver output metadata the FFI doesn't need yet. +- **Add `ad_focused_window()` or `focused_only` flag?** Flag on `ad_list_windows`. One function, one API surface, no duplicate paths. +- **`AdAction` ergonomics: tagged union vs constructor helpers?** Keep the current variant-struct shape (simpler ABI, cbindgen-friendly) but add `ad_action_click()`, `ad_action_type_text(text)`, `ad_action_scroll(dir, amount)`, `ad_action_press(combo)`, `ad_action_drag(from, to, duration_ms)` constructor helpers that zero-init unused fields. Python / Go consumers no longer have to construct the full union manually. Rust caller-side tests verify each helper produces the expected `AdAction` layout. +- **Generated header: commit or gitignore?** Keep committed. CI regens + diffs. Alternative (OUT_DIR only + separate xtask) adds tooling for marginal benefit. + +### Deferred to Implementation + +- **Exact macro shape for `ffi_try!`** — final form depends on whether `std::panic::catch_unwind` needs `AssertUnwindSafe` wrappers around captured `adapter: *const AdAdapter` pointers. Work out during Unit 1. +- **Whether `release_handle` needs to tolerate null pointers** — decide when writing macOS impl; likely yes (matches `ad_adapter_destroy` null-tolerance). +- **Whether the compile-time variant-count assertion (R23) needs `variant_count` nightly feature or a manual match-arm count.** Prefer stable-only approach (manual match that compiles to a constant); decide in Unit 12. +- **BFS traversal: recursive build-list-then-emit vs single-pass queue** — both produce correct output; pick based on which is clearer. Benchmark if 10k-node Electron tree builds non-trivially slower. +- **Whether `ad_free_handle` is adapter-scoped or global** — `adapter.release_handle(handle)` means consumer must pass the adapter that created the handle. Mildly awkward but avoids a thread-local registry. Lock in during Unit 7. + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +**Panic boundary macro shape (Unit 1):** + +``` +macro_rules! ffi_try { ($body:block) => { { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| $body)) { + Ok(result_code) => result_code, + Err(_) => { + error::set_last_error_static("rust panic in FFI boundary"); + AdResult::ErrInternal + } + } +}}} + +// Usage per extern fn: +#[no_mangle] +pub unsafe extern "C" fn ad_foo(...) -> AdResult { ffi_try!({ + // existing body unchanged, must end in AdResult +})} +``` + +**BFS flat-tree layout (Unit 4):** + +`root → [a → [a1, a2], b]` becomes: + +| index | role | parent_index | child_start | child_count | +|---|---|---|---|---| +| 0 | root | -1 | 1 | 2 | +| 1 | a | 0 | 3 | 2 | +| 2 | b | 0 | 5 | 0 | +| 3 | a1 | 1 | 5 | 0 | +| 4 | a2 | 1 | 5 | 0 | + +`root.children = nodes[1..3] = [a, b]` ✅ +`a.children = nodes[3..5] = [a1, a2]` ✅ + +Siblings are always contiguous. Child ranges always address *direct* children. The same layout holds for any tree shape. + +**Opaque list handle shape (Unit 5):** + +``` +// Rust side (not exposed to header): +pub struct AdWindowList { + items: Box<[AdWindowInfo]>, +} + +// Header (cbindgen emits forward declaration): +typedef struct AdWindowList AdWindowList; + +AdResult ad_list_windows(adapter, app_filter, focused_only, AdWindowList** out); +uint32_t ad_window_list_count(const AdWindowList* list); +const AdWindowInfo* ad_window_list_get(const AdWindowList* list, uint32_t index); +void ad_window_list_free(AdWindowList* list); +``` + +No caller-supplied `count` → count mismatches impossible. Same shape for `AdAppList`, `AdSurfaceList`, `AdNotificationList`. + +## Output Structure + +New files introduced by this plan, repo-relative: + + crates/ffi/src/ + ├── ffi_try.rs # Unit 1: panic boundary macro + static error helper + ├── enum_validation.rs # Unit 2: TryFrom<i32> macro + per-enum impls + ├── notifications.rs # Unit 8: notification FFI surface + ├── observation.rs # Unit 9: ad_find / ad_get / ad_is + └── action_helpers.rs # Unit 10: ad_action_click / _type_text / etc. + + crates/ffi/tests/ + └── layout_contract.rs # Cross-unit: asserts C struct layouts (size/align/offsets) + + crates/core/src/ + (adapter.rs modified for release_handle trait method) + + crates/macos/src/ + (tree/element.rs or new adapter method for release_handle impl) + + skills/agent-desktop-ffi/ + ├── SKILL.md # Unit 14 + └── references/ + ├── ownership.md # Who allocates / who frees for every pointer type + ├── error-handling.md + └── threading.md + + .github/workflows/ + (ci.yml modified for header-drift check) + +This tree is a scope declaration. Implementers may reshape if a better layout emerges — the per-unit `**Files:**` sections remain authoritative. + +## Implementation Units + +### Phase 0 — Refactor to modular CLAUDE.md-compliant structure + +Precondition for every later unit. Lands in a single reviewable PR (or a small stack if size dictates) that **changes no behavior** — only moves, splits, strips commentary, and tightens pub boundaries. Passes `cargo clippy --all-targets -- -D warnings`, `cargo test --lib -p agent-desktop-ffi`, and the cbindgen-generated header diff should be byte-identical or differ only in ordering (no new / removed / renamed symbols). + +- [ ] **Unit R: Modular refactor + CLAUDE.md alignment** + +**Goal:** Bring `crates/ffi/` to the same subfolder discipline the platform crates use. One domain type per file. No inline comments. Explicit pub boundaries. Zero `unwrap`. No dead-code annotations unless justified. Every file under 400 LOC with headroom for the fixes that follow. + +**Requirements:** R25 (+ pre-satisfies the file-level mechanics of R16, R17, R19, R20; those R-rows stay in their later units for the non-mechanical portions — explicit re-exports of *new* types from Unit 5+, inline comments introduced by the BFS rewrite, `unwrap`s introduced during fixes.) + +**Dependencies:** None. Must land first. + +**Files — target structure after refactor:** + +``` +crates/ffi/src/ +├── lib.rs # mod declarations + explicit re-exports only (<80 LOC) +├── adapter.rs # AdAdapter wrapper + build_adapter + adapter create/destroy/check_permissions +├── error.rs # AdResult + StoredError + last-error surface (no unwrap, CStr fallback) +├── types/ +│ ├── mod.rs # re-exports only (one pub use per type, no wildcard) +│ ├── point.rs # AdPoint +│ ├── rect.rs # AdRect +│ ├── native_handle.rs # AdNativeHandle +│ ├── ref_entry.rs # AdRefEntry +│ ├── action_kind.rs # AdActionKind +│ ├── direction.rs # AdDirection +│ ├── modifier.rs # AdModifier +│ ├── key_combo.rs # AdKeyCombo +│ ├── scroll_params.rs # AdScrollParams +│ ├── drag_params.rs # AdDragParams +│ ├── action.rs # AdAction +│ ├── element_state.rs # AdElementState +│ ├── action_result.rs # AdActionResult +│ ├── app_info.rs # AdAppInfo +│ ├── window_info.rs # AdWindowInfo +│ ├── window_op_kind.rs # AdWindowOpKind +│ ├── window_op.rs # AdWindowOp +│ ├── surface_info.rs # AdSurfaceInfo +│ ├── mouse_button.rs # AdMouseButton +│ ├── mouse_event_kind.rs # AdMouseEventKind +│ ├── mouse_event.rs # AdMouseEvent +│ ├── image_format.rs # AdImageFormat +│ ├── image_buffer.rs # AdImageBuffer +│ ├── screenshot_kind.rs # AdScreenshotKind +│ ├── screenshot_target.rs # AdScreenshotTarget +│ ├── node.rs # AdNode +│ ├── node_tree.rs # AdNodeTree +│ └── tree_options.rs # AdTreeOptions +├── convert/ +│ ├── mod.rs # re-exports +│ ├── string.rs # c-string helpers (string_to_c, opt_string_to_c, c_to_str, free_c_string) +│ ├── rect.rs # rect_to_c +│ ├── window.rs # window_info_to_c, free_window_info_fields +│ ├── app.rs # app_info_to_c, free_app_info_fields +│ └── surface.rs # surface_info_to_c, free_surface_info_fields +├── tree/ +│ ├── mod.rs # re-exports +│ ├── flatten.rs # AccessibilityNode → AdNodeTree +│ ├── get.rs # ad_get_tree +│ └── free.rs # ad_free_tree +├── actions/ +│ ├── mod.rs # re-exports +│ ├── conversion.rs # direction_from_c, key_combo_from_c, action_from_c +│ ├── resolve.rs # ad_resolve_element +│ ├── execute.rs # ad_execute_action +│ ├── result.rs # action_result_to_c, ad_free_action_result +│ └── native_handle.rs # (Unit 6c) ad_free_handle — placeholder file added empty in Unit R so Unit 6c is a pure insert +├── apps/ +│ ├── mod.rs +│ ├── list.rs # ad_list_apps + ad_free_apps (pre-Unit 5a; Unit 5a replaces with opaque list) +│ ├── launch.rs # ad_launch_app +│ └── close.rs # ad_close_app +├── windows/ +│ ├── mod.rs +│ ├── to_core.rs # ad_window_to_core helper +│ ├── list.rs # ad_list_windows + ad_free_windows (pre-Unit 5b) +│ ├── focus.rs # ad_focus_window +│ ├── free_one.rs # ad_free_window (single-struct — will be renamed to ad_release_window_fields in Unit 5b) +│ └── op.rs # ad_window_op +├── input/ +│ ├── mod.rs +│ ├── clipboard.rs # ad_get_clipboard, ad_set_clipboard, ad_clear_clipboard, ad_free_string +│ ├── mouse.rs # ad_mouse_event, mouse_button_from_c +│ └── drag.rs # ad_drag +├── screenshot/ +│ ├── mod.rs +│ ├── capture.rs # ad_screenshot +│ └── free.rs # ad_free_image +└── surfaces/ + ├── mod.rs + ├── list.rs # ad_list_surfaces + └── free.rs # ad_free_surfaces +``` + +**Migration map (what goes where):** + +| Current file | New home | +|---|---| +| `crates/ffi/src/lib.rs` (adapter logic) | `crates/ffi/src/adapter.rs` | +| `crates/ffi/src/lib.rs` (re-exports) | `crates/ffi/src/lib.rs` (kept; wildcards removed, explicit list added) | +| `crates/ffi/src/types.rs` (28 items) | `crates/ffi/src/types/*.rs` (one per struct/enum) | +| `crates/ffi/src/convert.rs` | `crates/ffi/src/convert/{string,rect,window,app,surface}.rs` | +| `crates/ffi/src/tree.rs` | `crates/ffi/src/tree/{flatten,get,free}.rs` | +| `crates/ffi/src/actions.rs` | `crates/ffi/src/actions/{conversion,resolve,execute,result}.rs` | +| `crates/ffi/src/apps.rs` | `crates/ffi/src/apps/{list,launch,close}.rs` | +| `crates/ffi/src/windows.rs` | `crates/ffi/src/windows/{to_core,list,focus,free_one,op}.rs` | +| `crates/ffi/src/input.rs` | `crates/ffi/src/input/{clipboard,mouse,drag}.rs` | +| `crates/ffi/src/screenshot.rs` | `crates/ffi/src/screenshot/{capture,free}.rs` | +| `crates/ffi/src/surfaces.rs` | `crates/ffi/src/surfaces/{list,free}.rs` | + +**Approach:** +- Refactor in one atomic commit per subtree (types → convert → tree → actions → apps → windows → input → screenshot → surfaces → adapter). Small, reviewable steps; each commit compiles + passes clippy. +- **No behavior change.** Every FFI symbol keeps the same signature and same module path as seen by C consumers (cbindgen treats Rust module paths as invisible; the generated header stays identical). Run `cargo build -p agent-desktop-ffi` after each sub-commit and `diff include/agent_desktop.h` — must be empty (or ordering-only if cbindgen's topological sort changes). +- **Inline comments stripped.** Every `//` that isn't a `///` doc-comment on a public item is deleted. A `///` doc-comment is retained only when the name alone doesn't communicate intent — e.g., `/// # Safety` blocks on `unsafe extern "C" fn`, lifetime documentation on the `ad_last_error_*` errno-style semantics. Descriptive rehash-of-code comments are slop and go. +- **Explicit re-exports.** `lib.rs` line `pub use types::*` is replaced with: + ```rust + pub use types::action::AdAction; + pub use types::action_kind::AdActionKind; + pub use types::action_result::AdActionResult; + // ...one per exported type + ``` + `types/mod.rs` does the same: `pub mod point; pub mod rect; ...` (each submodule is `pub(crate)` internally; the type name is re-exported via explicit `pub use`). +- **Zero `unwrap` / `expect`** in non-test code. `error.rs:56` `.unwrap()` on the infallible-today `CString::new("(message contained null byte)")` becomes a `'static CStr` literal: `static NUL_BYTE_FALLBACK: &CStr = c"(message contained null byte)";` and `set_last_error` falls back to it via a `MessageSource::Static` variant (or equivalent — the exact shape stays the implementer's call). `build.rs` `.expect()` calls are replaced with graceful handling that still fails the build loudly (see Unit 12 for the full safety pass; Unit R only removes the `.unwrap()`s, doesn't rewrite `build.rs` end-to-end). +- **`#[allow(dead_code)]` audit.** Every annotation in `crates/ffi/src/` is checked against actual reachability from a `#[no_mangle]` export. Reachable functions lose the annotation. Remaining annotations (cdylib false positives on `pub(crate)` helpers called across module boundaries that linker visibility hides from rustc) are consolidated to a single `#![allow(dead_code)]` at the file head with a one-line `// cdylib false-positive: see docs/solutions/cdylib-dead-code.md` explanation (or a `///` doc-comment at crate level if one exists). No sprayed per-item annotations. +- **`c_to_str` unbound lifetime** is fixed in this unit (pre-existing footgun): signature becomes `fn c_to_str(ptr: *const c_char) -> Option<&'_ str>` with an elided-but-bounded lifetime tied to `ptr`, or returns `Option<String>` (always cloned). Pick the cloned-return variant for simplicity; it's only called by short-lived `to_owned()` paths today. +- **Naming normalization.** `ad_free_window` (singular, for the struct from `ad_launch_app`) stays named `ad_free_window` in this unit but the docstring clarifies it's interior-fields-only. Unit 5b renames it to `ad_release_window_fields`. This unit doesn't rename to keep the no-behavior-change invariant intact. +- **cbindgen.toml** left untouched. The header regeneration after refactor must diff to empty (or ordering-only). If cbindgen's output changes because of module-path-based symbol ordering, commit the re-ordered header as part of Unit R's final commit and note it in the PR body. +- **`#[macro_use]` placement** for later `ffi_try!` macro (Unit 2) is left to that unit. Unit R doesn't introduce new macros. + +**Patterns to follow:** +- `crates/macos/src/` layout — the direct template for the ffi refactor. Note the `tree/`, `actions/`, `input/`, `system/` subfolders mirror what ffi/ now becomes. +- `crates/macos/src/lib.rs` — mod declarations + explicit re-exports only; no behavior. +- `crates/macos/src/adapter.rs` — single responsibility (PlatformAdapter impl); same shape for `crates/ffi/src/adapter.rs` (AdAdapter wrapper + lifecycle FFI exports). +- `crates/core/src/commands/mod.rs` — one command per file pattern. + +**What "slop" means here (the cleanup list):** +- Inline `//` comments that narrate what the next line does (`// free the boxed slice`, `// convert to C`, etc.) — delete. +- Redundant module-level `#[allow(dead_code)]` when items are reachable — delete. +- `TODO`/`FIXME`/`XXX` markers without ticket references — delete or replace with `tracing::warn!` if the note is truly operational. +- Wildcard imports / re-exports — replace with explicit lists. +- Duplicated helper logic between modules (e.g., null-check + early-return boilerplate) — consolidate into a single helper in `convert/string.rs` or `error.rs`. +- `let _ = ...;` for deliberate result-drops — replace with `drop(...)` to be explicit about intent; prefer not silently discarding `Result`s at all where `?`-propagation works. +- Self-documenting obvious code wrapped in comments — delete. +- Multi-line `//!` or `///` docstrings on implementation details that don't cross the FFI boundary — condense to a single sentence or remove. + +**Test scenarios:** +- Happy path: `cargo build -p agent-desktop-ffi` compiles clean after refactor +- Happy path: `cargo test --lib -p agent-desktop-ffi` — all 26 existing tests still pass unchanged +- Happy path: `cargo clippy --all-targets -p agent-desktop-ffi -- -D warnings` passes +- Happy path: generated `include/agent_desktop.h` byte-identical (or ordering-only diff) pre- and post-refactor. `git diff include/agent_desktop.h | grep -E '^[+-](typedef|struct|enum|AD_|ad_)'` returns zero lines (or only lines where order changed but no additions/removals) +- Verification: `grep -rn '^//[^/!]' crates/ffi/src/` returns only `// SAFETY:` comments inside `unsafe` blocks (and no `// descriptive-text-about-next-line` comments) +- Verification: `grep -rn '\.unwrap()\|\.expect(' crates/ffi/src/ --include='*.rs' | grep -v '#\[cfg(test)\]'` returns zero hits outside `#[cfg(test)]` blocks +- Verification: `grep -rn 'pub use .*::\*' crates/ffi/src/` returns zero hits +- Verification: `wc -l crates/ffi/src/**/*.rs` — every file under 400 LOC (and most should be under 150, with headroom for Units 1-13 additions) + +**Verification:** Refactor commit(s) land on `feat/c-bindings` before any Unit 1+ work begins. The PR description notes the refactor is behavior-neutral. A reviewer who diffs the generated header sees no ABI change. + +--- + +### Phase 1 — Safety Foundation + +- [ ] **Unit 0: Publish guard + custom FFI release profile** + +**Goal:** Cargo.toml shape-changes land before Unit 1's macro work. Prevent accidental `cargo publish` of the cdylib until the distribution plan lands. Make `release-ffi` profile the canonical build mode for the cdylib. + +**Requirements:** R4 (profile prerequisite), scope-guardian P1 (publish guard) + +**Dependencies:** None. + +**Files:** +- Modify: `Cargo.toml` (add `[profile.release-ffi] inherits = "release" panic = "unwind"`) +- Modify: `crates/ffi/Cargo.toml` (add `[package] publish = false` — prevents accidental cargo-publish; removed when distribution plan lands) +- Modify: `crates/ffi/Cargo.toml` (add `libc = "0.2"` under `[target.'cfg(target_os = "macos")'.dependencies]` for `pthread_main_np` used by Unit 11) + +**Approach:** +- Custom profile; CLI release profile unchanged. +- `publish = false` in FFI crate prevents `cargo publish -p agent-desktop-ffi` until the distribution plan explicitly removes it. +- Pre-verified: spike compile of an extern fn with `catch_unwind` under `release-ffi` caught the panic and returned 0; under the `release` profile (panic=abort) the same code aborts. + +**Patterns to follow:** +- Existing `[profile.release]` block in `Cargo.toml` lines 22-27. + +**Test scenarios:** +- Happy path: `cargo build --profile release-ffi -p agent-desktop-ffi` compiles clean +- Happy path: `cargo build --release` still compiles the CLI under `panic = "abort"` (both profiles coexist) +- Error path: `cargo publish -p agent-desktop-ffi --dry-run` refuses (publish=false) + +**Verification:** Both profiles build without errors; `cargo publish --dry-run` on ffi crate returns an error citing publish=false. + +--- + +- [ ] **Unit 1: Panic-unwind cdylib boundary** + +**Goal:** Every `extern "C"` fn catches Rust panics and returns `AD_RESULT_ERR_INTERNAL` instead of aborting the host process. + +**Requirements:** R4 + +**Dependencies:** Unit 0 (custom profile must exist). + +**Files:** +- Create: `crates/ffi/src/ffi_try.rs` (macro + static last-error helper for panic path) +- Modify: `crates/ffi/src/lib.rs` (declare `mod ffi_try`, re-export `ffi_try!`) +- Modify: `crates/ffi/src/error.rs` (add `set_last_error_static(&'static CStr)` that writes a pointer to a `'static CStr` — never allocates, safe for panic paths) +- Modify: each of the 8 module files containing `#[no_mangle] pub extern "C" fn` exports — wrap every extern body in `ffi_try!`: `lib.rs` (adapter create/destroy, check_permissions), `tree.rs` (get_tree, free_tree), `actions.rs` (resolve_element, execute_action, free_action_result, free_handle), `apps.rs` (list_apps, launch_app, close_app + new opaque list accessors), `windows.rs` (list_windows, focus_window, window_op + new opaque list accessors), `input.rs` (clipboard, mouse, drag, free_string), `screenshot.rs` (screenshot, free_image), `surfaces.rs` (list_surfaces + new opaque list accessors) +- Test: `crates/ffi/src/ffi_try.rs` (unit tests) + `crates/ffi/tests/panic_boundary.rs` (integration; reuses the spike example) +- Promote: `crates/ffi/examples/panic_spike.rs` (from Unit 0 verification) stays as a permanent regression example showing `catch_unwind` working under `release-ffi` + +**Approach:** +- Declarative macro `ffi_try!` wraps a block in `catch_unwind(AssertUnwindSafe(|| { $body }))`. +- On panic: set last-error to a `'static CStr` `"rust panic in FFI boundary"` via pointer-to-static (never allocates — allocation inside the panic handler could double-panic). Payload is discarded intentionally; no panic-sourced data reaches the FFI error surface. +- Custom profile `release-ffi` is the only way `catch_unwind` actually catches in optimized builds; workspace profile stays `abort` for the CLI binary. +- Two-phase guard: `ad_adapter_destroy` and all `ad_free_*` fns use the macro too; panicking inside `Drop` during destroy must not escape. +- Transitive-dep audit: verify no `crates/ffi` dependency registers a callback invoked from foreign-C frames (CF run loops, ObjC exception bridging via `objc2`, Unix signal handlers). If any such path exists, `catch_unwind`-at-entry is insufficient and the plan must add inner catch_unwind around the callback. Document the audit result in Unit 1's verification. + +**Patterns to follow:** +- No existing catch_unwind pattern in workspace — this unit establishes it. +- Mirror the `#[no_mangle] pub unsafe extern "C" fn` signature pattern that every current FFI export uses; the macro sits inside the fn body, not decorating the signature. + +**Test scenarios:** +- Happy path: `ffi_try!` returning `AdResult::Ok` propagates correctly +- Happy path: `ffi_try!` returning `AdResult::ErrInvalidArgs` propagates correctly with last-error already set by fn body +- Error path: `ffi_try!` body calls `panic!("synthetic panic")` → return value is `AD_RESULT_ERR_INTERNAL`, `ad_last_error_message()` returns `"rust panic in FFI boundary"` +- Error path: `ffi_try!` body triggers arithmetic overflow in debug → same translation +- Error path: two sequential panicking calls do not leak or double-set last-error +- Integration: release build with the per-package profile override actually catches panic (not aborts); integration test uses `std::process::Command` to run a test binary that panics inside an extern fn, asserts exit code 0 (caught) not 134/SIGABRT +- Integration: debug build also catches (workspace default `panic = "unwind"` in debug) + +**Verification:** Cargo `release` profile show-config for `agent-desktop-ffi` reports `panic = "unwind"`; panic test binary exits with `AD_RESULT_ERR_INTERNAL` in both debug and release. + +--- + +- [ ] **Unit 2: Enum validation at C boundary** + +**Goal:** No invalid `#[repr(i32)]` discriminant from C ever reaches Rust enum code; all out-of-range values return `AD_RESULT_ERR_INVALID_ARGS`. + +**Requirements:** R3 + +**Dependencies:** Unit 1 (panic boundary already catches any miss, but explicit validation is safer and diagnoses better). + +**Files:** +- Create: `crates/ffi/src/enum_validation.rs` (declarative macro generating `TryFrom<i32>` + variant-range check for every `#[repr(i32)]` C enum) +- Modify: `crates/ffi/src/types.rs` (apply macro invocations to `AdActionKind`, `AdDirection`, `AdModifier`, `AdMouseButton`, `AdMouseEventKind`, `AdWindowOpKind`, `AdScreenshotKind`, `AdImageFormat` — 8 enums) +- Modify: `crates/ffi/src/actions.rs`, `windows.rs`, `input.rs`, `screenshot.rs`, `tree.rs` (all FFI fns that read enum fields from C structs) +- Test: `crates/ffi/src/enum_validation.rs` (unit tests) + +**Approach:** +- Every enum-typed field on a C struct is read as `i32`, then `try_into()` or macro-generated `AdActionKind::from_c(raw: i32)` returning `Option<AdActionKind>`. +- On `None`: `error::set_last_error(AdapterError::new(ErrorCode::InvalidArgs, "unknown X enum value: N"))`, return `AD_RESULT_ERR_INVALID_ARGS`. +- Preserve the existing exhaustive `match` arms in each fn body — they run only after validation succeeds. + +**Patterns to follow:** +- Existing `mouse_button_from_c` / `direction_from_c` in `crates/ffi/src/{input,actions}.rs` — these are the functions to reshape (they currently trust the enum, fix is to validate first). + +**Test scenarios:** +- Happy path: valid discriminant for each of the 8 enums round-trips correctly +- Edge case: first valid variant (0) and last valid variant (enum's max) accepted +- Error path: discriminant = -1 returns `ErrInvalidArgs` with diagnostic last-error +- Error path: discriminant = 999 returns `ErrInvalidArgs` +- Error path: discriminant = `i32::MAX` returns `ErrInvalidArgs` +- Integration: calling `ad_execute_action` with `AdAction.kind = 999` returns `ErrInvalidArgs`, doesn't reach the match arms, doesn't UB + +**Verification:** Fuzz test (in-Rust, not C) that iterates `-100..=100` through each enum slot and asserts no panic, no UB marker (Miri-clean), correct return code. + +--- + +- [ ] **Unit 3: Errno-style last-error lifetime** + +**Goal:** `ad_last_error_{code,message,suggestion,platform_detail}` pointers remain valid across every subsequent successful FFI call; only the next *failing* call overwrites. + +**Requirements:** R2 + +**Dependencies:** Unit 1 (panic-path sets static last-error; that code must be aware of the new lifetime contract). + +**Files:** +- Modify: `crates/ffi/src/error.rs` (remove `clear_last_error` from success paths by no longer providing it; consolidate to `set_last_error` replacement on failure) +- Modify: every FFI fn in `crates/ffi/src/{apps,windows,tree,actions,input,screenshot,surfaces,lib}.rs` that currently calls `clear_last_error()` on success — delete those calls +- Modify: `crates/ffi/include/agent_desktop.h` (via rustdoc on `ad_last_error_*` functions — document lifetime semantics) +- Test: `crates/ffi/src/error.rs` + new integration test `crates/ffi/tests/error_lifetime.rs` + +**Approach:** +- Current behavior: every successful FFI call drops the thread-local `StoredError`, invalidating pointers consumer may still hold → UAF. +- New behavior: successful calls leave the last-error slot untouched. Consumers read last-error after a failure and can keep reading the same pointers across any number of successful calls until a new failure occurs. +- Matches POSIX `errno` semantics exactly. +- `clear_last_error` stays as a private helper usable only when explicitly starting a new error chain (in practice: only called by `ad_adapter_create` to start fresh). + +**Patterns to follow:** +- POSIX errno (`<errno.h>`) — the canonical model. Every library exposing errno-like last-error follows this contract. + +**Test scenarios:** +- Happy path: failing call sets last-error; pointer returned by `ad_last_error_message()` is readable +- Happy path: after ten successful calls following a failure, `ad_last_error_message()` returns the same non-dangling string +- Happy path: a second failing call overwrites the previous last-error; the old pointer is no longer guaranteed valid +- Edge case: thread-local isolation — Thread A's last-error is not visible to Thread B (existing behavior, confirm preserved) +- Edge case: panic path (from Unit 1) sets last-error to static string; subsequent successful calls don't invalidate the static pointer +- Integration: C harness that reproduces the UAF pattern (`ad_foo` fails → read message → call `ad_check_permissions` succeeds → use cached message pointer) — must not segfault or read garbage + +**Verification:** Integration test simulates the UAF pattern from the review and passes under Miri / ASan. + +### Phase 2 — Tree Correctness + +- [ ] **Unit 4: BFS flat-tree layout with iterative traversal** + +**Goal:** `AdNode.child_start..child_start + child_count` over `AdNodeTree.nodes` returns direct children only — no grandchildren leak, no siblings skipped. Handles trees of arbitrary depth without Rust stack pressure from consumers on thin stacks (JNI ~256KB, Python ctypes worker threads). + +**Requirements:** R1 + +**Dependencies:** None (tree.rs is self-contained; depends only on established panic boundary from Unit 1 and the `try_into` enum pattern from Unit 2 if surface enum is involved — but tree.rs doesn't consume a surface enum today). + +**Files:** +- Modify: `crates/ffi/src/tree.rs` (rewrite `flatten_tree` + `flatten_recursive` as iterative BFS; update `AdNode` field semantics to BFS layout) +- Modify: rename test `test_flatten_depth_first_order` → `test_flatten_breadth_first_layout`; update to iterate actual child ranges (`nodes[child_start..child_start+child_count]`) and assert direct-child roles (not just field values); add multi-level nested test and wide-shallow test +- (Inline-comment strip in rewritten tree.rs is handled in Unit 13 alongside the other doc-cleanup work — avoids duplication) +- Test: `crates/ffi/src/tree.rs` (unit tests) + +**Approach:** +- Single pass: walk the source tree with a `VecDeque<(AccessibilityNode, parent_index)>`. Emit each node; defer children; after all nodes from the current level are emitted, process the next level. +- Each emission records `(own_index, parent_index)`; at end of each parent's child-emission batch, patch the parent's `child_start` and `child_count`. +- Pre-count nodes in a first pass for `Vec::with_capacity` — avoids the 14-realloc issue on a 10k-node tree. Alternative: single-pass with capacity hint (`count_nodes_for_capacity`) to avoid two passes. +- No recursion → no FFI-layer depth guard needed; we inherit `ABSOLUTE_MAX_DEPTH=50` from the macOS adapter. + +**Patterns to follow:** +- Standard BFS queue pattern. No workspace precedent — this unit establishes it. + +**Test scenarios:** +- Happy path: single root with no children → single-node tree, child_count=0 +- Happy path: root with three flat children → root.children returns all three, each child has child_count=0 +- Happy path: review's failing case `root → [a → [a1, a2], b]` → iterating root's children yields `[a, b]` (NOT `[a, a1]`), iterating a's children yields `[a1, a2]`, iterating b's children yields `[]` +- Happy path: deeply nested tree (10 levels, one child per level) — child ranges walk correctly at every level +- Happy path: wide tree (one root with 1000 direct children) — all 1000 accessible, no reallocation observed (tracked via capacity test) +- Edge case: empty tree via `AccessibilityNode` with zero children produces single-node array, `ad_free_tree` handles the zero-child root correctly +- Edge case: tree emitted at full `max_depth` pruning boundary — remaining children should have `child_count = 0` (or the current pruning semantics preserved), not dangling ranges + +**Verification:** All existing tree tests pass. New test `test_direct_child_iteration_returns_only_direct_children` iterates `nodes[child_start..child_start+child_count]` at every level and asserts exact role sequence. + +### Phase 3 — ABI-Locked Shape Changes + +- [ ] **Unit 5: Opaque list handles + image buffer length encapsulation** + +**Goal:** Replace every `(*mut T, count)` list-returning API with an opaque handle. Caller never sees the backing allocation. `AdImageBuffer` tracks allocation length internally, not in a C-mutable field. + +**Requirements:** R8, R22 + +**Dependencies:** Unit 1 (panic boundary wraps new fns), Unit 2 (if any enum validation applies). + +**Files:** +- Modify: `crates/ffi/src/types.rs` (add forward-declared `AdWindowList`, `AdAppList`, `AdSurfaceList`, `AdNotificationList` — private structs with `Box<[T]>` inner; make `AdImageBuffer.data_len` private by replacing with an opaque wrapper struct) +- Rewrite: `crates/ffi/src/apps.rs` (`ad_list_apps` returns `AdAppList*`; new `ad_app_list_count`, `ad_app_list_get`, `ad_app_list_free`; remove `ad_free_apps`) +- Rewrite: `crates/ffi/src/windows.rs` (same pattern for `AdWindowList`; also see Unit 6 for `focused_only` addition) +- Rewrite: `crates/ffi/src/surfaces.rs` (same pattern for `AdSurfaceList`) +- Rewrite: `crates/ffi/src/screenshot.rs` (`AdImageBuffer` now opaque or embeds a private length field not read back from C) +- Rename: existing `ad_free_window` → `ad_release_window_fields` (interior-string free for the single `AdWindowInfo` from `ad_launch_app`) to disambiguate from the list case +- Test: `crates/ffi/src/{apps,windows,surfaces,screenshot}.rs` unit tests + new `crates/ffi/tests/opaque_lists.rs` + +**Approach:** +- `AdWindowList`, etc. are forward-declared in the header via cbindgen's opaque-struct support. Rust defines them as `pub struct AdWindowList { items: Box<[AdWindowInfo]> }` — no `#[repr(C)]`, intentionally no layout guarantee. +- Accessors: `_count(list)` returns `items.len() as u32`; `_get(list, i)` returns `items.get(i as usize).map_or(null(), |w| w as *const _)`; `_free(list)` drops the `Box`. +- `ad_app_list_get` returning null on out-of-bounds is explicitly documented; caller must check. +- `AdImageBuffer`: keep the struct for cbindgen compat but store the allocation length in a shadow thread-local map keyed on `data` pointer, OR wrap in a private `Box` allocated structure where `ad_screenshot` returns `AdImageBuffer*` (opaque) with accessors `_data()`, `_size()`, `_width()`, `_height()`, `_format()`. Second approach is cleaner; use it. +- Rename breaks existing API: `ad_free_window` naming collision with `ad_free_windows` caused the review's heap-corruption hazard. Rename to `ad_release_window_fields` and document that it's for the single struct from `ad_launch_app`. + +**Patterns to follow:** +- Rust `Box::into_raw` / `Box::from_raw` for ownership transfer across the FFI boundary. +- Cbindgen opaque struct declaration: the Rust struct carries no `#[repr(C)]`; cbindgen emits a forward `typedef struct AdWindowList AdWindowList;`. + +**Test scenarios:** +- Happy path: `ad_list_apps` returns non-null list; `ad_app_list_count` matches expected; iterating via `_get` returns each app +- Happy path: empty list from `ad_list_apps` (no apps running in test env) — list is non-null, count is 0, `_get(0)` returns null +- Happy path: `ad_list_windows(adapter, app_filter, focused_only=false, &list)` populates list; out-of-bounds `_get(count)` returns null +- Edge case: `_free(null)` is a no-op (matches review's current free-fn tolerance pattern) +- Edge case: double-free under Miri produces expected error (or documented caller responsibility — pick; I recommend double-free is UB and the skill doc says so) +- Error path: caller-supplied adapter is null → `ErrInvalidArgs` +- Integration: assert that struct layout of `AdWindowList` is NOT exposed in the header — grep the generated `agent_desktop.h` for `AdWindowList` should find only `typedef struct AdWindowList AdWindowList;` and function signatures, no field declarations + +**Verification:** Header review shows only opaque typedefs for the four list types; no count-mismatch API path exists. + +--- + +- [ ] **Unit 6: ABI surface completion — surface / focused_only / RefEntry / handle** + +**Goal:** Every ABI addition that would otherwise be a breaking change post-publication lands in this unit. After this, the C struct layouts are frozen. + +**Requirements:** R9, R10, R5 + +**Dependencies:** Unit 2 (surface enum validation), Unit 5 (focused_only lands on the reshaped `ad_list_windows`). + +**Files:** +- Modify: `crates/ffi/src/types.rs`: + - Add `AdSnapshotSurface` enum (`Window=0, Focused, Menu, Menubar, Sheet, Popover, Alert`) + - Add `surface: AdSnapshotSurface` field to `AdTreeOptions` (default=0=Window preserves parity with the current hardcoded `SnapshotSurface::Window` at `crates/ffi/src/tree.rs:158`; since the header is unpublished, this is a pre-freeze shape correction, not an ABI-compatible addition) +- Modify: `crates/ffi/src/tree.rs` (`ad_get_tree` maps surface enum variants to `SnapshotSurface` core variants; no longer hardcodes Window) +- Modify: `crates/ffi/src/windows.rs` (`ad_list_windows` signature adds `focused_only: bool` parameter; `WindowFilter` construction uses the new value) +- Modify: `crates/core/src/adapter.rs` (add trait method `release_handle(&self, handle: &NativeHandle) -> Result<(), AdapterError>` with `not_supported()` default) +- Modify: `crates/macos/src/adapter.rs` (impl `release_handle`: `CFRelease(handle.as_raw() as CFTypeRef)`; handle nulls safely) +- Create or modify: `crates/ffi/src/actions.rs` — add `ad_free_handle(adapter: *const AdAdapter, handle: *const AdNativeHandle) -> AdResult` +- Modify: `crates/ffi/include/agent_desktop.h` — regenerated with new signatures +- Test: `crates/ffi/src/{tree,windows,actions}.rs` unit tests; integration test `crates/ffi/tests/surfaces_and_focused.rs` + +**Approach:** +- `AdSnapshotSurface`: straightforward `#[repr(i32)]` enum mapping 1:1 to `SnapshotSurface`. +- `focused_only` on `ad_list_windows`: second-last parameter (before `out`). Matches `WindowFilter::focused_only`. +- `release_handle` in core: the first handle-releasing trait method. Default `not_supported()` preserves backward compat with Windows/Linux stubs. +- macOS impl: carefully calls `CFRelease` (not `CFRetain(… - 1)`), follows `Drop` pattern in `crates/macos/src/tree/element.rs:24-39`. +- `ad_free_handle`: takes adapter + handle by pointer. Both nulls tolerated (matches `ad_adapter_destroy`). Calls adapter's `release_handle` under `ffi_try!`. +- Windows/Linux stubs at `crates/windows/src/lib.rs` and `crates/linux/src/lib.rs` require no modification — they inherit the `not_supported()` default for `release_handle`. + +**Patterns to follow:** +- `AXElement::Drop` in `crates/macos/src/tree/element.rs` — reference for correct `CFRelease` discipline. +- `SnapshotSurface` match in `crates/macos/src/adapter.rs:38-52` — the mapping the FFI layer mirrors. +- Adapter default-impl pattern elsewhere in `crates/core/src/adapter.rs` — every new trait method ships with `not_supported()` default. + +**Test scenarios:** +- Happy path: `ad_get_tree` with each of the seven `AdSnapshotSurface` variants succeeds against an app that exposes the corresponding surface +- Happy path: `ad_list_windows(adapter, null_filter, focused_only=true, &list)` returns at most one window (the focused one) +- Happy path: `ad_list_windows(adapter, null_filter, focused_only=false, &list)` returns all windows +- Happy path: `ad_resolve_element` → `ad_free_handle` → repeat 1000 times → no Core Foundation retain count leak (measured via Activity Monitor or `vm_stat` in the test harness; also verify via Leaks instrument in a separate validation run) +- Edge case: `ad_free_handle(null_adapter, _)` returns `ErrInvalidArgs` +- Edge case: `ad_free_handle(adapter, null_handle)` is a no-op returning `Ok` +- Edge case: surface enum out-of-range is caught by Unit 2's validation → `ErrInvalidArgs` +- Error path: platform with `not_supported()` default for `release_handle` returns `ActionNotSupported` — windows/linux stubs; macOS never hits this + +**Verification:** Run a 10-minute soak test resolving + freeing handles in a loop; `leaks --atExit` reports zero `AXUIElement` retains accumulated from agent-desktop allocations. Generated header contains `AdSnapshotSurface` enum with all seven variants and `focused_only` parameter on `ad_list_windows`. + +### Phase 4 — Input Hygiene + +- [ ] **Unit 7: Out-param zeroing, string NUL handling, window validation** + +**Goal:** Every fallible FFI fn zeroes out-params before any fallible work. Mandatory string fields never return null due to interior NUL. Window targeting by title never silently picks the wrong window. + +**Requirements:** R6, R13, R15 + +**Dependencies:** Unit 1 (panic boundary), Unit 5 (new opaque list signatures). + +**Files:** +- Modify: `crates/ffi/src/{actions,apps,screenshot,tree,windows}.rs` — audit every fn with out-params, add zeroing at entry (some already have it: `ad_get_tree`, `ad_list_windows` under the new list shape; missing in: `ad_launch_app`, `ad_execute_action`, `ad_resolve_element`, `ad_screenshot`) +- Modify: `crates/ffi/src/convert.rs` — create two helper functions: `string_to_c_lossy(&str) -> *mut c_char` (mandatory fields: replace interior NUL with U+FFFD before `CString::new`; infallible after replacement) and `opt_string_to_c(&Option<String>) -> *const c_char` (optional fields: returns null on `None` OR on interior NUL). Apply `string_to_c_lossy` to all mandatory string fields in `actions.rs`, `windows.rs`, `apps.rs`, `tree.rs`, `surfaces.rs`, `notifications.rs` (from Unit 8); apply `opt_string_to_c` to optional ones (description, hint, value, bundle_id, etc.) +- Modify: `crates/ffi/src/windows.rs::ad_window_to_core` — null-check `w.id` and `w.title`; return `AdapterError::new(InvalidArgs, "window id or title is null")` on null +- Test: `crates/ffi/src/{convert,windows}.rs` + new `crates/ffi/tests/nul_handling.rs` + +**Approach:** +- Zeroing: `unsafe { *out = std::mem::zeroed(); }` immediately after pointer validation, before any call that could fail. +- NUL replacement: one pass replacing `\0` → U+FFFD bytes (3-byte UTF-8 sequence `0xEF 0xBF 0xBD`) in the string, then `CString::new` becomes infallible. Allocates a new `String` only when a NUL is present (fast path: no allocation if source has no NUL). +- `ad_window_to_core` validation: matches the defensive-validation pattern in `crates/ffi/src/apps.rs:70-77` (null-checked `c_to_str` with early return). + +**Patterns to follow:** +- Existing `c_to_str` null-check in `apps.rs` `ad_launch_app` line 69-78. +- `ad_get_tree` current zeroing pattern at `tree.rs:40-42`. + +**Test scenarios:** +- Happy path: `ad_execute_action` succeeds, out-param populated correctly +- Error path: `ad_execute_action` with invalid action kind → out-param is zeroed, caller calling `ad_free_action_result` on it is a no-op (inner pointers all null) +- Error path: `ad_resolve_element` with stale entry → `AdNativeHandle.ptr = null` (not garbage) +- Edge case: string with interior NUL at position 5 of 20 → mandatory field preserves first 5 chars, replaces NUL with U+FFFD, keeps remaining 14 chars +- Edge case: Chromium string like `"foo\0\0bar"` → mandatory field returns `"foo\u{FFFD}\u{FFFD}bar"` +- Edge case: optional field with interior NUL → returns null pointer (unchanged behavior) +- Error path: `ad_get_tree(adapter, window_with_null_title, opts, out)` → `ErrInvalidArgs`, no "match empty string" behavior +- Error path: `ad_launch_app` with non-UTF-8 id bytes → `ErrInvalidArgs` + +**Verification:** Miri-clean on all tests; consumer stack-allocating a struct and passing it into a failing FFI fn can then call the paired `_free` safely. + +### Phase 5 — Coverage Parity + +- [ ] **Unit 8: Notification FFI surface** + +**Goal:** All four notification commands exposed via the C ABI. + +**Requirements:** R11 (notifications portion) + +**Dependencies:** Unit 5 (opaque list shape — notifications list reuses it as `AdNotificationList`). + +**Files:** +- Create: `crates/ffi/src/notifications.rs` (new module — ~250 LOC estimate) +- Modify: `crates/ffi/src/types.rs` (add `AdNotificationInfo`, `AdNotificationFilter`, opaque `AdNotificationList`) +- Modify: `crates/ffi/src/convert.rs` (add `notification_info_to_c`, `free_notification_info_fields`) +- Modify: `crates/ffi/src/lib.rs` (`mod notifications`) +- Test: `crates/ffi/src/notifications.rs` (unit tests) + +**Approach:** +- Exports: `ad_list_notifications(adapter, filter, &list)`, `ad_dismiss_notification(adapter, index, app_filter)`, `ad_dismiss_all_notifications(adapter, app_filter, &dismissed_list, &failed_list)`, `ad_notification_action(adapter, index, action_name, &result)`. +- Index stability contract: documented in header — dismiss-by-index is valid only against the *most recent* list call; adapter re-queries internally. +- `dismiss_all` surfaces both dismissed and failed as separate opaque lists (`AdNotificationList` + `AdStringList` for failure reasons). Failed dismissals do not overwrite last-error if any succeeded; caller inspects `failed_list` instead. + +**Patterns to follow:** +- `crates/core/src/commands/{list_notifications,dismiss_notification,dismiss_all_notifications,notification_action}.rs` — the command-layer logic; FFI is a thin wrapper. +- Opaque list pattern from Unit 5. + +**Test scenarios:** +- Happy path: `ad_list_notifications(adapter, null_filter, &list)` returns notification list on a system with notifications present +- Happy path: `ad_dismiss_notification(adapter, 0, null)` dismisses the first notification; subsequent list returns one fewer +- Happy path: `ad_notification_action(adapter, 0, "Reply", &result)` triggers the Reply button +- Edge case: filter by app name `"Slack"` returns only Slack notifications +- Edge case: filter by text `"deploy"` returns only matching notifications +- Edge case: `dismiss_all` with partial failure — `dismissed_list` non-empty, `failed_list` non-empty +- Error path: `ad_dismiss_notification` with out-of-range index → `ErrNotificationNotFound` (error code -11, which already exists in `AdResult`) +- Error path: `ad_notification_action` with unknown action name → `ErrActionNotSupported` + +**Verification:** FFI consumer can replicate every `agent-desktop list-notifications` / `dismiss-notification` / `notification-action` CLI flow. + +--- + +- [ ] **Unit 9: Observation primitives — find / get / is** + +**Goal:** Cheap single-element lookups without building a full snapshot tree. + +**Requirements:** R11 (find/get/is portion) + +**Dependencies:** Unit 4 (BFS tree — `ad_find` walks the tree), Unit 6 (`ad_free_handle` — `ad_find` returns a handle the caller must release). + +**Files:** +- Create: `crates/ffi/src/observation.rs` +- Modify: `crates/ffi/src/types.rs` (add `AdFindQuery { role, name_substring, value_substring }`) +- Modify: `crates/ffi/src/lib.rs` (`mod observation`) +- Test: `crates/ffi/src/observation.rs` + `crates/ffi/tests/observation.rs` + +**Approach:** +- `ad_find(adapter, app, query, &handle)`: walks `get_tree` result, finds first match, resolves to `NativeHandle` via `resolve_element`, returns `AdNativeHandle` in out-param. Caller must `ad_free_handle`. +- `ad_get(adapter, handle, property, &str_out)`: property is an enum or magic string (`"value"`, `"bounds"`, `"role"`, `"name"`). Dispatches to `get_live_value` / `get_element_bounds` / etc. Returns allocated string via `ad_free_string`. +- `ad_is(adapter, handle, property, &bool_out)`: boolean state check. `"focused"`, `"enabled"`, `"selected"`, `"checked"`, `"expanded"`. Reads from resolved element's state set. + +**Patterns to follow:** +- `crates/core/src/commands/find.rs` — reference tree-walk logic (FFI re-implements rather than calling core's command layer to avoid a JSON round-trip). +- `crates/core/src/commands/get.rs` and `is.rs` — reference for property dispatch. + +**Test scenarios:** +- Happy path: `ad_find(adapter, "Finder", { role: "button", name_substring: "OK" }, &h)` returns a handle to the first matching button +- Happy path: `ad_get(adapter, h, "value", &s)` returns the element's current value +- Happy path: `ad_is(adapter, h, "focused", &b)` returns true for the currently focused element +- Edge case: `ad_find` with no matches returns `ElementNotFound` and out-handle is zeroed +- Edge case: `ad_get` for unknown property returns `InvalidArgs` +- Error path: stale handle (UI repainted) returns `StaleRef` on `ad_get` +- Integration: full flow `ad_find → ad_get → ad_is → ad_free_handle` with no leaks (soak test, same as Unit 6) + +**Verification:** FFI consumer can do `find → is(focused) → if yes then get(value)` without a single `ad_get_tree` call. + +--- + +- [ ] **Unit 10: Action constructor helpers** + +**Goal:** Consumers building `AdAction` from Python / Go / Swift no longer zero-init unused variant fields. + +**Requirements:** R11 (ergonomics sub-point surfaced by flow analysis) + +**Dependencies:** Unit 1 (every `extern "C"` constructor helper must go through the `ffi_try!` panic boundary — even pure-value constructors can panic on OOM). + +**Files:** +- Create: `crates/ffi/src/action_helpers.rs` +- Modify: `crates/ffi/src/lib.rs` (`mod action_helpers`) +- Test: `crates/ffi/src/action_helpers.rs` + +**Approach:** +- Per-variant constructor: `ad_action_click() -> AdAction`, `ad_action_double_click() -> AdAction`, `ad_action_type_text(text: *const c_char) -> AdAction`, `ad_action_scroll(dir: AdDirection, amount: u32) -> AdAction`, `ad_action_press(combo: *const AdKeyCombo) -> AdAction`, etc. +- Each returns a fully-zeroed `AdAction` with only the relevant variant-fields populated. +- `text`-carrying actions borrow the pointer; caller retains ownership of the backing string during `ad_execute_action`. +- No new `ad_free_action` helper — `AdAction` is a value type, dropped when it goes out of scope on the C caller's stack. + +**Patterns to follow:** +- Existing `mouse_button_from_c` / `direction_from_c` — direct conversion helpers. +- `#[no_mangle] pub extern "C" fn` returning a value-type struct is unusual for FFI but widely supported; cbindgen handles it. + +**Test scenarios:** +- Happy path: each helper produces an `AdAction` whose relevant fields are set and irrelevant variant fields are zeroed +- Happy path: `ad_action_type_text("hello")` produces an action that, passed to `ad_execute_action`, types "hello" +- Edge case: `ad_action_type_text(null)` returns an `AdAction` with kind `TypeText` and a null text pointer; `ad_execute_action` then returns `InvalidArgs` (no segfault) +- Integration: round-trip in Rust — construct via helper, pass to `action_from_c`, verify core `Action` matches expectation + +**Verification:** Consumer can write `ad_execute_action(adapter, handle, &ad_action_click(), &result)` as a one-liner. + +### Phase 6 — Production Readiness + +- [ ] **Unit 11: macOS main-thread enforcement and NativeHandle review** + +**Goal:** Debug builds assert that every macOS-sensitive FFI call happens on the main thread. The `NativeHandle` thread-safety justification is narrowed to match reality. + +**Requirements:** R7, R21 + +**Dependencies:** Unit 1 (macro wrapper is the natural insertion point for per-fn assertion). + +**Files:** +- Create: `crates/ffi/src/main_thread.rs` (shim: `debug_assert_main_thread()` — macOS calls `pthread_main_np()`, other platforms no-op) +- Create: the assertion helper lives in `crates/ffi/src/main_thread.rs` (new file, already listed above) and exports `ffi_macos_main_thread!` — a macro that composes `ffi_try!` with a prepended `debug_assert!(is_main_thread(), ...)`. The macro is applied only to macOS-sensitive fns: clipboard, window ops, actions, screenshot. Not applied to: `ad_adapter_create`, `ad_adapter_destroy`, `ad_last_error_*`, `ad_free_*`, `ad_action_*` constructors. The plain `ffi_try!` remains the default; `ffi_macos_main_thread!` is a composed variant, not a replacement. +- Modify: `crates/core/src/adapter.rs` — narrow the `unsafe impl Send + Sync for NativeHandle` comment to current-reality-accurate justification +- Modify: `crates/ffi/include/agent_desktop.h` — add prominent warning block about main-thread requirement (via `///` doc-comment on `ad_adapter_create` or a top-level `//!` header comment) +- Modify: `crates/ffi/src/lib.rs` — crate-level `//!` rustdoc covering thread-safety model +- Test: `crates/ffi/tests/main_thread.rs` + +**Approach:** +- `pthread_main_np()` available via `libc::pthread_main_np()` (macOS only). Returns non-zero if current is main. +- `debug_assert!(is_main_thread(), "must be called on main thread");` — panics in debug, no-op in release. Panic → caught by Unit 1 boundary → returns `ErrInternal` with message "called off main thread". Safer than UB in release builds. +- Non-macOS: `fn is_main_thread() -> bool { true }` (Windows UIA and Linux AT-SPI have no main-thread restriction; the check is informational only). +- Retain `unsafe impl Send + Sync` on `NativeHandle` — the FFI contract makes the consumer responsible. Update the comment from "Phase 1 single-threaded CLI" to "FFI contract: single-owner, single-thread. Consumer synchronizes. Violations are UB." + +**Patterns to follow:** +- `libc::pthread_main_np` — standard macOS idiom. No workspace precedent. + +**Test scenarios:** +- Happy path: main-thread calls succeed in debug build +- Error path: spawned thread calls `ad_get_clipboard` in debug build → assertion fires → caught by panic boundary → returns `ErrInternal` with "must be called on main thread" last-error +- Edge case: release build does not perform the check (debug_assert is removed) — worker-thread call proceeds and may silently UB on macOS; documented loudly in the header +- Edge case: non-macOS builds skip the check entirely and succeed on any thread +- Integration: `ad_adapter_create` / `ad_adapter_destroy` / `ad_free_*` do NOT enforce the check (they're safe off-thread) + +**Verification:** `pthread_main_np` call sites confined to debug-only path. Header has a "⚠ THREAD SAFETY" block near the adapter-creation functions. + +--- + +- [ ] **Unit 12: Build hygiene — cbindgen pinning, header drift, dead_code, unwrap removal** + +**Goal:** Build tooling fails loudly when cbindgen breaks. Generated header cannot silently drift. Zero `.unwrap()` / `.expect()` in non-test code. Compile-time assertion guards `ErrorCode`/`AdResult` parity. + +**Requirements:** R14, R19, R20, R23 + +**Dependencies:** None. + +**Files:** +- Modify: `crates/ffi/Cargo.toml` — `cbindgen = "= 0.27.0"` (exact pin; remove the caret) +- Modify: `crates/ffi/build.rs`: + - Replace `.expect()` with graceful `eprintln!("cargo:warning=...")` fallbacks on env-var read failures + - Replace `if let Ok(bindings) = ...` with `match` that panics loudly on cbindgen error (build-time only, not runtime) + - Replace `.ok()` on `fs::copy` with `eprintln!` on failure +- Modify: `.github/workflows/ci.yml` — add step after `cargo build --release` that runs `cargo build -p agent-desktop-ffi` and then `git diff --exit-code crates/ffi/include/` +- Modify: `crates/ffi/src/error.rs`: + - Remove the line-56 `.unwrap()` on `CString::new("(message contained null byte)")` — replace with static bytes + - Add compile-time assertion: `const _: () = assert!(ErrorCode::VARIANTS == AdResult::non_ok_variant_count());` (use a manual const-fn match counter if `variant_count` is nightly-only) +- Modify: `crates/ffi/src/convert.rs` — remove file-scoped `#![allow(dead_code)]` OR keep with an explicit comment explaining the cdylib false-positive rationale +- Audit and remove: every `#[allow(dead_code)]` in `crates/ffi/src/` that annotates a fn reachable from a `#[no_mangle]` export; consolidate remaining to module-level where needed +- Test: build-script test harness + CI workflow dry-run + +**Approach:** +- Exact cbindgen pin (`= 0.27.0`) eliminates formatting drift from patch-version changes. Bumping cbindgen becomes an explicit PR with a regenerated header. +- Header drift check: CI runs `cargo clean -p agent-desktop-ffi && cargo build -p agent-desktop-ffi` then `git diff --exit-code`. Any header change that wasn't committed fails the build. +- Compile-time variant assertion: compile-time check that every `ErrorCode` variant has a matching `AdResult` entry. Prevents silent misses when core adds an error. + +**Patterns to follow:** +- CI `fail if cargo tree -p core contains platform crates` pattern in `.github/workflows/ci.yml:43-49` — similar shape for the new drift check. + +**Test scenarios:** +- Happy path: `cargo build -p agent-desktop-ffi` succeeds; header is regenerated identically +- Error path: if someone modifies a type in `types.rs` without committing the regenerated header, CI fails with a clear diff +- Error path: if someone adds a new `ErrorCode::Foo` variant without adding `AdResult::ErrFoo`, `cargo build` fails with the const-assert message +- Error path: if `cbindgen.toml` is malformed or cbindgen itself errors, `cargo build` fails loudly (not silently emits a stale header) + +**Verification:** Grep `unwrap\|expect` in `crates/ffi/src/*.rs` (non-test sections) returns zero hits. CI header-drift step exists and passes. + +### Phase 7 — Documentation + +- [ ] **Unit 13: Skill + rustdoc + CLAUDE.md + code cleanup** + +**Goal:** Every agent / developer / consumer reading project docs sees accurate FFI guidance. Per-field nullability is documented in the generated header. Wildcard re-exports eliminated. Inline comments stripped. + +**Requirements:** R12, R16, R17, R18, R24 + +**Dependencies:** All prior units (doc-writes reference the final API shape). + +**Files:** +- Create: `skills/agent-desktop-ffi/SKILL.md` (frontmatter + overview) +- Create: `skills/agent-desktop-ffi/references/ownership.md` (who allocates / who frees table, per pointer type) +- Create: `skills/agent-desktop-ffi/references/error-handling.md` (errno-style contract, last-error lifetime, enum validation) +- Create: `skills/agent-desktop-ffi/references/threading.md` (macOS main-thread rule, Python permission attachment quirk, single-owner handle rule) +- Modify: `crates/ffi/src/lib.rs` — add `//!` crate-level rustdoc: purpose, ownership model, error pattern, thread safety, example build/link +- Modify: `crates/ffi/src/lib.rs` — replace `pub use types::*` with explicit re-exports (list every type the FFI actually exports) +- Modify: `crates/ffi/src/types.rs` — add `///` doc-comments on every field for nullability / lifetime / sentinel-value documentation (cbindgen propagates to header) +- Modify: `crates/ffi/src/tree.rs` — strip inline `//` comments on lines 43, 60, 215, 220, 231, 244, 251, 255, 260 +- Modify: `CLAUDE.md` line 79 — update "binary crate is the only place that wires platform → core" to reflect FFI being the second legitimate wiring point +- Modify: `CLAUDE.md` workspace tree section — add `crates/ffi/` to the layout diagram +- Modify: `skills-lock.json` (auto-updated by clawhub, but verify the new skill is tracked) + +**Approach:** +- `SKILL.md` frontmatter: `name: agent-desktop-ffi`, version matching `Cargo.toml` (0.1.x), tags `ffi, c-bindings, cdylib`, description covering the Python/Swift/Go/Node use case. +- Reference docs split by topic so an AI or human consumer can load the relevant file without reading everything. +- Ownership table: for every `ad_*` fn that allocates, list the matching `ad_free_*` / `ad_*_list_free` / `ad_release_*_fields` / `ad_free_handle` / `ad_free_string` / `ad_free_action_result` / `ad_free_image` / `ad_free_tree`. +- Threading doc: explicit warning about `AXIsProcessTrusted` attaching to the hosting executable (Python, not agent-desktop) — consumers will hit this on day one. + +**Patterns to follow:** +- Existing `skills/agent-desktop/SKILL.md` frontmatter + reference structure. +- `docs/prd-addendum-skill-maintenance.md` mandates — every new platform / interface adds a skill directory. +- Rustdoc field-level docs in `crates/core/src/adapter.rs` — pattern for per-field documentation cbindgen propagates. + +**Test scenarios:** +- Test expectation: none — this unit is pure documentation. Verification is via the review checklist below. + +**Verification:** +- `skills/agent-desktop-ffi/SKILL.md` exists with valid frontmatter (clawhub sync dry-run passes) +- Generated `agent_desktop.h` contains `///` comments on every field for at least: `AdWindowInfo.bounds` (nullability sentinel `has_bounds`), `AdAppInfo.bundle_id` (optional, null if unset), `AdSurfaceInfo.title` / `.item_count`, `AdNode.{ref_id,name,value,description,hint}`, `AdActionResult.ref_id` +- `lib.rs` has no wildcard re-exports (grep `pub use types::\*` returns zero) +- `tree.rs` has no inline `//` comments (only `///` doc-comments) +- `CLAUDE.md` FFI mention at the workspace-tree section and the platform-wiring paragraph +- Clawhub sync picks up the new skill on the next release + +## System-Wide Impact + +- **Interaction graph:** The FFI crate is the second platform → core wiring point (after `src/`). CI dependency-isolation check currently only guards `agent-desktop-core`; this plan doesn't add a new gate for `agent-desktop-ffi` because its platform imports are intentional (parallel to the binary crate). CLAUDE.md updated to reflect the two-wiring-point reality. +- **Error propagation:** The errno-style last-error lifetime change (Unit 3) affects every FFI fn's success path — all `clear_last_error()` calls on success become no-ops. Downstream: consumers who were accidentally depending on errors being cleared on success must re-read the documentation (but no current consumer exists — PR unmerged). +- **State lifecycle risks:** `ad_free_handle` (Unit 6) is the first handle-releasing surface; forgetting to call it in an agent loop is a steady leak of Core Foundation retains on macOS. Skill doc must make this prominent. `NativeHandle` single-owner invariant is newly documented but was always assumed. +- **API surface parity:** Commands now exposed via FFI (notifications + find/get/is) reach parity with the documented "50 commands" count from the CLI. The gap list (wait/batch/press_key_for_app/get_live_value) is the only Phase 2 FFI coverage work. +- **Integration coverage:** Unit tests cover Rust-side correctness. The plan explicitly does not add Python/Swift/Go/Node integration tests — each is its own test scaffolding work (deferred). Cross-language layout parity IS implicitly tested by the `crates/ffi/tests/layout_contract.rs` test asserting struct size/align/offsets match known-good expectations. +- **Unchanged invariants:** `PlatformAdapter` trait remains the sole abstraction. Core never imports platform crates (CI still enforces). `ErrorCode` enum ordering is preserved (new variants, if any, go to the end). CLI command count (53) is unchanged — this plan doesn't add CLI commands, only FFI exports. + +## Risks & Dependencies + +| Risk | Mitigation | +|------|------------| +| Per-package `panic = "unwind"` adds landing-pad size to the cdylib | Measure cdylib size before/after; accept if <500 KB additional. FFI cdylib is not under the 15 MB CLI budget. | +| BFS rewrite introduces a regression in tree output format | New test asserts direct-child iteration. Existing `test_flatten_depth_first_order` renamed and reworked. Golden-fixture tests (if any added later) snapshot the BFS format. | +| Core trait addition (`release_handle`) breaks Windows / Linux adapter stubs at compile | Default `not_supported()` impl means Win/Linux stubs compile without change. macOS impl lands in this plan. | +| `ad_free_handle` consumer forgetfulness causes CFRetain leak anyway | Skill doc's Flow B example makes `ad_free_handle` the mandatory final step. Future static analyzer for C consumers could detect missing frees; out of scope. | +| Header drift CI check produces false positives when developer machine's `cargo build` runs before CI | Documented in README (Development section): after modifying FFI types, commit the regenerated header immediately. | +| Cbindgen 0.27 gets yanked or has CVE | Exact-pin forces an explicit bump; worst case, upgrade PR touches `Cargo.toml` + regenerates header in one commit. Low likelihood. | +| The opaque list pattern locks out future zero-copy access | Accept: opaque handles are net better for safety now; zero-copy can be added non-breakingly by exposing a `_data_ptr()` accessor later. | +| Panic boundary's static-message approach loses dynamic context in a panic | Accept: the alternative (allocate inside panic handler) risks double-panic → abort. Static message names the boundary; full diagnostic via `ad_last_error_platform_detail()` on recoverable errors. | +| Main-thread `debug_assert` in release builds is a silent UB channel | Header's prominent warning + skill doc make the constraint loudly known. Consumers that violate get what they ask for. | +| NULL-replacement in mandatory fields masks upstream Electron AX bugs | Replacement is logged via `tracing` at `warn` level so test runs still surface occurrences. | + +## Documentation / Operational Notes + +- **CLAUDE.md update** is part of Unit 13. Lines to touch: the "binary crate is the only place that wires platform → core" claim and the workspace tree diagram that omits `crates/ffi/`. +- **`skills/agent-desktop-ffi/`** is a new skill directory; the release pipeline (`clawhub sync --root skills/ --all`) picks it up automatically on the next release, so no release-workflow edits needed. +- **No rollout gate.** The fixes all target an unmerged PR. Merge = ship (within the scope of the Phase 1 plan). +- **Post-merge follow-ups tracked by this plan:** + 1. Run `ce:compound` on each of the 15 review findings to seed `docs/solutions/`. + 2. Open a separate plan for `ad_wait`, `ad_batch`, `ad_press_key_for_app`, `ad_get_live_value` FFI exposure. + 3. Open a separate plan for cdylib artifact distribution (release pipeline, npm slot for native libraries, cross-platform build matrix). + +## Sources & References + +- **Origin review:** [PR #22 review 4102879703](https://github.com/lahfir/agent-desktop/pull/22#pullrequestreview-4102879703) — `CHANGES_REQUESTED` by [@lahfir](https://github.com/lahfir) +- **Origin branch:** `feat/c-bindings` (PR #22 by Jake Rosoman) +- **Project standards:** `CLAUDE.md` (LOC limits, no unwrap, explicit re-exports, conventional commits, skill maintenance) +- **Trait surface:** `crates/core/src/adapter.rs` (28 methods, `PlatformAdapter`, `NativeHandle`, `SnapshotSurface`, `WindowFilter`) +- **Retain discipline reference:** `crates/macos/src/tree/element.rs:24-39` (`AXElement` `Drop`+`Clone` with `CFRelease`/`CFRetain`) +- **Command-layer logic to thin-wrap:** `crates/core/src/commands/{list_notifications,dismiss_notification,dismiss_all_notifications,notification_action,find,get,is}.rs` +- **cbindgen config:** `crates/ffi/cbindgen.toml` +- **CI workflow:** `.github/workflows/ci.yml` +- **Release workflow:** `.github/workflows/release.yml` +- **Skill precedent:** `skills/agent-desktop/SKILL.md` diff --git a/docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md b/docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md new file mode 100644 index 0000000..0fc5526 --- /dev/null +++ b/docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md @@ -0,0 +1,1858 @@ +--- +title: Phase 2 — Windows Adapter + Cross-Platform Feature Parity +type: feat +status: active +date: 2026-04-18 +origin: docs/brainstorms/2026-04-18-phase2-windows-crossplatform-brainstorm.md +deepened: 2026-04-18 +--- + +# Phase 2 — Windows Adapter + Cross-Platform Feature Parity + +## Overview + +Phase 2 brings agent-desktop to Windows and closes every cross-platform feature-parity gap surfaced after v0.1.13. A single release ships 18 objectives (P2-O1 … P2-O18) across ~15 implementation units. The core CLI, JSON envelope, and ref system are preserved; what grows is the adapter surface (new trait methods), the type surface (new `Action`/`ErrorCode`/`PermissionReport` variants, stable-selector fields), the FFI surface (deterministic codegen, `ad_abi_version`), the Windows shell-surface command surface, and the platform count (macOS + Windows, both x86_64 and ARM64). Version target: **v0.2.0** (breaking ABI + JSON schema). + +## Headless-First Invariant (CORE PRINCIPLE) + +**Every command in agent-desktop — existing and Phase 2 additions — must work headlessly inside the current user's active desktop session.** "Headless" means: (a) agent-desktop's own process has no GUI, no Dock icon, no visible window, no menu bar; (b) the target app does NOT need to be foregrounded or focused to be observed/driven; (c) no user-visible focus changes are side-effects of automation; (d) no physical cursor movement unless the caller explicitly invokes a mouse command. It does **not** mean bypassing the OS desktop model: Session 0, Server Core, secure desktop, locked desktop, and other-user sessions are unsupported for accessibility/capture and must return structured `PlatformNotSupported`, `PermDenied`, or `WindowNotFound` errors. + +This is the core agent-automation contract. Any Phase 2 design choice that violates it is rejected. + +### Headless rules by category + +| Operation | Headless path | Forbidden | +|---|---|---| +| **macOS observation** | AX API (`AXUIElementCopy*`) — works on any process with granted Accessibility TCC; no focus needed | Reading AX from non-main thread (Apple DTS: all AX calls main-thread only) | +| **macOS action** | `AXUIElementPerformAction` (press, raise, show-menu) — headless on any visible or minimized window | Mouse-cursor synthesis when AX action is available; `NSApp.activate` as an action side-effect | +| **macOS events (`watch_element`)** | `AXObserver` bound to the **main-thread `CFRunLoop`** (bootstrapped by the CLI) | Worker-thread CFRunLoop with observer (unsupported by Apple per Topic-A research) | +| **macOS file delivery** | `NSWorkspace.open(urls:withApplicationAt:configuration:)` with `activates: false`, or pasteboard + `CGEventPostToPid(cmd+v)` to the target PID | `NSDraggingSession` (requires NSApp event loop + focus steal — not headless) | +| **macOS screenshot** | ScreenCaptureKit `SCScreenshotManager.captureImage` — headless, but requires Screen Recording TCC | `/usr/sbin/screencapture` subprocess (legacy only, behind `--screenshot-backend legacy`) | +| **Windows observation** | `IUIAutomation.ElementFromHandle(hwnd)` — works on same-user, same-session visible/minimized windows at an accessible integrity level without foregrounding | Cross-session, secure desktop, locked desktop, Session 0, Server Core | +| **Windows action** | UIA pattern invocation (`InvokePattern.Invoke`, `ValuePattern.SetValue`, `TogglePattern.Toggle`, `ExpandCollapsePattern.Expand`, `SelectionItemPattern.Select`) — all focus-independent | `SendInput` as a primary path — it IS focus-dependent; only fallback when no UIA pattern applies, and gated by `AttachThreadInput + SetFocus` worker-thread dance | +| **Windows events (`watch_element`)** | UIA event handler on dedicated MTA apartment thread — UIA explicitly supports cross-thread event delivery per Microsoft's 2025 threading doc | Caching `IUIAutomationElement` across apartment boundaries (apartment-affine handles invalidate) | +| **Windows file delivery** | App/shell delivery first: app URI handlers, `ShellExecuteEx`, `IFileOperation` for filesystem destinations, and `CF_HDROP` clipboard paste where accepted | Cursor-synthesized drag as primary path; using `IDataObject + DoDragDrop` before a policy-gated spike proves target behavior | +| **Windows screenshot** | `windows-capture` (`Windows.Graphics.Capture` — active interactive DWM session, Windows 10 1903+) | Session 0 / Server Core / locked or secure desktop capture; `PrintWindow` is fallback-only | +| **Skeleton traversal** | Core `snapshot_ref.rs` + `adapter.get_subtree(handle, opts)` is platform-agnostic — skeleton works on any window without focus, on both platforms | — | +| **Clipboard** | `NSPasteboard.general` (macOS) / `OpenClipboard` (Windows, no HWND required when passing `NULL`) | — | + +### Verification + +Every Phase 2 integration test MUST assert headless-ness explicitly: +- Target window is **NOT** the focused window at test entry (send test driver to background first). +- agent-desktop CLI runs with stdin/stdout/stderr as pipes (no TTY) where possible. +- Before and after the command, `list-windows --focused-only` returns the SAME focused window — no focus steal. +- Cursor position is unchanged for commands that aren't `hover`/`drag`/`mouse-*`. Click uses semantic accessibility paths by default; coordinate clicking requires an explicit physical path. + +### Skeleton traversal invariant + +The `--skeleton` / `--root @ref` progressive traversal pattern (P2-O-skeleton, shipped in v0.1.11 for macOS) is preserved and extended to Windows in Unit 3. The contract: +- `snapshot --skeleton` clamps depth to 3 and annotates truncated containers with `children_count`. +- Named / described containers at the depth boundary receive refs as drill-down targets. +- `snapshot --root @ref` walks from a previous-snapshot ref with **scoped invalidation** (only that ref's subtree refs are replaced on re-drill). +- Refmap write-side 1 MB guard prevents runaway ref counts. +- Works on unfocused windows on both platforms (Windows `ElementFromHandle(hwnd)` + macOS `AXUIElementCreateApplication(pid)` are both focus-independent). + +Windows implementation notes (research-driven): +- Use **`ControlViewWalker`** (NOT `RawViewWalker` or `ContentViewWalker`) — `IsControlElement` auto-filters layout noise and complements the Electron depth-skip in Unit 4. +- `children_count` via `FindAll(TreeScope_Children, TrueCondition)` — single COM round-trip, no per-child property fetch. +- Fresh `UICacheRequest` per drill-down call — cached elements do not survive CLI process boundaries. +- Expected token savings on VS Code / Slack Electron track macOS's 50-100× once U4's `--force-electron-a11y` + empty-`UIA_Group`/`UIA_Custom` depth-skip are in place. + +## Problem Frame + +Three orthogonal problems share one release (see origin §What Phase 2 is solving): + +1. **Three-platform reach.** macOS is the only shipping platform. Phase 2 brings Windows online with an identical command surface and JSON contract, ahead of Linux in Phase 3. +2. **Identifier instability.** Today every ref resolves via `(pid, role, name, bounds_hash)`. Electron trees, localized apps, and custom-rendered controls fray this and inflate `STALE_REF` rates. Stable-selector fields (`identifier`, `subrole`, `role_description`, `placeholder`, `dom_id`, `dom_classes`) — free on macOS and native to UIA — collapse the churn. +3. **Polling-shaped waiting.** `wait --element` polls every 100 ms. `watch_element` replaces it with sub-500 ms push notifications on both platforms. + +Five smaller gaps are cheap individually but collectively move agent-desktop from "macOS-only observation tool" to "cross-platform agent automation runtime": modern screenshot APIs (ScreenCaptureKit / windows-capture), text-range primitives, new `Action` variants, new surfaces (Toolbar / Spotlight / Dock / MenuBarExtras / Windows shell surfaces), tri-state permission probing, and an FFI registry migration that makes the Phase 4 MCP crate trivial to ship. + +Nothing is deferred to Phase 3 that was in the Phase 2 brainstorm scope — the earlier 2a/2b split was rejected (see origin §D1). + +## Requirements Trace + +Each requirement maps 1:1 to a `phases.md` P2-O* objective (see origin §Acceptance criteria). GA ships when every requirement's metric is green. + +- **R1 (P2-O1)** — Windows adapter: `snapshot --app Explorer` returns a valid tree with refs; same for Notepad, Settings, VS Code, Edge. +- **R2 (P2-O2 — review-refined)** — Cross-platform parity on a structurally-identical app (Calculator on both platforms): `role` set jaccard ≥ 0.85; `identifier` equality where non-empty on both sides; ref count within ±15%; `available_actions` set union is a superset of common-actions across the role map. "Structurally identical" replaced the earlier byte-identical aspiration, which UIA↔AX role-mapping cannot guarantee. +- **R3 (P2-O3)** — Windows input: `click @e5`, `type @e2 "hello"`, `press ctrl+c`, every mouse command succeed against a test app. +- **R4 (P2-O4)** — Windows screenshot: `screenshot --app Notepad` produces a valid PNG via `windows-capture`, SSIM-matches `PrintWindow` fallback. +- **R5 (P2-O5)** — Windows clipboard: get/set/clear roundtrip for ASCII and Unicode. +- **R6 (P2-O6)** — Windows CI: `windows-latest` runs build, clippy, unit, contract, and non-interactive tests on every PR. UIA/shell integration tests that require Explorer, Start, Action Center, Quick Settings, or an unlocked desktop run on a labeled interactive/self-hosted Windows job or assert structured unsupported behavior when the shell is absent. +- **R7 (P2-O7)** — Windows release: x86_64 + aarch64 `.exe` and FFI archives ship with the Phase 2 tag; `npm install` works on both. +- **R8 (P2-O8)** — Stable-selector fields populated on both platforms; measurable `STALE_REF` rate drop vs Phase 1 baseline. +- **R9 (P2-O9 — research-refined)** — Action variants (`LongPress`, `ForceClick`, `ShowMenu`, `DeliverFiles` (renamed from `FileDrop` because `NSDraggingSession` is not headless-compatible — see Unit 12), `WindowRaise`, `Cancel`, `SelectRange`, `InsertAtCaret`) exposed via CLI and each green in a platform-appropriate integration test. Semantic actions assert no focus steal. Explicit focus/window/physical actions assert the side effect is policy-authorized and documented. +- **R10 (P2-O10)** — ErrorCode variants (`PermissionRevoked`, `ResourceExhausted`, `AxMessagingTimeout`, `AutomationPermissionDenied`) each have a runtime producer. +- **R11 (P2-O11)** — `watch --event value-changed --ref @e5 --timeout 3000` receives an event within 500 ms of a programmatic value change on both platforms. +- **R12 (P2-O12)** — `text select-range` + `text get-selection` roundtrip; `text insert-at-caret` advances caret correctly on both platforms. +- **R13 (P2-O13)** — Modern screenshot cold-latency <50 ms on both platforms vs ~300 ms macOS subprocess baseline; default is modern, legacy behind `--screenshot-backend legacy`. +- **R14 (P2-O14)** — `snapshot --surface toolbar` on Safari (macOS) and Edge (Windows) works; macOS lists Spotlight / Dock / MenuBarExtras; Windows lists present shell surfaces (`Taskbar`, `SystemTray`, `SystemTrayOverflow`, `StartMenu`, `ActionCenter`, `QuickSettings`); tray commands function. +- **R15 (P2-O15)** — Electron compat on Windows: VS Code snapshot with `--force-electron-a11y` exposes ≥100 refs at default depth. +- **R16 (P2-O16)** — FFI registry: adding a command requires only a new file under `crates/core/src/commands/`; CLI, FFI wrappers, and (future) MCP tools auto-register; `ad_abi_version()` exported; `ad_set_log_callback` receives tracing output during `ad_get_tree`. +- **R17 (P2-O17)** — Permission tri-state: `permissions` output shows AX, Screen Recording, Automation independently on macOS. Denied Screen Recording returns `PermDenied` with Screen-Recording-specific suggestion; denied Automation for a target app returns `AutomationPermissionDenied`. +- **R18 (P2-O18)** — Windows shell coverage: Start menu/search, taskbar, system tray/overflow, Action Center/notification center, Quick Settings, multi-monitor/DPI, virtual desktop detection, UAC/elevated targets, RDP/locked-session behavior, and Explorer-specific file destinations are explicitly covered by commands, surfaces, tests, or documented `PLATFORM_NOT_SUPPORTED` behavior. Windows-only commands still live in core command files with adapter defaults. + +## Scope Boundaries + +- **Included**: every P2-O* objective in `docs/phases.md §Phase 2`; v0.2.0 breaking ABI + JSON schema bump; MSRV bump to 1.82; ARM64 Windows (build-only until GH runner arrives); tri-state permission model on macOS; registry-driven FFI codegen; `ad_abi_version()` export. + +### Deferred to Separate Tasks + +- **Linux adapter** — Phase 3 (separate plan). Trait methods ship with default `not_supported()` implementations in U1 so Linux mirrors later without re-opening core. +- **MCP server mode** — Phase 4. +- **Daemon, sessions, audit log, policy engine, OCR fallback** — Phase 5. +- **Streamable HTTP transport** — Phase 4 (stdio confirmed sufficient for MS Agent Framework 1.0). +- **Package-manager distribution (brew/winget/snap)** — Phase 5. +- **Async FFI** — Phase 4 once MCP streaming arrives (see origin §D9). +- **`ForceClick` on Linux** — permanent per-platform divergence (returns `ActionNotSupported`), see origin §D8, R9. + +## Context & Research + +### Relevant Code and Patterns + +- `crates/core/src/node.rs` — `AccessibilityNode` (10 fields today, L3–L33). Unit 1 nests new selectors via `#[serde(flatten)]`. +- `crates/core/src/error.rs` — `ErrorCode` enum (12 variants, L4–L19; not yet `#[non_exhaustive]`). Paired with `AdResult` variant-count `const _: () = assert!(…)` in `crates/ffi/src/error.rs:57`. +- `crates/core/src/action.rs` — `Action` enum (already `#[non_exhaustive]`, 21 variants). Unit 1 adds 8 new variants. +- `crates/core/src/adapter.rs` — `PlatformAdapter` trait (~28 methods with `not_supported()` defaults, L117–L278). New methods land as additive defaults. +- `crates/core/src/refs.rs` — `RefEntry` (9 fields, L13–L28). Unit 1 adds `identifier: Option<String>` for selector-preferred resolution. +- `crates/core/src/commands/` — 54 files (53 commands + `helpers.rs`), one per command; `click.rs` (14 L), `list_windows.rs` (18 L), `type_text.rs` (24 L), `snapshot.rs` (226 L), `wait.rs` (229 L) are the representative patterns for Unit 6 / Unit 7 / Unit 8 new commands. The command count is used throughout this plan (document-review note). +- `crates/macos/src/tree/element.rs` (404 L) — where Unit 5 stable-selector reads land. File is at the 400-LOC cap; plan mandates a 2-way split into `element.rs` (core) + `element_selectors.rs` (new selector readers) before adding reads. +- `crates/macos/src/tree/resolve.rs:20` — `resolve_depth = 50` already matches the `ABSOLUTE_MAX_DEPTH` target Unit 4 mirrors on Windows. +- `crates/ffi/build.rs` — existing cbindgen invocation + header-path stamp pattern; Unit 2 extends this, it does not replace it. +- `crates/ffi/src/error.rs:5-60` — the parity `const` assertion pair; Unit 1 updates both arrays atomically with new `ErrorCode` variants. +- `crates/ffi/include/agent_desktop.h` — committed ABI contract; drift check at `.github/workflows/ci.yml` step "FFI header drift check". +- `.github/workflows/release.yml` — FFI matrix already includes `x86_64-pc-windows-msvc`; Unit 13 adds the `aarch64-pc-windows-msvc` FFI row, the Windows CLI binary row, and npm postinstall branches. +- `.github/workflows/ci.yml` — `test` job runs only on `macos-latest` today; Unit 13 adds a sibling `test-windows` job. +- `.githooks/pre-commit` — runs `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`, `cargo test --lib --workspace` when Rust/TOML files are staged; awareness only, no change required. +- `skills/agent-desktop/SKILL.md` (227 L) and `skills/agent-desktop-ffi/SKILL.md` (78 L) exist; `skills/agent-desktop-macos/` **does not** exist — macOS content lives at `skills/agent-desktop/references/macos.md`. Unit 14 creates `skills/agent-desktop-windows/` as a sibling and updates the core skill to three-platform. + +### Institutional Learnings + +- `docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md` — **DRY via config structs, threshold "4+ positional params + 1 new distinguishing value."** Extraction bar: shared parameter types at trait boundaries and shared policy (not shared implementations). Phase 2 extracts **4** shared types in `crates/core`: + - `ActionDispatchConfig` — parameter type for action execution. Carries policy only: `fallback_to_cursor: bool`, `timeout_ms`, `blocked_combos: &'static [KeyCombo]`. **Phantom DRY warning:** the dispatch chain itself (AX action strings on macOS vs UIA pattern interfaces on Windows) is categorically different types — do NOT extract a shared dispatch chain. Share only the policy. + - `WatchElementConfig` — `watch_element` trait method parameter. Carries `events: &[EventKind]`, `timeout_ms`, `max_subscriptions`, `hard_join_multiplier` (default 2.0). + - `TextRangeConfig` — text primitive trait method parameter. Carries `utf16_semantics: true` marker + `check_password_field: bool` (default true; see Unit 8 security gate) + per-platform text-pattern entrypoint hints. + - `ScreenshotBackendConfig` — `get_screenshot_with_backend` parameter. Carries `backend: ScreenshotBackend`, `dimensions: Option<(u32, u32)>`, `pixel_format`, `encoding`. + **Rejected (reviewed and dropped — YAGNI and duplication risk):** + - `TreeWalkConfig` — duplicates the existing `TreeOptions` struct at `crates/core/src/adapter.rs:27`. Extend `TreeOptions` with `force_electron_a11y` instead of creating a parallel struct. + - `SurfaceDetectionConfig` — per-platform surface policy (window-class lists, AX-role overrides) should not leak platform specifics into core types. Keep platform-local in `crates/macos/src/tree/surfaces.rs` and `crates/windows/src/tree/surfaces.rs`. + - `SelectorReadConfig` — the 6 selector attribute names are categorically different types across platforms (macOS AX string constants vs Windows UIA property IDs). No shared shape exists to extract. + - `EventWorker` trait — the "spawn → attach → mpsc → timeout → join" shell is ~15 LOC and the teardown primitives (`CFRunLoopStop` vs `PostThreadMessage(WM_QUIT)`) are categorically different. Inline in each platform's `watch_element` implementation. Revisit at Phase 3 when a third platform proves the shape. + - `NotificationSession` trait — macOS `nc_session.rs` and Windows `action_center.rs` have superficially-similar "open/dismiss/close" lifecycles but radically different state (NSUserNotificationCenter observer vs UIA tree walk). No code is generic over both. Document shape alignment in code comments; revisit at Phase 3. +- `docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md` — **Separate `INVALID_ARGS` (bad selector syntax), `STALE_REF` (valid syntax, gone element), and `TIMEOUT`.** Apply to `watch_element` in Unit 7 and text-range commands in Unit 8. Boundary-node pattern at subscription caps. +- `docs/solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md` — **`build.rs` stamps absolute paths for generated artifacts; committed copy is the ABI contract; never self-heal.** Apply to Unit 2's generated `ad_*` wrappers and JSON schemas, and to Unit 1's `ad_abi_version` constant. CI drift-checks each. +- `docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md` — **Stable identifiers are optional fingerprints carried alongside index/handle; tri-state UTF-8 decode at FFI boundary.** Directly applies to Unit 5 (`identifier` field) and Unit 10 (tray/notification indices on Windows). +- **Private-memory rule: port `electron-compat.md` and `macos-ax-gotchas.md` from `~/.claude/.../memory/` into `docs/solutions/` before Unit 4 opens.** Windows contributors must inherit the Electron depth-skip rules and TCC traps without relying on private auto-memory. Tracked in Unit 14's doc cleanup. + +### External References + +- UIA + `uiautomation 0.24` crate docs (framework-docs-researcher cache): `UITreeWalker`, `UICacheRequest`, pattern interface set (`InvokePattern`, `ValuePattern`, `ExpandCollapsePattern`, `SelectionItemPattern`, `TogglePattern`, `ScrollPattern`, `TextPattern`, `TextRange`). +- `windows 0.62.2` crate: `Windows.Graphics.Capture`, `Direct3D11CaptureFramePool`, `SendInput`, `OpenClipboard`, `IDataObject`. +- `windows-capture 1.5.4`: pinned exactly because newer majors may move the capture API surface — re-evaluate post-Phase 2. `Capture::start` + frame-callback API validated against 1.5.4. +- ScreenCaptureKit via `screencapturekit 1.5` (doom-fish fork) — `SCShareableContent.windows`, `SCScreenshotManager.captureImage(contentFilter:config:)`. +- `objc2 0.6` replaces ad-hoc `objc` message sends, scoped to `system/screenshot.rs` + `system/permissions.rs`. +- **`inventory` and `linkme` both rejected** (research Topic B). Replaced with `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe across ld64, ld-prime, GNU ld, lld, MSVC link.exe, zero linker magic. +- `schemars 1.2` — required by P2-O16 for command-arg JSON Schema generation. +- MS Agent Framework 1.0 MCP transport reference — stdio is sufficient for Phase 4; no streamable HTTP required. + +## Key Technical Decisions + +Carried forward from origin §Decisions (D1–D17). Each is pinned here with its unit anchor so planning enforces them. + +- **KD1 (D1)** Single release, no 2a/2b split. Scope managed via units, not sub-phases. *Enforced by:* every unit has a Phase 2 GA gate. +- **KD2 (D2)** ~15 implementation units, one PR each; dependencies flow left-to-right in the graph (§High-Level Technical Design). +- **KD3 (D3)** `WindowInfo.pid` stays `i32`; Windows adapter narrow-casts `u32 → i32` at the adapter boundary via a helper `fn narrow_pid(dword: u32) -> Result<i32, AdapterError>` that returns `ErrorCode::ResourceExhausted` (reusing the new variant added in Unit 1) with `platform_detail: "PID exceeds i32::MAX; Windows kernel should never produce this value"` for values above `i32::MAX`. `Internal` is wrong here because this is a boundary failure, not a bug in agent-desktop; `ResourceExhausted` costs zero additional ABI surface and is semantically correct (PID space exhausted relative to `i32` representation). Documented in Unit 1; implemented in Unit 3. +- **KD4 (D4)** `ErrorCode` gains `#[non_exhaustive]` as the **first** sub-PR in Unit 1, before any variant addition. The parity `const _: () = assert!(…)` in `crates/ffi/src/error.rs` is updated atomically with each new variant — the assertion is the compile-time gate. +- **KD5 (D5)** Registry migration (Unit 2) ships before the Windows adapter so every new Phase 2 command is born in the registry. +- **KD6 (D6)** Modern screenshot is the **default** on both platforms. `--screenshot-backend legacy` reaches the Phase 1 subprocess (macOS) / `PrintWindow` (Windows) path for WDA-protected windows and restricted environments. +- **KD7 (D7)** All 6 stable-selector fields ship, nested as a `StableSelectors` sub-struct on `AccessibilityNode` with `#[serde(flatten)]` + per-field `skip_serializing_if`. JSON wire shape preserved. Rust field count on `AccessibilityNode` becomes 11; grandfathered (KD15). +- **KD8 (D8)** All 9 new `Action` variants ship. `Watch(WatchSpec)` is **not** an `Action` — `watch_element` is an adapter method (origin §D9). `ForceClick` returns `ActionNotSupported` on Linux (Phase 3) — legitimate platform divergence, not a deferral. +- **KD9 (D9 + review-refined)** `watch_element` is synchronous at the public API. Per-call worker thread (macOS: `CFRunLoop` + `AXObserver`; Windows: MTA thread + UIA event handler) → `std::sync::mpsc` → caller. Trait method takes `&RefEntry` (not `&NativeHandle`) — worker re-resolves the element fresh on its own thread, preserving the `NativeHandle` `!Send`/`!Sync` invariant. **The orchestration shell (~15 LOC: spawn → attach → mpsc → timeout → join) is inlined in each platform's `watch_element` implementation** — no shared `EventWorker` trait. Rationale: with N=2 platforms and categorically different teardown primitives (`CFRunLoopStop` vs `PostThreadMessage(WM_QUIT)`), a trait with one abstract method per side is a framework, not DRY. Revisit at Phase 3 when AT-SPI proves the three-way shape. **AX threading invariant:** `AXObserverCreate` and callback dispatch on the observer's CFRunLoop thread is Apple-blessed for non-main threads; `AXUIElementCopy*` calls from the worker are validated by Unit 7's opening spike against macOS 14/15. If the spike shows thread-hostility on specific calls, the worker hops to main via a `with_main_thread(|| …)` helper that uses `dispatch_sync` to the main queue. Same pattern on Windows: COM apartment hostility escapes via `SendMessage` to a main-thread windowless receiver. Hard-join timeout = `2 × user_timeout_ms`; refuse-to-exit worker returns `ErrorCode::Internal`. Async surfaces are Phase 4's problem. Thread pool not shared with Unit 9. +- **KD10 (D10)** `PermissionReport` migrates to a struct with `accessibility`, `screen_recording`, `automation` tri-state fields. Breaking JSON schema on the `permissions` command and breaking FFI ABI. Lands atomically with `ad_abi_version()` bump in Unit 1. +- **KD11 (D11)** `BLOCKED_COMBOS` becomes a `PlatformAdapter` trait method `fn blocked_combos(&self) -> &'static [KeyCombo]`. macOS blocks `cmd+q`, `cmd+shift+q`. Windows blocks `alt+f4`, `ctrl+alt+delete`, `win+l`. `key-down` / `key-up` safety check rejects any combo that matches `blocked_combos()` as a whole — **no persistent modifier-state file** (keeps the tool stateless; rejecting solo modifiers is never needed because blocked combos always require at least one non-modifier key). +- **KD12 (D12)** Electron/WebView2 compat on Windows mirrors macOS: depth-skip for non-semantic wrappers, resolver depth 50, surface detection that treats the focused window AS the target surface, `--force-electron-a11y` CLI override. +- **KD13 (D13)** ARM64 Windows ships in Phase 2 alongside x86_64. `aarch64-pc-windows-msvc` is build-only until a GH runner arrives (test matrix adds it post-hoc in a follow-up PR). `npm/scripts/postinstall.js` gains `win32-x64` and `win32-arm64` branches. +- **KD14 (D14)** CI matrix = macOS + Windows for full tests; Ubuntu for fmt. `cargo tree -p agent-desktop-core` isolation check runs on both. +- **KD15 (D15 + research-refined)** Dependencies: + - Windows: `uiautomation 0.24`, `windows 0.62.2` (matches `windows-capture 1.5`'s own pin), `windows-capture = "1.5.4"` (latest stable, published crates.io) + - macOS: `objc2 0.6`, `screencapturekit = "1.5"` (**published crates.io** — the doom-fish fork is the canonical maintained crate as of Q1 2026; NOT a git-SHA pin) + - Cross-platform: `schemars 1.2` (deferred to Phase 4 — see KD16 below) + - **NO `inventory` / `linkme`** (research Topic B — link-GC risk across ld64/ld-prime/GNU ld/lld/MSVC is real; both crates' ctor-based / linker-section patterns are unreliable for cdylib consumers) + - Command registry uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` — deterministic, cdylib-safe, zero linker magic (research Topic B recommendation) + - MSRV: `1.82` (required by `windows 0.62.2`). `AccessibilityNode` 10-field grandfather preserved — only new selectors nest into `StableSelectors`. +- **KD16 (D16)** Cross-compile-first workflow holds: macOS dev → `cargo check --target x86_64-pc-windows-msvc` → Windows CI integration. Pre-commit hook runs the cross-check best-effort (warn, never fail). +- **KD17 (D17 + review-refined)** Pre-1.0 FFI policy, published at `crates/ffi/README.md` during the v0.1.14 prep release (see §Phased Delivery). Policy matrix: + - **`#[non_exhaustive]` enum variant addition (additive)** → **no major bump**. C consumers MUST use `default:` / wildcard fallthrough; library documents this as a hard contract. Defense-in-depth: a reserved `AD_RESULT_UNKNOWN = -99` sentinel is exported so consumers can map any unrecognized integer to a known value explicitly. Rust never produces this sentinel; it exists purely for consumer dispatch tables. Guarded by the compile-time `const _: () = assert!(ErrorCode vs AdResult parity)` gate. + - **FFI struct layout change (field add/remove/reorder)** → **major bump**. + - **FFI function signature change (parameter or return type)** → **major bump**. + - **New `extern "C" ad_*` function (additive)** → **no bump**; consumers must check symbol presence via `dlsym`. + - **Removal of any exported symbol** → **major bump**. + Consumer version handshake: `ad_init(expected_major: u32) -> AdResult` — the ONLY FFI function that must be called before any other `ad_*`. Fails closed with `AdResult::ErrInvalidArgs` if `expected_major != AD_ABI_VERSION_MAJOR`. Without this call, subsequent `ad_*` calls fail closed with `ErrInternal + "ad_init not called"`. Converts the "consumers SHOULD check" advisory into an enforced handshake. `ad_init` ships in v0.1.14 returning `MAJOR = 1`. + **v0.1.14 prep release** ships `#[non_exhaustive]` + `ad_abi_version()` + `ad_init()` + `AD_RESULT_UNKNOWN` sentinel + `crates/ffi/README.md` policy doc, with **no variant additions, no struct-layout changes**. v0.2.0 ships `AD_ABI_VERSION_MAJOR = 2` atomically with the `PermissionReport` tri-state layout change in sub-PR 1g. Phase 3 Linux adapter adds variants additively — no bump. + +## Open Questions + +### Resolved During Planning + +- **StableSelectors shape** (origin open question): Use an inline `StableSelectors` sub-struct with `#[serde(flatten)]` + per-field `#[serde(skip_serializing_if = "…")]`. No `Option<StableSelectors>` wrapper. Rationale: preserves exact JSON wire shape; each selector is individually skippable when empty; cbindgen emits cleaner output for a nested struct than 6 more optional top-level fields. Landed in Unit 1. +- **Modifier-state tracking for `key-down` / `key-up` safety check** (origin open question): Do **not** persist modifier state. The check evaluates the combo passed to `key-down` / `key-up` directly and rejects if it matches any entry in `blocked_combos()` as a whole combo. Blocked combos always include a non-modifier key (e.g., `cmd+q`, `alt+f4`), so solo modifier key-downs are never rejected. Keeps the tool stateless — matches the Phase 1 invariant. Landed in Unit 1 / Unit 3. +- **Thread-pool consolidation** (origin open question): Keep per-call worker threads in Units 7 and 9. Different lifetimes (watch_element: seconds-to-tens-of-seconds observer loop; screenshot: sub-second single-shot capture). Re-evaluate when Phase 4 daemon arrives. +- **`wait --event` CLI shape** (origin open question): `wait --event <kind> --ref @e5 --timeout 3000`. `--event` repeats for multi-subscription (`--event value-changed --event selection-changed`). `<kind>` accepts the 10 `EventKind` variants named in `docs/phases.md §Phase 2`: `focus-changed`, `value-changed`, `selection-changed`, `children-changed`, `window-opened`, `window-closed`, `menu-opened`, `menu-closed`, `notification-posted`, `element-destroyed`. Event filter expressions beyond kind + ref are deferred. +- **`AccessibilityNode` 10-field grandfather** (origin open question): Do not refactor existing flat fields. Only new `StableSelectors` is nested. Smaller blast radius; pre-existing JSON consumers unaffected. +- **`inventory` vs `linkme`** (origin §D15 ambiguity — **research-resolved**): **Neither.** Research Topic B found neither crate reliably survives link-GC across ld64, ld-prime, GNU ld, lld, and MSVC link.exe for cdylib consumers; `deterministic registry metadata` ctor sites are stripped when an `rlib` is linked into a binary that never references a symbol from that rlib. Instead, the command registry is built at compile time via `crates/core/build.rs` that enumerates `crates/core/src/commands/*.rs` (one-command-per-file is already a CLAUDE.md invariant) and codegens a `pub fn descriptors() -> &'static [CommandDescriptor]` static. Deterministic, cdylib-safe, zero linker magic. FFI wrapper codegen in `crates/ffi/` uses the same `build.rs` approach reading the same source listing. Simplification: no xtask crate needed. + +### Deferred to Implementation + +- **Exact `uiautomation` crate API surface** for `UICacheRequest` batching — requires a spike against real apps on Windows CI before Unit 3 freezes. +- **Exact `AXObserver` teardown sequence on `CFRunLoop` stop** — named as Unit 7's opening spike. Validates non-main-thread AX observer behavior against Finder, TextEdit, VS Code on macOS 14/15. +- **`windows-capture 1.5.4` API against `windows 0.62.2`** — verify `Capture::start` + frame-callback API compiles together before merging Unit 9. Future patch bumps require the same spike before changing the pin. +- **`SCShareableContent` windowing** — exact API shape for filtering to a `CGWindowID` via `SCContentFilter` on macOS 14 vs 15. Unit 9's macOS spike. +- **Event handler lifetime on Windows** — `IUIAutomation.AddAutomationEventHandler` must be removed before thread exit, confirmed by Unit 7 spike. +- **`DeliverFiles` per-app URL scheme registry for Tier 1** — Unit 12 builds a small `crates/macos/src/actions/deliver_files_registry.rs` mapping known bundle IDs to their CLI/URL scheme. Initial entries (VS Code, Finder, Preview, TextEdit, Safari, Chrome) are defined at implementation time; the registry is extensible per-release. +- **Exact `AEDeterminePermissionToAutomateTarget` bundle-id argument** for Unit 11 — depends on which target app the user is automating; solved at call site, not trait signature. + +## Output Structure + +New/rewritten directory layouts. Paths shown are repo-relative. + +``` +crates/ +├── core/ +│ └── src/ +│ ├── node.rs # +StableSelectors sub-struct; flatten into AccessibilityNode +│ ├── error.rs # +4 ErrorCode variants; +#[non_exhaustive] +│ ├── action.rs # +8 Action variants +│ ├── adapter.rs # +blocked_combos, watch_element, text-range, get_screenshot_with_backend; PermissionReport → tri-state struct +│ ├── refs.rs # +identifier field on RefEntry +│ ├── event.rs # NEW — EventKind, ElementEvent, WatchSpec (supporting types for watch_element) +│ ├── text_range.rs # NEW — TextRange, TextSelection (supporting types for text primitives) +│ ├── screenshot_backend.rs # NEW — ScreenshotBackend enum (Modern / Legacy) +│ ├── permission.rs # NEW — Tri-state PermissionReport struct, extracted from adapter.rs +│ ├── commands/ +│ │ ├── watch.rs # NEW — Unit 7 command +│ │ ├── text_get_selection.rs # NEW — Unit 8 command +│ │ ├── text_select_range.rs # NEW — Unit 8 command +│ │ ├── text_insert_at_caret.rs # NEW — Unit 8 command +│ │ ├── text_at_offset.rs # NEW — Unit 8 command +│ │ ├── list_tray_items.rs # NEW — Unit 3b / Unit 10 command +│ │ ├── click_tray_item.rs # NEW — Unit 3b / Unit 10 command +│ │ └── open_tray_menu.rs # NEW — Unit 3b / Unit 10 command +│ └── registry.rs # NEW — Unit 2: CommandDescriptor type + include!(registry.rs from $OUT_DIR) +├── windows/ +│ └── src/ +│ ├── lib.rs # mod + re-exports (rewritten) +│ ├── adapter.rs # WindowsAdapter: PlatformAdapter impl (Unit 3) +│ ├── tree/ # element, builder (UITreeWalker + UICacheRequest), roles, resolve, surfaces +│ ├── actions/ # dispatch, activate (smart chain), extras, file_drop (U12), force_click (U12) +│ ├── input/ # keyboard (SendInput), mouse (SendInput), clipboard (Win32) +│ ├── events/ # NEW — watch (U7): MTA thread, UIA event handlers +│ ├── text/ # NEW — U8: TextPattern helpers +│ ├── notifications/ # U3a: list, dismiss, interact +│ ├── tray/ # U3b: list, interact (Shell_TrayWnd UIA) +│ └── system/ # app_ops, window_ops, key_dispatch, permissions, screenshot (U9 modern + legacy), wait +├── macos/ +│ └── src/ +│ ├── tree/ +│ │ ├── element.rs # SPLIT (was 404 L) — keep attribute reads +│ │ └── element_selectors.rs # NEW — Unit 5: AXIdentifier / Subrole / RoleDescription / PlaceholderValue / DOMIdentifier / DOMClassList readers +│ ├── events/ # NEW — Unit 7: AXObserver + CFRunLoop worker +│ ├── text/ # NEW — Unit 8: parameterized-attribute helpers +│ └── system/ +│ ├── screenshot.rs # Unit 9: ScreenCaptureKit default, subprocess legacy (split into modern.rs + legacy.rs if LOC pressure) +│ └── permissions.rs # Unit 11: tri-state (AX + Screen Recording + Automation) +├── ffi/ +│ ├── build.rs # Unit 2: extend — uses build-helpers::enumerate_commands to generate ad_* wrappers alongside cbindgen header +│ └── src/ +│ ├── generated/ # NEW — include!() target for build.rs output +│ │ └── wrappers.rs # generated from registry; committed-and-drift-checked like include/agent_desktop.h +│ ├── abi_version.rs # NEW — Unit 1: ad_abi_version() export + AD_ABI_VERSION_MAJOR cbindgen define +│ ├── log_callback.rs # NEW — Unit 2: ad_set_log_callback installs a tracing_subscriber layer +│ └── ... # existing adapter.rs, error.rs, ffi_try.rs, etc. unchanged +skills/ +├── agent-desktop/ # Unit 14: update core skill for three-platform +├── agent-desktop-ffi/ # Unit 14: update for ad_abi_version + ad_set_log_callback +└── agent-desktop-windows/ # NEW — Unit 14: SKILL.md + references/uia.md, references/windows-permissions.md, references/chromium.md +.github/workflows/ +├── ci.yml # Unit 13: +test-windows job +└── release.yml # Unit 13: +aarch64-pc-windows-msvc + Windows CLI row +npm/ +└── scripts/ + └── postinstall.js # Unit 13: +win32-x64 + win32-arm64 branches +``` + +The implementer may adjust subfolder shape during implementation if it improves clarity; the per-unit `**Files:**` sections are authoritative for what each unit creates. + +## High-Level Technical Design + +> *The diagrams below illustrate intended approach and are directional guidance for review, not implementation specification.* + +### Dependency graph across units + +```mermaid +flowchart LR + U1[U1: Core pre-work<br/>types + trait methods + MSRV] + U2[U2: Registry migration<br/>build.rs filesystem enumeration codegen] + U3[U3: Windows adapter<br/>UIA tree/actions/input/system] + U3a[U3a: Windows notifications] + U3b[U3b: Windows tray] + U4[U4: Windows Electron compat] + U5[U5: Stable-selector population] + U6[U6: Action variants<br/>LongPress/ShowMenu/WindowRaise/Cancel] + U7[U7: watch_element<br/>AXObserver + UIA events] + U8[U8: Text range primitives] + U9[U9: Modern screenshot] + U10[U10: New surfaces] + U11[U11: Permission tri-state macOS] + U12[U12: DeliverFiles + ForceClick] + U13[U13: Windows CI + release matrix] + U14[U14: Skills + README + phases.md sync] + + U1 --> U2 + U1 --> U3 + U1 --> U4 + U1 --> U5 + U1 --> U6 + U1 --> U7 + U1 --> U8 + U1 --> U9 + U1 --> U10 + U1 --> U11 + U1 --> U12 + U2 --> U3 + U2 --> U5 + U2 --> U6 + U2 --> U7 + U2 --> U8 + U2 --> U9 + U2 --> U10 + U2 --> U11 + U2 --> U12 + U3 --> U3a + U3 --> U3b + U3 --> U4 + U3 --> U7 + U3 --> U8 + U3 --> U9 + U3 --> U10 + U3 --> U12 + U13 -.parallel to U3..U12.-> U3 + U3 --> U13 + U13 --> U14 + U3a --> U14 + U3b --> U14 + U4 --> U14 + U5 --> U14 + U6 --> U14 + U7 --> U14 + U8 --> U14 + U9 --> U14 + U10 --> U14 + U11 --> U14 + U12 --> U14 +``` + +### `watch_element` lifecycle (directional) + +``` +CLI: wait --event value-changed --ref @e5 --timeout 3000 + │ + ▼ +commands/watch.rs + │ resolve ref → NativeHandle + ▼ +PlatformAdapter::watch_element(&handle, &[ValueChanged], 3000ms) + │ + ├─ macOS: spawn worker → CFRunLoopRun + │ AXObserverCreate(pid) → AddNotification(kAXValueChangedNotification) + │ callback pushes ElementEvent into mpsc::Sender + │ timeout: CFRunLoopStop + JoinHandle::join + │ + └─ Windows: spawn worker → CoInitializeEx(MTA) + IUIAutomation.AddPropertyChangedEventHandler + handler pushes ElementEvent into mpsc::Sender + timeout: RemoveEventHandler + JoinHandle::join + │ + ▼ +main thread: recv with Duration until deadline → Vec<ElementEvent> + │ + ▼ +JSON envelope with events array +``` + +### Registry → codegen flow (Unit 2 — research-refined: build.rs filesystem enumeration) + +``` +crates/core/src/commands/<cmd>.rs ← source of truth (file system) + │ + │ top-of-file marker: + │ ///! command_meta { name = "click", summary = "Click an element by ref" } + │ + │ body: + │ pub fn descriptor() -> CommandDescriptor { … } + ▼ +build-helpers::enumerate_commands(Path) ← pure file walk + regex, zero linker magic + │ + ├─ crates/core/build.rs → emit $OUT_DIR/registry.rs + │ pub static DESCRIPTORS: &[CommandDescriptor] = &[ click::descriptor(), … ]; + │ + ├─ crates/ffi/build.rs → emit $OUT_DIR/wrappers.rs + │ #[no_mangle] pub extern "C" fn ad_click(…) -> AdResult { … } + │ #[no_mangle] pub extern "C" fn ad_type_text(…) -> AdResult { … } + │ … + │ + ├─ src/dispatch.rs (CLI) → DESCRIPTORS.iter().find(|d| d.name == name) + │ + └─ crates/mcp/ (Phase 4) → same build-helpers::enumerate_commands → + emit rmcp #[tool] per descriptor + +NO deterministic registry metadata, NO linkme, NO xtask, NO ctor sites, +NO link-GC mitigation needed — extern "C" symbols are directly exported +from the cdylib and visible via nm -g. +``` + +## Implementation Units + +Each unit is one reviewable PR unless explicitly flagged as multi-sub-PR (Unit 1 and Unit 2 are multi-sub-PR due to blast radius). Dependencies follow §High-Level Technical Design. The checkbox syntax drives progress tracking. + +### - [ ] Unit 1: Core pre-work — types, trait method stubs, MSRV bump, FFI ABI version + +**Goal:** Land every additive type change, trait method stub, and MSRV bump before any adapter code opens. Every P2-O* objective that mutates core types resolves its type surface here. + +**Requirements:** R8, R9, R10, R11, R12, R13, R14, R16, R17 + +**Dependencies:** None (first unit) + +**Files:** +- Modify: `Cargo.toml` (workspace `rust-version = "1.82"`) +- Modify: `rust-toolchain.toml` (no target change yet; that lives in U13) +- Modify: `crates/core/src/error.rs` (add `#[non_exhaustive]`; add `PermissionRevoked`, `ResourceExhausted`, `AxMessagingTimeout`, `AutomationPermissionDenied`) +- Modify: `crates/core/src/action.rs` (add `LongPress { duration_ms: u64 }`, `ForceClick`, `ShowMenu`, `DeliverFiles(Vec<std::path::PathBuf>)` (renamed from `FileDrop` per research), `WindowRaise`, `Cancel`, `SelectRange { start: u32, length: u32 }`, `InsertAtCaret(String)`) +- Modify: `crates/core/src/node.rs` (introduce `StableSelectors` struct; add `#[serde(flatten)] pub selectors: StableSelectors` field on `AccessibilityNode`) +- Modify: `crates/core/src/refs.rs` (add `identifier: Option<String>` to `RefEntry`; populate in allocator when available; prefer-identifier logic is Unit 5) +- Modify: `crates/core/src/adapter.rs` (extract `PermissionReport` into `crates/core/src/permission.rs` as tri-state struct; add trait methods `blocked_combos`, `watch_element`, `get_text_selection`, `set_text_selection`, `get_text_at`, `insert_text_at_caret`, `get_screenshot_with_backend`; all default to `not_supported()`) +- Create: `crates/core/src/event.rs` (`EventKind` enum with 10 variants; `ElementEvent` struct; `WatchSpec`) +- Create: `crates/core/src/text_range.rs` (`TextRange`, `TextSelection`) +- Create: `crates/core/src/screenshot_backend.rs` (`ScreenshotBackend { Modern, Legacy }`) +- Create: `crates/core/src/permission.rs` (tri-state `PermissionReport`; `TriState { Granted, Denied { suggestion: String }, Unknown }`) +- Modify: `crates/core/src/lib.rs` (re-exports) +- Modify: `crates/ffi/src/error.rs` (extend both `const fn` variant-count arrays with the 4 new variants; add matching `AdResult::Err*` discriminants preserving existing ordering; update `error_code_to_result` match) +- Modify: `crates/ffi/src/adapter.rs` (FFI-facing `AdPermissionReport` struct mirroring tri-state; FFI conversion helper) +- Create: `crates/ffi/src/abi_version.rs` (`pub const AD_ABI_VERSION_MAJOR: u32 = 1;` at v0.1.14 anchor; bumped to `2` at sub-PR 1g when layout actually changes) + `ad_abi_version()` extern "C" returning `u32` + `ad_init(expected_major: u32) -> AdResult` enforced version-negotiation handshake + `pub const AD_RESULT_UNKNOWN: i32 = -99;` sentinel exported to cbindgen +- Modify: `crates/ffi/cbindgen.toml` (add `AD_ABI_VERSION_MAJOR` to `[defines]`) +- Modify: `crates/ffi/include/agent_desktop.h` (regenerated, committed) +- Modify: `src/cli_args.rs` (new arg structs are stubbed/empty until U6, U7, U8 fill; cli.rs arm names reserved to prevent renumbering) +- Modify: `src/dispatch.rs` (map reserved arms to new command modules with `unimplemented!()` gated behind test-only until U6/U7/U8 land — **except** the Unit 2 dispatcher variant, which lands in U2 and never uses `unimplemented!()` in shipped binaries) +- Test: `crates/core/src/node.rs` (new tests: flatten shape, serde roundtrip, skip_serializing_if per selector field) +- Test: `crates/core/src/error.rs` (new variants serialize to SCREAMING_SNAKE_CASE) +- Test: `crates/core/src/action.rs` (new variants serde roundtrip including `PathBuf` in `DeliverFiles`) +- Test: `crates/core/src/permission.rs` (tri-state struct serde; JSON shape matches spec) +- Test: `crates/core/src/event.rs`, `text_range.rs`, `screenshot_backend.rs` (serde roundtrips) +- Test: `crates/ffi/tests/abi_version.rs` (ad_abi_version returns `AD_ABI_VERSION_MAJOR`) +- Test: `crates/ffi/src/error.rs` (parity-count assertion passes; `ErrorCode::PermissionRevoked` → `AdResult::ErrPermissionRevoked`) + +**Approach:** +**Sub-PR ordering (refined by deepening pass — `ad_abi_version` first so every later ABI-affecting sub-PR can CI-assert it bumped):** + +- **Sub-PR 1a: `ad_abi_version()` export + FFI policy publication (document-review refinement).** Add `crates/ffi/src/abi_version.rs` with `pub const AD_ABI_VERSION_MAJOR: u32 = 1;` (anchor Phase 1 implicit value explicitly — **not** 2 yet, to avoid the mid-series lie where consumers see `ad_abi_version() = 2` while the struct layout is still v1 shape) and `ad_abi_version()` extern "C". Add the `AD_ABI_VERSION_MAJOR` cbindgen `[defines]` entry. Create `crates/ffi/README.md` documenting the policy matrix from KD17. Regenerate committed header. This lands **first** so every subsequent sub-PR in 1b–1j can assert via CI grep: "if `crates/ffi/src/error.rs` OR `crates/ffi/src/generated/` OR `crates/core/src/permission.rs` is touched and the change is ABI-breaking, `AD_ABI_VERSION_MAJOR` must have bumped since the last main-branch commit." +- Sub-PR 1b: `ErrorCode` gets `#[non_exhaustive]` alone. No variant additions. Parity const assertions unchanged. Breaks nothing. Per KD17, `#[non_exhaustive]` addition does NOT bump `AD_ABI_VERSION_MAJOR`. +- Sub-PR 1c: Add 4 new `ErrorCode` variants atomically with matching `AdResult::Err*` discriminants and the `error_code_to_result` arm. Parity assertion passes — this is the gate. Per KD17, additive variants under `#[non_exhaustive]` do NOT bump `AD_ABI_VERSION_MAJOR`. +- Sub-PR 1d: Add 8 new `Action` variants. Platform adapters' existing `execute_action` arms fall through to `not_supported()` via the `#[non_exhaustive]` default — no adapter changes yet. +- Sub-PR 1e: Introduce `StableSelectors` on `AccessibilityNode` with `#[serde(flatten)]`. Construct `StableSelectors::default()` everywhere an `AccessibilityNode` is built today (macOS `tree/builder.rs`, test fixtures). Wire shape unchanged. +- Sub-PR 1f: `RefEntry.identifier: Option<String>` with serde `skip_serializing_if`. Populated with `None` for now — U5 adds the reader logic. +- Sub-PR 1g: `PermissionReport` extracted to `crates/core/src/permission.rs` as tri-state struct. The `permissions` command JSON output changes shape — **breaking**. This sub-PR **atomically bumps `AD_ABI_VERSION_MAJOR` from 1 → 2** (document-review refinement — the version bumps only when the layout actually changes, avoiding mid-series lies). macOS adapter's existing single-state response maps to `accessibility: Granted | Denied`; `screen_recording` and `automation` start `Unknown` until U11. +- Sub-PR 1h: Add trait methods with `not_supported()` defaults. Signatures: + - `fn blocked_combos(&self) -> &'static [KeyCombo]` + - `fn watch_element(&self, entry: &RefEntry, spec: &WatchSpec) -> Result<Vec<ElementEvent>, AdapterError>` — takes `&RefEntry`, **not** `&NativeHandle`, so the worker thread can re-resolve on its own thread and the `NativeHandle` `!Send`/`!Sync` invariant is preserved (deepening-pass refinement — KD9). + - `fn get_text_selection(&self, handle: &NativeHandle, config: &TextRangeConfig) -> Result<TextSelection, AdapterError>` + - `fn set_text_selection(&self, handle: &NativeHandle, config: &TextRangeConfig, range: TextRange) -> Result<(), AdapterError>` + - `fn get_text_at(&self, handle: &NativeHandle, config: &TextRangeConfig, range: TextRange) -> Result<String, AdapterError>` + - `fn insert_text_at_caret(&self, handle: &NativeHandle, config: &TextRangeConfig, text: &str) -> Result<(), AdapterError>` + - `fn get_screenshot_with_backend(&self, target: ScreenshotTarget, config: &ScreenshotBackendConfig) -> Result<ImageBuffer, AdapterError>` — takes `&ScreenshotBackendConfig`, **not** just `ScreenshotBackend`, so per-backend options (dimensions, pixel format, encoding) travel through one argument (deepening-pass refinement). +- Sub-PR 1i: Create `event.rs`, `text_range.rs`, `screenshot_backend.rs` supporting types + `ActionDispatchConfig`, `WatchElementConfig`, `TextRangeConfig`, `ScreenshotBackendConfig` (the 4 shared parameter types per §Context & Research). Extend `TreeOptions` with `force_electron_a11y: bool`. Serde roundtrip tests. No `EventWorker` or `NotificationSession` traits — both dropped as premature abstraction per scope review. +- Sub-PR 1j: Reserve CLI arms for U6/U7/U8 commands — `src/cli.rs` gains variants marked `#[clap(hide = true)]` (document-review refinement — so unimplemented commands do NOT appear in `agent-desktop --help` output during the Phase 2 intermediate window). Their `dispatch.rs` arms route to `AppError::invalid_input("command not yet implemented")` (NOT `unimplemented!()`) until the later units fill them. Each unit that implements a reserved command also flips `hide = false`. This reserves name-ordering so each subsequent unit's PR is smaller, without leaking broken commands into user-facing help. + +**Execution note:** Land sub-PRs serially (1a → 1j). Each must keep `cargo test --workspace` green. The `const _: () = assert!(…)` parity gate in `crates/ffi/src/error.rs` **is the atomic safeguard** — breaking either side of the pair fails the build at sub-PR 1b. + +**Technical design:** + +```rust +// node.rs — directional guidance, not final +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StableSelectors { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identifier: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subrole: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role_description: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub placeholder: Option<String>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dom_id: Option<String>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dom_classes: Vec<String>, +} + +pub struct AccessibilityNode { + // existing 10 fields unchanged + #[serde(flatten, default)] + pub selectors: StableSelectors, +} +``` + +```rust +// permission.rs — tri-state +pub enum TriState { + Granted, + Denied { suggestion: String }, + Unknown, +} +pub struct PermissionReport { + pub accessibility: TriState, + pub screen_recording: TriState, + pub automation: TriState, +} +``` + +**Patterns to follow:** +- Parity `const _: () = assert!(…)` pair in `crates/ffi/src/error.rs:5-60` — extend both arrays symmetrically. +- Serde `skip_serializing_if` idiom used throughout `crates/core/src/node.rs`. +- `#[non_exhaustive]` on `Action` (existing) — mirror for `ErrorCode`. + +**Test scenarios:** +- **Happy path:** `AccessibilityNode` with populated `StableSelectors` serializes with `identifier`, `subrole`, `dom_classes` as flat top-level fields in JSON. +- **Edge case:** `AccessibilityNode` with `StableSelectors::default()` emits no selector fields in JSON (roundtrip preserves `None`/empty). +- **Happy path:** `PermissionReport { accessibility: Granted, screen_recording: Denied { ... }, automation: Unknown }` serializes to the expected tri-state object shape. +- **Happy path:** `ErrorCode::PermissionRevoked.as_str() == "PERMISSION_REVOKED"`; ditto the other three new variants. +- **Integration:** parity `const _: () = assert!(…)` in `crates/ffi/src/error.rs` compiles (gate). +- **Integration:** `ad_abi_version()` extern "C" returns `2u32`; cbindgen header exports `AD_ABI_VERSION_MAJOR 2` as a preprocessor define. +- **Integration:** every existing macOS unit test still passes after `PermissionReport` tri-state migration (accessibility field carries the legacy boolean behavior; others `Unknown`). +- **Error path:** `Action::DeliverFiles(vec![PathBuf::from("relative/path")])` serde roundtrips preserving `PathBuf`. + +**Verification:** +- `cargo test --workspace` green. +- `cargo clippy --all-targets -- -D warnings` clean. +- `cargo build -p agent-desktop-ffi` green; `scripts/update-ffi-header.sh` shows no drift beyond the intended additions. +- `cargo tree -p agent-desktop-core` still free of platform crates. + +--- + +### - [ ] Unit 2: Registry migration — `build.rs` filesystem enumeration + codegen for FFI wrappers + +**Goal:** Migrate the existing `ad_*` FFI wrappers from hand-written to codegen. Each command's `commands/<name>.rs` file is the source of truth; `build.rs` enumerates the filesystem at compile time and emits (a) a `CommandDescriptor` static array in `crates/core/src/generated/registry.rs` and (b) one `extern "C" fn ad_<name>` wrapper per command in `crates/ffi/src/generated/wrappers.rs`. This is P2-O16. + +**Research-driven architecture shift:** the origin brainstorm's `inventory 0.3` + xtask proposal is REPLACED with pure `build.rs` filesystem enumeration (research Topic B — `inventory`/`linkme` link-GC is unreliable across ld64, ld-prime, GNU ld, lld, MSVC for cdylib consumers; `build.rs` is deterministic and has zero linker dependencies). The "one command per file" CLAUDE.md invariant is now load-bearing — it's the codegen contract. + +**Scope (single-concern):** `ad_set_log_callback` is Unit 2.5; Phase 1.5 FFI backfill is Unit 2.6. + +**Requirements:** R16 + +**Dependencies:** Unit 1 (needs `ad_abi_version`, new trait method stubs, tri-state `PermissionReport`, and the 4 shared config structs). + +**Files:** +- Create: `build-helpers/` (new workspace member — tiny crate exposing `fn enumerate_commands(dir: &Path) -> Vec<CommandMeta>`. No runtime deps; pure file I/O + regex.) +- Modify: workspace `Cargo.toml` (add `build-helpers` to `[workspace.members]`; **NO** `inventory`, **NO** `linkme`, **NO** `schemars` — deferred to Phase 4 MCP) +- Create: `crates/core/build.rs` (uses `build-helpers::enumerate_commands`; emits `$OUT_DIR/registry.rs` with `pub static DESCRIPTORS: &[CommandDescriptor] = &[…];`) +- Create: `crates/core/src/registry.rs` (`pub struct CommandDescriptor { name, dispatch_fn, args_parse_fn }` + `include!(concat!(env!("OUT_DIR"), "/registry.rs"))`) +- Modify: `crates/core/src/commands/*.rs` (each file gets a top-of-file `///! command_meta { name = "...", summary = "..." }` marker parsed by `build-helpers` + a `pub fn descriptor() -> CommandDescriptor { … }` function. ONE registration point per command file — no macros, no inventory, no ctor.) +- Modify: `crates/core/src/lib.rs` (re-export `registry::DESCRIPTORS`) +- Modify: `src/dispatch.rs` (replace the giant `match` with `DESCRIPTORS.iter().find(|d| d.name == cmd_name).map(|d| (d.dispatch_fn)(args, adapter))`; retain clap-to-descriptor bridge in `src/cli.rs`) +- Modify: `crates/ffi/build.rs` (extend: uses `build-helpers::enumerate_commands` — SAME enumeration as core's build.rs; emits `$OUT_DIR/wrappers.rs` with one `extern "C" fn ad_<name>(...)` per command; stamp path in `target/ffi-wrappers-path.txt` — same pattern as cbindgen header stamp) +- Create: `crates/ffi/src/generated.rs` (`include!(concat!(env!("OUT_DIR"), "/wrappers.rs"));`) +- Modify: `crates/ffi/src/lib.rs` (wire `mod generated;` after hand-written wrappers are removed) +- Delete: hand-written `ad_<name>` wrappers across `crates/ffi/src/{actions,apps,input,notifications,observation,screenshot,surfaces,tree,windows}/`. Marshaling primitives in `crates/ffi/src/convert/`, `adapter.rs`, `error.rs`, `ffi_try.rs`, `main_thread.rs`, `pointer_guard.rs`, `types/`, `enum_validation.rs` stay. +- Modify: `crates/ffi/include/agent_desktop.h` (regenerated with generated wrappers in identical order) +- Modify: `.github/workflows/ci.yml` (add a "FFI header drift check" step — already present — plus the registry enumeration runs identically across all build profiles because it's pure file-walk, no linker involved) +- Modify: `scripts/update-ffi-header.sh` → rename to `scripts/update-ffi.sh` (shim for old name kept for backward compat); regenerates the cbindgen header +- Test: `crates/core/tests/registry_coverage.rs` (asserts `DESCRIPTORS.len()` equals the count of `.rs` files in `crates/core/src/commands/` excluding `mod.rs`/`helpers.rs`; fails loudly if a command file exists without a descriptor or vice versa) +- Test: `crates/core/tests/cli_registry_parity.rs` (cross-checks that every CLI subcommand name enumerated from clap has a matching `CommandDescriptor` AND vice versa; closes rename-detection gap) +- Test: `tests/integration/per_command_fixture_diff.rs` (for each migrated command, pre-migration JSON fixture vs post-migration byte-diff — empty diff or whitelisted with justification) + +**Approach:** +- Migrate one command category at a time: **observation → interaction → system → clipboard → notifications → batch**. Each sub-PR keeps `cargo test --workspace` + `cargo test -p agent-desktop-ffi --tests` green. +- `CommandDescriptor` carries: `name: &'static str`, `dispatch_fn`, `args_parse_fn` (from `&clap::ArgMatches`). No schemars — deferred to Phase 4 MCP where schemas are actually consumed. +- **Codegen mechanism (research-refined):** pure `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs`, NOT `inventory` / `linkme` / xtask. Research Topic B found neither inventory nor linkme survives link-GC across ld64, ld-prime, GNU ld, lld, and MSVC link.exe for cdylib consumers; `deterministic registry metadata` ctor sites are stripped when an `rlib` is linked into a binary that never references a symbol from that rlib. `build.rs` reads source files at compile time, parses a `///! command_meta { … }` block, and emits a deterministic static array. Zero linker magic, zero ctor sites, cdylib-safe by construction. +- The opening **spike sub-PR** validates the codegen on ONE command (`click`): builds `build-helpers::enumerate_commands`, emits generated `ad_click`, passes fixture byte-diff against hand-written `ad_click`, ships alone. Only after the spike merges does the category migration open. +- `build-helpers` workspace crate owns the enumeration logic (used by both `crates/core/build.rs` and `crates/ffi/build.rs`). Single source of truth for "what is a command". +- **Per-command fixture diff gate:** before migrating a category, the sub-PR captures pre-migration JSON fixtures for every command in that category. Post-migration, `per_command_fixture_diff.rs` asserts byte-equivalent output (or whitelisted + justified diff). "Green tests" alone is not sufficient gating. +- **No link-GC mitigation needed:** generated `extern "C" fn ad_<name>` symbols are directly exported from the cdylib (`nm -g` shows them). Cargo's default visibility for `pub extern "C"` items in cdylib targets keeps them live. Zero `#[used]` annotations, zero `--whole-archive` flags, zero CI matrix for link profiles. + +**Execution note:** The spike PR opens first, on `click` only. Capture the codegen decision (build.rs filesystem vs alternatives) in the spike's commit message so reviewers see the chosen mechanism and its rationale (research Topic B). + +**Patterns to follow:** +- `crates/ffi/build.rs` stamp-path idiom (learning 3). +- `crates/ffi/src/error.rs` `const` parity assertion — mirror for registry count: `const _: () = assert!(ffi_wrapper_count() == command_count(), "…");`. +- `crates/ffi/src/convert/` marshaling helpers stay per-type (strings, rects, windows, notifications). Generated wrappers call them. + +**Test scenarios:** +- **Happy path:** `DESCRIPTORS.len() == 53` after migration; asserted by `crates/core/tests/registry_coverage.rs`. +- **Happy path:** every existing CLI command dispatches via registry; `cargo test --lib --workspace` preserves all existing assertions. +- **Happy path:** `ad_<name>` extern "C" symbols exist for every `CommandDescriptor` — checked by `nm -g target/debug/libagent_desktop_ffi.*` against the file enumeration count. +- **Integration:** `ad_click @e5` passes identical JSON to the generated wrapper as the hand-written wrapper did (pre/post-migration fixture byte-diff). +- **Integration:** `ad_abi_version()` still returns `AD_ABI_VERSION_MAJOR` unchanged through the migration. +- **Error path:** malformed JSON input to any generated `ad_<name>` wrapper returns `AdResult::ErrInvalidArgs` with a `set_last_error` message — not a panic or crash. +- **Integration:** CI header-drift check green after regenerating on migration PRs. +- **Edge case:** adding a new file `crates/core/src/commands/hello_world.rs` with a descriptor auto-registers — no other file needs editing; CLI, FFI wrapper, and registry all pick it up on next build. +- **Edge case:** removing a command file removes the descriptor and the `ad_<name>` wrapper without a runtime check (compile-time enumeration). + +**Verification:** +- `cargo test --workspace` + `cargo test -p agent-desktop-ffi --tests` green. +- Generated artifacts under `$OUT_DIR/` regenerate deterministically (same input = same output, same byte ordering). +- `crates/ffi/include/agent_desktop.h` regenerated with identical symbol ordering as before (ordering test pins this). +- No workspace-level `inventory` / `linkme` / `xtask` dependencies introduced. + +--- + +### - [ ] Unit 2.5: `ad_set_log_callback` with redaction layer + +**Goal:** Ship the FFI log-forwarding callback with a mandatory redaction layer, as a small standalone unit. Split out from Unit 2 by review — three concerns in one unit was rejected. + +**Requirements:** R16 (partial — log callback portion) + +**Dependencies:** Unit 2 (registry migration lands first so the FFI crate is stable) + +**Files:** +- Create: `crates/ffi/src/log_callback.rs` (`ad_set_log_callback(cb: extern "C" fn(level: i32, msg: *const c_char))` installs a filtered `tracing_subscriber::Layer`) +- Create: `crates/ffi/src/log_redaction.rs` (**security-critical**: filters tracing events before emission to the callback) +- Modify: `crates/ffi/src/lib.rs` (export `ad_set_log_callback`) +- Modify: `skills/agent-desktop-ffi/references/threading.md` (callback lifetime invariants) +- Modify: `crates/ffi/README.md` (callback security contract — section "Log callback responsibilities") +- Test: `crates/ffi/tests/log_callback.rs` (invoking any `ad_*` emits a tracing event; callback receives a redacted message) +- Test: `crates/ffi/tests/log_redaction.rs` (tracing events containing `value=<secret>`, `password=<secret>`, `clipboard=<secret>`, `token=<secret>` arrive at the callback with values replaced by `<redacted>`) +- Test: `crates/ffi/tests/log_callback_reentry.rs` (setting the callback twice returns `ErrorCode::InvalidArgs` with the "callback already installed" suggestion) + +**Approach:** +- Global `OnceCell<extern "C" fn(i32, *const c_char)>` stores the callback; second registration fails closed with `InvalidArgs`. +- The redaction layer sits between `tracing_subscriber` and the callback. It filters: + 1. **Field-name allowlist/denylist:** any event field with a name matching `value`, `text`, `content`, `clipboard`, `password`, `secret`, `token`, `credential`, `auth` (case-insensitive) is replaced with `<redacted>` in the emitted string. + 2. **Level filter:** `TRACE`-level events are dropped by default; consumers opt in via `AGENT_DESKTOP_LOG_TRACE=1` env var. + 3. **Size cap:** any single event message longer than 4 KB is truncated with a `[truncated]` marker to prevent memory amplification attacks. +- Callback invocation is wrapped in `catch_unwind` — a consumer-side panic does not propagate back through the FFI boundary. +- Documented contract: "Callbacks must not persist received events to storage or network without explicit user consent. agent-desktop ships redaction as defense-in-depth; the primary trust boundary is the FFI consumer." + +**Patterns to follow:** +- Existing `crates/ffi/src/ffi_try.rs` `trap_panic` wrapper shape. +- `crates/ffi/src/main_thread.rs` for global-state access discipline. + +**Test scenarios:** +- **Happy path:** `ad_set_log_callback(cb)` + any `ad_*` call → `cb` receives a C-string message. +- **Security:** `type-text @e5 "hunter2"` emits a tracing event; the callback receives `type-text ref=@e5 text=<redacted>` (not the literal password). +- **Security:** clipboard-set emits `clipboard=<redacted>` regardless of content. +- **Edge case:** message > 4 KB → truncated with marker. +- **Edge case:** `TRACE` events dropped unless env var set. +- **Edge case:** callback that panics → `catch_unwind` isolates; library continues. +- **Error path:** second `ad_set_log_callback` call returns `InvalidArgs`. + +**Verification:** +- All tests green. +- `crates/ffi/README.md` section on callback responsibilities published and linked from `skills/agent-desktop-ffi/SKILL.md`. + +--- + +### - [ ] Unit 2.6: Phase 1.5 FFI backfill — `ad_snapshot` / `ad_execute_by_ref` / `ad_wait` / `ad_version` / `ad_status` + +**Goal:** Backfill the five FFI wrappers identified as missing in `docs/plans/2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md`. Originally bundled into Unit 2; split out by review so Unit 2 stays single-concern. + +**Requirements:** R16 (Phase 1.5 backfill portion) + +**Dependencies:** Unit 2 (registry migration) — the 5 backfilled wrappers are new `CommandDescriptor` entries, not hand-written wrappers. + +**Files:** +- Modify: `crates/core/src/commands/snapshot.rs` (confirm the descriptor — added by U2 via build.rs filesystem enumeration — carries refmap-pipeline args correctly; includes `skeleton` and `root_ref` CLI bindings) +- Modify: `crates/core/src/commands/status.rs`, `version.rs`, `wait.rs` (same — verify descriptors are correct) +- Create: `crates/core/src/commands/execute_by_ref.rs` (new thin wrapper that takes `(ref_id, action_json)` and dispatches via `PlatformAdapter::execute_action`) +- Modify: `src/cli.rs` + `src/dispatch.rs` + `src/cli_args.rs` (add `ExecuteByRef` command arm) +- Test: `crates/ffi/tests/backfill_gaps.rs` (asserts `ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_version`, `ad_status` are all exported and callable; closes the v0.1.13 known-gaps list) +- Test: `crates/core/src/commands/execute_by_ref.rs` (unit — resolves ref then dispatches action; propagates STALE_REF and INVALID_ARGS correctly) + +**Approach:** +- `execute_by_ref` accepts a serialized `Action` JSON + ref id. Resolves the ref via `RefMap::load`, then calls `adapter.execute_action`. This is the CLI-shell-independent agent primitive that FFI consumers use for agent loops. +- The other four (`snapshot`, `wait`, `version`, `status`) already exist as CLI commands; this unit only confirms that their `CommandDescriptor` entries carry the full arg surface (refmap pipeline for snapshot, multi-mode flags for wait, structured output for status). + +**Patterns to follow:** +- `crates/core/src/commands/click.rs` minimal shape. +- `crates/core/src/commands/snapshot.rs` for the refmap pipeline. + +**Test scenarios:** +- **Happy path:** `ad_execute_by_ref` with valid ref + `{"type":"Click"}` action → calls `PlatformAdapter::execute_action(Click)` → returns ActionResult JSON. +- **Error path:** stale ref → `AdResult::ErrStaleRef` propagates cleanly. +- **Error path:** malformed action JSON → `AdResult::ErrInvalidArgs` with diagnostic. +- **Integration:** `ad_snapshot` exposes all flags from CLI (skeleton, root, surface, max-depth, include-bounds, interactive-only, compact). + +**Verification:** +- Backfill test green; `crates/ffi/include/agent_desktop.h` exports all 5 `ad_*` functions. + +--- + +### - [ ] Unit 3: Windows adapter foundation + +**Goal:** `WindowsAdapter` implements every existing 53-command's trait method via UIA (`uiautomation 0.24`) + `windows 0.62.2`. Base capability parity with macOS. Skeleton traversal (`--skeleton`, `--root @ref`) works identically on Windows. P2-O1, P2-O2, P2-O3, P2-O5, P2-O6, P2-O7 (partial). + +**Requirements:** R1, R2, R3, R5, R6, R7 + +**Dependencies:** Unit 1, Unit 2 + +### Windows Engineering Invariants (research-driven, MUST land in sub-PR 3.0 before any other U3 sub-PR) + +All invariants validated against Microsoft docs (UIA Threading 2025-07-14, UIA Security 2026-02-18) and production UIA tooling. Source: `/tmp/agent-desktop-research/windows-headless.md`. + +1. **DPI awareness at process startup:** `main.rs` Windows branch calls `SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2)` before any UIA call. Without this, UIA returns coordinates in virtualized pixels and click/hover synthesis drifts on mixed-DPI setups. +2. **COM apartment on the main thread and UIA workers:** `CoInitializeEx(NULL, COINIT_MULTITHREADED)` is called once at startup and once per dedicated UIA worker thread. Research-backed: UIA explicitly prefers MTA for cross-thread event delivery; `IUIAutomationElement` is apartment-affine — caching elements across thread boundaries invalidates them. +3. **Never cache `IUIAutomationElement` across apartments.** NativeHandle stores an element obtained in the current apartment and is not safe to pass to a thread of different apartment affinity. Workers that need elements re-resolve from `RefEntry` on their own thread. Event handlers are created, registered, removed, and drained on the same dedicated MTA thread. +4. **UIA-first, SendInput-fallback.** Every action attempts the UIA pattern first (`InvokePattern`, `ValuePattern`, `TogglePattern`, `ExpandCollapsePattern`, `SelectionItemPattern`) — all focus-independent and headless-safe. Only when no pattern applies does the adapter fall back to `SendInput`, gated by an `AttachThreadInput` + `SetFocus` worker-thread dance. SendInput is documented as focus-dependent and silently blocked by UIPI against elevated targets. +5. **`PostMessage WM_KEYDOWN` is DEAD for modern apps.** Chromium (Electron, Edge, Chrome, VS Code, Slack, Cursor), UWP apps, and games all ignore posted keyboard messages. It's NOT a viable SendInput alternative; do not implement. +6. **UIPI elevation detection.** `GetTokenInformation(TokenIntegrityLevel)` compares the agent-desktop process's integrity level against the target window's (`GetWindowThreadProcessId` → open process → query integrity). If mismatch, return `PermDenied` with `platform_detail: "Target window runs at higher integrity level; SendInput and some UIA patterns are blocked by UIPI. Run agent-desktop with matching elevation."` Do NOT ship the `uiAccess=true` manifest flag by default — it requires code signing and a specific install location; document as an optional signed-release path. +7. **`RemoveAutomationEventHandler` teardown race.** Per Microsoft's 2025 UIA threading doc, the handler object must outlive the final dispatched callback. Implementation: store handlers in an `Arc<Handler>` clone map keyed by `(element_id, event_kind)`; `RemoveAutomationEventHandler` is called on the UIA client, but the `Arc<Handler>` is dropped only AFTER a post-remove barrier call (a no-op UIA method call that drains any in-flight callback dispatch). +8. **HRESULT encoding in `platform_detail`.** Standardized format: `COM HRESULT 0x80070005 (E_ACCESSDENIED: Access is denied)` — hex + FACILITY + symbolic name + `FormatMessageW` description. Parsed by a shared helper `crate::error::format_hresult(hr: HRESULT) -> String`. +9. **DWM composition caveat for legacy screenshot.** `PrintWindow(hwnd, hdc, 0)` returns black frames for composited windows on Windows 10+. `PrintWindow(hwnd, hdc, PW_RENDERFULLCONTENT)` mitigates — use the flag unconditionally in legacy backend. `windows-capture` (modern) handles composition correctly. +10. **`ElementFromHandle(hwnd)` is headless-safe within the current desktop session** — works on same-user, same-session visible or minimized windows at an accessible integrity level regardless of focus. This is the foundation for observation headlessness. No focus side-effect. +11. **`Windows.Graphics.Capture` availability:** requires DWM and Windows 10 1903+ in an active interactive session. Fails in Session 0, Server Core, secure desktop, locked desktop, and some remote/virtualized sessions — document and return `PlatformNotSupported` in those environments. agent-desktop's own process does NOT need an HWND — `windows-capture` uses `GraphicsCaptureItem.CreateFromWindowHandle(target_hwnd)`. +12. **Session isolation:** agent-desktop cannot drive windows in other user sessions, Session 0, secure desktop, or locked desktops. Document; return `WindowNotFound`, `PermDenied`, or `PlatformNotSupported` with explanatory `platform_detail`. +13. **Foreground APIs are explicit only:** `SetForegroundWindow`, `SetFocus`, `AttachThreadInput`, and `SetWindowPos(HWND_TOP)` are allowed only in commands/policies that explicitly request focus/window/physical side effects. They are never fallback paths for semantic ref actions. +14. **Windows shell is part of the product surface:** Start menu/search, taskbar, system tray/overflow, Action Center, Quick Settings, UAC/integrity, virtual desktop detection, and mixed-DPI monitor geometry are first-class Phase 2 cases. Each must be handled by a command/surface, an integration test, or a documented structured unsupported response. + +### Skeleton traversal on Windows + +The skeleton primitive (`snapshot --skeleton`, `snapshot --root @ref`) is platform-agnostic at the core level (`crates/core/src/snapshot_ref.rs`). Windows adapter contribution is ~50 LOC: + +- **`get_subtree(handle, opts)`** delegates to `build_subtree(element, opts)` where `element` is the `IUIAutomationElement` held by `NativeHandle`. Mirrors macOS's `adapter.rs:221-241` shape. +- **Walker:** `ControlViewWalker` (NOT `RawViewWalker` or `ContentViewWalker`). Research Topic 4 confirmed: `IsControlElement` auto-filters layout noise and complements Unit 4's Electron depth-skip. +- **`children_count`** for skeleton boundary annotation: single `FindAll(TreeScope_Children, TrueCondition)` round-trip. Cheap enough — one COM call, no per-child property fetch. +- **Depth clamping** at `opts.max_depth = 3` in skeleton mode is enforced in core's `build_subtree`; Windows adapter just respects the passed depth. +- **Scoped invalidation** is entirely core-level (`RefMap::remove_by_root_ref`). Windows inherits with zero adapter code. +- **Fresh `UICacheRequest` per drill-down call** — cached elements do not survive CLI process boundaries. Each `snapshot --root @ref` allocates a new cache request; no attempt at cross-invocation caching. +- **Token savings target:** 50-100× on VS Code / Slack Electron once Unit 4's `--force-electron-a11y` + empty-`UIA_Group` / `UIA_Custom` depth-skip are in place. + +**Files:** +- Modify: `crates/windows/Cargo.toml` (target-gated `[target.'cfg(target_os = "windows")'.dependencies]` gains `uiautomation = "0.24"`, `windows = { version = "0.62.2", features = ["Win32_UI_Input", "Win32_UI_Input_KeyboardAndMouse", "Win32_System_Com", "Win32_System_DataExchange", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi"] }`; `windows-capture` is deferred to U9; OLE/clipboard features for U12) +- Modify: `crates/windows/Cargo.toml` (mirror the deps; inherit `agent-desktop-core` from workspace) +- Rewrite: `crates/windows/src/lib.rs` (mod declarations + re-exports of `WindowsAdapter`) +- Create: `crates/windows/src/adapter.rs` (`WindowsAdapter` struct + `impl PlatformAdapter`) +- Create: `crates/windows/src/tree/{mod,element,builder,roles,resolve,surfaces}.rs` (UITreeWalker + UICacheRequest) +- Create: `crates/windows/src/actions/{mod,dispatch,activate,extras}.rs` (InvokePattern → TogglePattern → coord fallback) +- Create: `crates/windows/src/input/{mod,keyboard,mouse,clipboard}.rs` (SendInput, Win32 clipboard) +- Create: `crates/windows/src/system/{mod,app_ops,window_ops,key_dispatch,permissions,screenshot,wait}.rs` (`screenshot` here is the Phase 1 `PrintWindow` equivalent — legacy backend; modern is U9) +- Create: `crates/windows/src/helpers/pid.rs` (`narrow_pid: u32 → Result<i32, AdapterError>` returning `ErrorCode::ResourceExhausted` per KD3; document-review refinement — this aligns with KD3's semantic choice) +- Modify: `src/main.rs` (`build_adapter()` `#[cfg(target_os = "windows")]` branch flips from stub to real adapter) +- Test: `crates/windows/src/tree/roles.rs` (`#[cfg(test)]` — unit: every `UIA_*ControlTypeId` maps to a known role string) +- Test: `crates/windows/tests/it_tree.rs` (integration, Windows CI only: snapshot of `explorer.exe`, `notepad.exe`, `calc.exe` — non-empty tree with refs) +- Test: `crates/windows/tests/it_actions.rs` (click button in a test WinUI app; verify via UIA property read after the action) +- Test: `crates/windows/tests/it_input.rs` (`SendInput` types "hello" into Notepad; verify `Value` property == "hello") +- Test: `crates/windows/tests/it_clipboard.rs` (get / set / clear roundtrip with ASCII + CJK via `CF_UNICODETEXT`) +- Test: `crates/windows/tests/it_window_ops.rs` (resize, move, minimize, maximize, restore against a newly launched Notepad) +- Test: `crates/windows/tests/it_app_lifecycle.rs` (launch Notepad, type, close) + +**Approach:** +- Tree walk: `IUIAutomation.ElementFromHandle(hwnd) → UITreeWalker.RawViewWalker.GetFirstChild/GetNextSibling` with `UICacheRequest` batching `Name, ControlType, AutomationId, LocalizedControlType, HelpText, BoundingRectangle, IsEnabled, HasKeyboardFocus, ExpandCollapseState, ToggleState, ValueValue` in one round-trip (origin §Windows API mapping). +- Role map: `UIA_*ControlTypeId` → unified role strings (matches macOS `tree/roles.rs` output). Table lives in `crates/windows/src/tree/roles.rs` and is exhaustive against the 50 UIA control types. +- Action dispatch: pattern-first — `InvokePattern.Invoke` → `TogglePattern.Toggle` → `ExpandCollapsePattern.Expand` → coordinate-click fallback via `SendInput`. Mirrors macOS smart-chain pattern; share dispatch config via `ActionDispatchConfig` in `crates/core` per learning 1. +- `blocked_combos()` returns `[alt+f4, ctrl+alt+del, win+l, ctrl+shift+esc]`. Key-down / key-up safety check matches the whole combo (KD11). +- PID narrowing: `fn narrow_pid(dword: u32) -> Result<i32, AdapterError>` errors with `ErrorCode::ResourceExhausted` (per KD3; document-review reconciliation — previously said `Internal`) + `platform_detail: "PID exceeds i32::MAX; Windows kernel should never produce this value"` for values above `i32::MAX`. Unlikely in practice; belt-and-braces. +- `NativeHandle` inner is `AutomationElement` (`uiautomation::UIElement`) held by pointer; `release_handle` calls `ComPtr::Release` equivalent. +- Clipboard: `OpenClipboard(null)` + `CF_UNICODETEXT` get/set with a UTF-16 ↔ String conversion. +- Screenshot: `PrintWindow(hwnd, hdc, PW_RENDERFULLCONTENT)` — this is the **legacy** path in this unit; U9 adds the modern `windows-capture` backend as the default. +- App launch: `CreateProcessW` + `WaitForInputIdle` + resolve via `ElementFromHandle` on the main window. + +**Execution note:** Keep each major surface (tree, actions, input, system) as its own sub-PR when possible to keep reviews manageable. The tree builder and UIA role map can land in one sub-PR (`tree/` complete); actions + input in a second; clipboard + app lifecycle + window ops in a third; permissions + screenshot (legacy) in a fourth. Windows CI from U13 must be in place before this unit's integration tests can run — land U13 **in parallel** against a draft branch early. + +**Patterns to follow:** +- Platform subfolder layout in `crates/macos/src/` (tree/, actions/, input/, system/). +- `AXElement` safety pattern (`pub(crate)` inner pointer, Clone/Drop via Retain/Release) — mirror for `UIElement` with `ComPtr`. +- Smart activation chain in `crates/macos/src/actions/activate.rs` and `chain_steps.rs` — define a cross-platform `ActionDispatchConfig` (learning 1) so Windows doesn't copy the macOS structure wholesale. + +**Test scenarios:** +- **Happy path:** `snapshot --app Explorer` returns a tree with ≥20 refs including the sidebar, breadcrumb, and file list (R1). +- **Happy path:** `snapshot --app Notepad` returns a tree with a `textfield` ref for the document surface. +- **Happy path:** `click @e<button_ref>` on a known Settings button fires `InvokePattern.Invoke` and the UIA state changes. +- **Happy path:** `type @e<textfield_ref> "hello"` sets the Value via `SendInput`; re-snapshot shows `value="hello"` (R3). +- **Edge case:** `press cmd+q` on Windows maps `Cmd → Win` and is **blocked** per `blocked_combos()`; returns `ActionNotSupported` with suggestion. +- **Edge case:** UIA element present but offscreen — bounds reported, `BoundingRectangle` reflects actual coordinates after scroll. +- **Edge case:** stale `@e5` (window closed) returns `ErrorCode::StaleRef` with snapshot refresh suggestion. +- **Error path:** calling `launch --app notexist.exe` returns `ErrorCode::AppNotFound` with the lookup failure surfaced in `platform_detail`. +- **Error path:** `OpenClipboard` contention (another process holds the clipboard) retries then returns `ActionFailed` with HRESULT in `platform_detail`. +- **Integration:** `clipboard-set "héllo ✓"` + `clipboard-get` roundtrips Unicode exactly (R5). +- **Integration:** cross-platform JSON identity — snapshot of Calculator on macOS and Windows, serialized, structurally identical on role set and ref ordering (R2). Fixture lives in `tests/fixtures/calculator.macos.json` and `tests/fixtures/calculator.windows.json`; a test asserts the union of refs and the identifier map. +- **Integration:** `window_op Maximize/Minimize/Restore` roundtrips via `ShowWindow`; state observable via `GetWindowPlacement`. +- **Integration:** PID narrowing test — mock `u32::MAX` → `narrow_pid` returns `ErrorCode::ResourceExhausted` (document-review refinement); any realistic PID (<2^31) roundtrips. + +**Verification:** +- Windows CI (`test-windows` job from U13) runs all integration tests green. +- `cargo tree -p agent-desktop-core` still platform-free on Windows. +- Binary size under 15 MB for `agent-desktop.exe` (check in `.github/workflows/release.yml`). + +--- + +### - [ ] Unit 3a: Windows notifications + +**Goal:** Windows Toast / Action Center parity with macOS 4-command surface (`list-notifications`, `dismiss-notification`, `dismiss-all-notifications`, `notification-action`) when the OS exposes a supported notification path. Primary path is `UserNotificationListener` with explicit user permission and app identity/capability; Action Center UIA is a best-effort fallback. Subset of P2-O14 and P2-O18. + +**Requirements:** R14 (Windows subset) + +**Dependencies:** Unit 3 + +**Files:** +- Create: `crates/core/src/notifications/session.rs` (**deepening-pass addition** — `NotificationSession` trait; existing macOS `crates/macos/src/notifications/nc_session.rs` refactors to implement it; Windows `listener.rs` and `action_center.rs` implement it. Prevents "batch dismiss" future fixes from landing on only one platform.) +- Modify: `crates/macos/src/notifications/nc_session.rs` (refactor to `impl NotificationSession for NcSession`) +- Create: `crates/windows/src/notifications/{mod,list,dismiss,interact,listener.rs,action_center.rs}` (`listener.rs` wraps `UserNotificationListener`; `action_center.rs` implements UIA fallback) +- Modify: `crates/windows/src/adapter.rs` (implement `list_notifications`, `dismiss_notification`, `dismiss_all_notifications`, `notification_action`) +- Test: `crates/windows/tests/it_notifications.rs` (CI integration — launch a known toast, list, dismiss, verify) +- Test: `crates/core/src/notifications/session.rs` (unit — trait contract tests that both platforms satisfy) + +**Approach:** +- Primary listener access: use `UserNotificationListener` when package identity/capability and explicit permission are present. Denied permission maps to `PERM_DENIED` with a targeted suggestion. +- Action Center fallback: open via `open-system-surface --surface action-center`; traverse exposed shell UIA only when the toast list and action buttons have stable names/descriptions. +- Each toast: listener metadata or UIA element with `Name` = title, `FullDescription` = body, app info, and child action buttons with `InvokePattern`. +- `dismiss_notification`: prefer listener dismissal where supported; otherwise locate the toast's close button by stable identity (e.g. `AutomationId == "DismissButton"` when exposed) and invoke it. +- `dismiss_all_notifications`: listener bulk clear if supported; otherwise Action Center's "Clear all" button. Mirror macOS `dismiss_all` semantics. +- `notification_action(index, identity, action_name)`: enforce identity-fingerprint match (learning 4) — if identity fields are `Some`, compare against the toast's title/app before invoking. Returns `NotificationNotFound` on mismatch. +- Focus Assist state: use supported shell APIs first; registry/WNF probes are best-effort diagnostics and must never be the sole correctness signal. + +**Patterns to follow:** +- `crates/macos/src/notifications/` (list.rs, actions.rs, nc_session.rs) — same PlatformAdapter arm shape, same JSON contract. +- Learning 4: identity fingerprint is optional, tri-state decoded at FFI. + +**Test scenarios:** +- **Happy path:** `list-notifications` after posting a test toast returns ≥1 entry with `title` and `body` populated through the listener path when permission/app identity is available. +- **Happy path:** `dismiss-notification --index 1` removes the toast; re-list shows count decremented. +- **Happy path:** `dismiss-all-notifications` clears Action Center; re-list returns empty. +- **Edge case:** identity mismatch — `notification-action --index 1 --title "Wrong"` returns `NotificationNotFound` with suggestion to re-list. +- **Edge case:** Action Center closed and listener unavailable → list returns empty or `PLATFORM_NOT_SUPPORTED` depending on whether the shell exposes a supported fallback. +- **Error path:** no notifications present → `dismiss-notification --index 1` returns `NotificationNotFound`. +- **Integration:** cross-platform contract — same JSON keys, same 1-based indexing (compare with `crates/macos/tests/it_notifications.rs`). + +**Verification:** +- Windows listener tests green where package identity is available; Action Center fallback tests green on interactive desktop runners; hosted CI keeps unit/mocked contract tests green when shell UI is absent. + +--- + +### - [ ] Unit 3b: Windows shell surfaces + system tray + +**Goal:** New command `open-system-surface --surface <kind>` plus `list-tray-items`, `click-tray-item`, `open-tray-menu` on Windows. Surfaces cover Start menu/search, taskbar, system tray/overflow, Action Center, Quick Settings, and shell flyouts. macOS gains the portable parts in Unit 10. Subset of P2-O14 and P2-O18. + +**Requirements:** R14 (tray subset) + +**Dependencies:** Unit 3 + +**Files:** +- Create: `crates/windows/src/tray/{mod,list,interact}.rs` +- Create: `crates/windows/src/system/shell_surfaces.rs` +- Create: `crates/core/src/commands/open_system_surface.rs` +- Create: `crates/core/src/commands/list_tray_items.rs`, `click_tray_item.rs`, `open_tray_menu.rs` (the core dispatchers; delegate to adapter trait methods) +- Modify: `crates/core/src/adapter.rs` (trait methods `open_system_surface`, `list_tray_items`, `click_tray_item`, `open_tray_menu` with `not_supported()` defaults) +- Modify: `crates/windows/src/adapter.rs` (implement) +- Modify: `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` (wire new commands) +- Modify: `crates/core/src/commands/mod.rs` (module list) +- Test: `crates/windows/tests/it_shell_surfaces.rs` (Start/taskbar/Action Center/Quick Settings surface open + snapshot, interactive runner) +- Test: `crates/windows/tests/it_tray.rs` (CI integration or self-hosted interactive job) + +**Approach:** +- Access `Shell_TrayWnd` window class via `FindWindow`; walk its UIA tree; tray items are `UIA_ButtonControlTypeId` leaves with `AutomationId` = ICON GUID when set. +- Overflow: `NotifyIconOverflowWindow` class; expand by clicking the chevron button. +- `open-system-surface --surface start-menu`: use the documented shell keyboard command through explicit shell-surface policy, then snapshot Start via UIA. This is not used for `launch`, which remains direct `CreateProcess` / `ShellExecuteEx`. +- `open-system-surface --surface quick-settings` and `action-center`: use supported shell invocation where available; return `PLATFORM_NOT_SUPPORTED` with `platform_detail` when the Windows build/session has no accessible shell surface. +- `snapshot --surface taskbar`: walk `Shell_TrayWnd` task list and expose pinned/running app buttons as refs. Invoking a taskbar button is a focus/window action and must go through explicit policy. +- Virtual desktop detection uses public `IVirtualDesktopManager` for current-desktop filtering/diagnostics. Moving windows across virtual desktops is deferred until a stable public write path is validated. +- Mixed-DPI geometry uses process-wide per-monitor awareness plus physical-pixel normalization before any coordinate fallback. +- `click_tray_item --name "Network"`: resolve by `Name` or by `identifier`; fallback to coordinate click via `SendInput` for items without UIA patterns. +- `open_tray_menu --name "Volume"`: click + wait for the resulting popup (UIA focus-changed event from U7, or poll fallback before U7 lands). + +**Patterns to follow:** +- Commands in `crates/core/src/commands/` stay slim — delegate to adapter (click.rs, list_windows.rs shape). +- Learning 1 — define `SurfaceDetectionConfig` so tray, menu, and notification surface walkers share shape. + +**Test scenarios:** +- **Happy path:** `open-system-surface --surface start-menu` opens Start; `snapshot --surface start-menu` returns search/results controls with refs. +- **Happy path:** `snapshot --surface taskbar` returns pinned/running app buttons with stable names. +- **Happy path:** `open-system-surface --surface quick-settings` opens Quick Settings on Windows builds that expose it; unsupported builds return `PLATFORM_NOT_SUPPORTED`. +- **Happy path:** `list-tray-items` returns volume, network, clock entries on a baseline interactive Windows VM. +- **Happy path:** `click-tray-item --name "clock"` opens the clock flyout; re-list-surfaces shows the popup surface. +- **Edge case:** unknown tray item name → `ElementNotFound` with suggestion to run `list-tray-items`. +- **Edge case:** overflow item — list includes items in `NotifyIconOverflowWindow` transparently. +- **Error path:** tray or shell surface not accessible (Explorer shell missing, locked desktop, Server Core, unsupported build) → `WindowNotFound` or `PlatformNotSupported` with platform detail. + +**Verification:** +- Windows interactive integration green; hosted CI keeps unit/mocked shell-surface contract tests green when Explorer shell is unavailable; `list-surfaces` on Windows now includes `SystemTray`, `Taskbar`, `StartMenu`, `ActionCenter`, and `QuickSettings` when present. + +--- + +### - [ ] Unit 4: Windows Electron / WebView2 compatibility + +**Goal:** Depth-skip for non-semantic UIA wrappers, resolver depth 50, surface detection treats focused window as the target surface, `--force-electron-a11y` CLI flag. Matches macOS Electron compat. P2-O15. + +**Requirements:** R15 + +**Dependencies:** Unit 3 (needs the tree walker to patch) + +**Files:** +- Modify: `crates/windows/src/tree/builder.rs` (web-wrapper depth-skip: `UIA_GroupControlTypeId` / `UIA_CustomControlTypeId` with empty `Name` AND empty `Value` do not consume depth budget) +- Modify: `crates/windows/src/tree/resolve.rs` (`ABSOLUTE_MAX_DEPTH = 50` mirror of macOS) +- Modify: `crates/windows/src/tree/surfaces.rs` (surface detection checks focused window itself against target surface shape) +- Modify: `crates/core/src/adapter.rs` (`TreeOptions.force_electron_a11y: bool`) +- Modify: `src/cli_args.rs` (snapshot args gain `--force-electron-a11y`) +- Modify: `crates/windows/src/adapter.rs` (if flag set, apply `renderer-accessibility` UIA override — check `ClassName == "Chrome_WidgetWin_1"` and set `AutomationMode` via `Direct UI` handshake where available; otherwise warn via `platform_detail`) +- Modify: `crates/macos/src/tree/builder.rs` (honor the new `force_electron_a11y` flag; pair macOS change lands in the same PR per origin §Phase 2 "land atomically") +- **Note:** the `docs/solutions/best-practices/electron-compat-cross-platform-2026-04-18.md` port from private memory moves to Unit 14 (document-review scope reconciliation — that port is an orthogonal documentation task, not Electron-compat implementation). +- Test: `crates/windows/tests/it_electron.rs` (CI: snapshot VS Code with and without `--force-electron-a11y`; assert ≥100 refs with the flag) +- Test: `crates/macos/tests/it_electron.rs` (existing; extend to validate the new flag is respected) + +**Approach:** +- Depth-skip: mirror macOS `is_web_wrapper` logic — `UIA_GroupControlTypeId` or `UIA_CustomControlTypeId` with `Name.is_empty() && Value.is_empty()` returns true; wrapper is traversed but does not increment depth. +- Chromium detection: `GetClassNameW` on the HWND; if `Chrome_WidgetWin_1` and tree is empty, emit warning in `platform_detail` with `--force-renderer-accessibility` hint. +- Surface detection for Electron modals: when `surface = Sheet | Alert`, check if the focused window IS the target surface (not only its children). This matches macOS's pattern — Electron wraps dialogs as independent windows. +- `--force-electron-a11y`: on macOS, sets `AXEnhancedUserInterface = YES` on app root (existing). On Windows, attempt `SetPropertyValue(AutomationElementInformation.AutomationId, true)` on the root element; if the app does not honor the flag, emit a warning rather than erroring. + +**Patterns to follow:** +- `crates/macos/src/tree/builder.rs` `is_web_wrapper` (exists today — check for `AXGroup` / `AXGenericElement` with empty name/value). +- Learning 4 — `Some("")` vs `None` bug: Chromium returns `Some("")`; always use `is_none_or(str::is_empty)`. + +**Test scenarios:** +- **Happy path:** VS Code snapshot **without** `--force-electron-a11y` returns ~3 refs (matches current behavior); **with** the flag returns ≥100 refs (R15). +- **Happy path:** Slack snapshot with depth-skip enabled returns ≥50 refs. +- **Edge case:** non-Chromium window with empty-name groups is NOT depth-skipped (ensures the heuristic doesn't over-trigger on, say, empty GroupBoxes in WinForms). +- **Edge case:** Chromium app in "accessibility disabled" mode → empty tree + warning in `platform_detail` suggesting `--force-renderer-accessibility`. +- **Integration:** file-picker dialog in VS Code → `--surface sheet` detects it as the sheet surface (the dialog is a separate Electron window). +- **Integration:** `--force-electron-a11y` JSON output is identical between macOS and Windows against the same VS Code version (structural). + +**Verification:** +- CI integration tests green on both platforms. +- Ported solutions doc committed under `docs/solutions/best-practices/`. + +--- + +### - [ ] Unit 5: Stable-selector field population + +**Goal:** Both platforms populate `StableSelectors` wherever the OS/app exposes them. `resolve_element` prefers `identifier` when non-empty; falls back to the existing `(pid, role, name, bounds_hash)` fingerprint. `RefEntry.identifier` carries the selector across snapshots. Tests require known controls with explicit IDs to preserve them; real apps may omit identifiers. P2-O8. + +**Requirements:** R8 + +**Dependencies:** Unit 1 (types), Unit 3 (Windows adapter present), Unit 2 (FFI regen) + +**Files:** +- Modify: `crates/macos/src/tree/element.rs` → 2-way split into `element.rs` (core `AXElement` safety + `fetch_node_attrs` batch) + `element_selectors.rs` (new selector readers). File is at 404 L; splitting enforces the 400-LOC rule before adding reads. (Document-review refinement — Context & Research previously mentioned a 3-way split with `element_attrs.rs`; the authoritative split is 2-way.) +- Create: `crates/macos/src/tree/element_selectors.rs` (readers for `kAXIdentifierAttribute`, `kAXSubroleAttribute`, `kAXRoleDescriptionAttribute`, `kAXPlaceholderValueAttribute`, `kAXDOMIdentifierAttribute`, `kAXDOMClassListAttribute`) +- Modify: `crates/macos/src/tree/builder.rs` (populate `StableSelectors` during node construction) +- Modify: `crates/windows/src/tree/element.rs` (readers for `AutomationId`, `LocalizedControlType`, `HelpText`, plus WebView2 `HtmlId` / `HtmlClass` via `UIA_HtmlIdProperty` / `UIA_HtmlClassProperty`) +- Modify: `crates/windows/src/tree/builder.rs` (populate) +- Modify: `crates/core/src/ref_alloc.rs` (populate `RefEntry.identifier` when `StableSelectors.identifier` is non-empty) +- Modify: `crates/macos/src/tree/resolve.rs` (prefer-identifier: if `entry.identifier.is_some()`, search by `AXIdentifier` match first; fall back to `(pid, role, name, bounds_hash)`) +- Modify: `crates/windows/src/tree/resolve.rs` (same, via `AutomationId`) +- Test: `crates/macos/tests/it_selectors.rs` (target: a test harness `.app` that exports `accessibilityIdentifier` on known buttons; assert those controls carry `identifier` and ordinary controls without IDs omit it) +- Test: `crates/windows/tests/it_selectors.rs` (target: Calculator; assert the `=` button carries `AutomationId = "equalButton"`) +- Test: `crates/core/src/ref_alloc.rs` (unit: `RefEntry.identifier` preserved through allocator) +- Test: `tests/integration/stale_ref_rate.rs` (regression: against a fixture of 100 Electron/localized elements, resolver success rate with identifier preference ≥ baseline + 20 pp; captures the "measurably drops" metric in R8) + +**Approach:** +- macOS: `AXUIElementCopyMultipleAttributeValues` request already batches reads; extend its attribute list with the 6 selector attributes. No extra round-trip. +- Windows: `UICacheRequest` already batches; add the 6 selector properties to the cache request. No extra round-trip. +- Resolver prefers: if `entry.identifier.is_some()`, walk the tree and match by `identifier` (O(n) per level but typically matches in first 3 levels). Only if identifier lookup fails, fall back to bounds-hash fingerprint. Matches learning 4 — identifier is an **optional** fingerprint, never mandatory. +- `dom_classes` on Windows requires `UIA_HtmlClassProperty` which is WebView2-only; returns empty Vec on non-Chromium apps. This is expected. + +**Patterns to follow:** +- `crates/macos/src/tree/element.rs:fetch_node_attrs` — mirror batching shape for the new attributes. +- Learning 1 — define `SelectorReadConfig` in `crates/core` so macOS / Windows / (future) Linux readers share shape. +- Learning 4 — tri-state UTF-8 at FFI; invalid bytes → `INVALID_ARGS`. + +**Test scenarios:** +- **Happy path (macOS):** A test-harness "Save" button has `subrole = "AXConfirmButton"` and `identifier = "save"` when the app exposes it. +- **Happy path (Windows):** Calculator's "=" button has `identifier = "equalButton"` and `role_description = "button"`. +- **Happy path:** `snapshot --app Calculator` JSON includes the `identifier` field on the `=` button; roundtrip through refmap preserves it. +- **Edge case:** element without an accessibility identifier emits no `identifier` field (skip_serializing_if). +- **Edge case:** WebView2 element in Edge has `dom_id` and `dom_classes` populated; non-WebView2 element in Notepad has neither. +- **Edge case:** localized app — same button carries the same `identifier` on English and German system locales (verifies identifier is locale-stable). +- **Integration:** `resolve_element` for a button whose `name` changed (e.g. Chrome tab title) but whose `identifier` is stable → resolves successfully; before this unit, returned `STALE_REF`. +- **Integration:** STALE_REF regression against a 100-element fixture: measured resolution success rate is at least +20 pp with identifier preference (the "measurably drops" metric in R8; baseline captured in a fixture generated once before this unit). + +**Verification:** +- Both platform CI integration tests green. +- `tests/integration/stale_ref_rate.rs` asserts the +20 pp metric. + +--- + +### - [ ] Unit 6: Action variant implementations — `LongPress`, `ShowMenu`, `WindowRaise`, `Cancel` + +**Goal:** Implement the four cross-platform "simple" new Action variants on both platforms. `SelectRange` and `InsertAtCaret` wait for Unit 8 (require text infrastructure). `DeliverFiles` + `ForceClick` ship in Unit 12 (require heavier platform-specific scaffolding). Subset of P2-O9. + +**Requirements:** R9 (subset) + +**Dependencies:** Unit 1 (variants declared), Unit 3 (Windows adapter present) + +**Pre-work (first sub-PR before any arm additions — deepening-pass addition):** `crates/macos/src/actions/chain_steps.rs` is **already at 407 LOC** (over the 400 cap today) and `crates/macos/src/actions/dispatch.rs` at 349 LOC will breach adding 4 arms. Before this unit's action-logic sub-PRs, land a refactor sub-PR that splits: +- `chain_steps.rs` → `chain_steps.rs` (step definitions) + `chain_steps_extra.rs` (the overflow) OR along responsibility seams; target each ≤350 LOC. +- `dispatch.rs` → `dispatch.rs` (primary dispatch) + `dispatch_variants.rs` (the 4 new Phase 2 arms + future U12 arms); target each ≤350 LOC. +This refactor is behavior-preserving (reorganizes internal module structure only, no public API change). U12 inherits the split, so its 2 additional arms (`DeliverFiles`, `ForceClick`) land in `dispatch_variants.rs` cleanly. + +**Files:** +- Modify: `crates/macos/src/actions/chain_steps.rs` (split pre-work — ≤350 LOC each side) +- Create: `crates/macos/src/actions/chain_steps_extra.rs` (split pre-work) +- Modify: `crates/macos/src/actions/dispatch.rs` (split pre-work; then arms for `LongPress`, `ShowMenu`, `WindowRaise`, `Cancel` land in `dispatch_variants.rs`) +- Create: `crates/macos/src/actions/dispatch_variants.rs` (new arms here — uses `ActionDispatchConfig` from core) +- Modify: `crates/windows/src/actions/dispatch.rs` + optional `dispatch_variants.rs` split if needed +- Create: `crates/core/src/commands/long_press.rs`, `show_menu.rs`, `window_raise.rs`, `cancel.rs` +- Modify: `crates/core/src/commands/mod.rs`, `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` +- Test: `crates/macos/tests/it_actions_new.rs` +- Test: `crates/windows/tests/it_actions_new.rs` +- Test: `crates/macos/src/actions/` (smoke test after split — all existing behavior preserved) + +**Approach:** +- `LongPress { duration_ms }`: macOS `CGEventCreateMouseEvent(MouseDown)` + sleep + `MouseUp` at element's `bounds.midpoint()`. Windows `SendInput` MOUSEDOWN + sleep + MOUSEUP. Duration clamped to `[50, 10_000]` ms with `InvalidArgs` outside. +- `ShowMenu`: macOS `AXPerformAction(kAXShowMenuAction)` if supported by the element; fallback to right-click. Windows `ExpandCollapsePattern.Expand` if available; fallback to `SendInput` right-click. +- `WindowRaise`: macOS `AXUIElementSetAttributeValue(kAXRaisedAttribute, true)` + `AXUIElementPerformAction(kAXRaiseAction)`. Windows `SetForegroundWindow(HWND)` + `SetWindowPos(HWND_TOP)`. This is an explicit focus/window command, not a fallback; its command policy must permit focus steal and tests assert the side effect is intentional. +- `Cancel`: macOS `AXPerformAction(kAXCancelAction)`. Windows `WindowPattern.Close` on dialog, or `InvokePattern.Invoke` on the cancel button if detectable; fallback synthesizes Escape. + +**Patterns to follow:** +- Existing `crates/macos/src/actions/dispatch.rs` match arms for `Click`, `Toggle`, `Expand`. +- `crates/core/src/commands/click.rs` as command-file shape (14 L). + +**Test scenarios:** +- **Happy path:** `long-press @e<ref> --duration 500` on a macOS button fires MouseDown then MouseUp 500 ms apart; element's `pressed` state observed transiently via probe. +- **Happy path:** `show-menu @e<ref>` on a Finder file opens the context menu; subsequent `list-surfaces` shows `menu` surface. +- **Happy path:** `window-raise` brings a background window to the front (verify `list-windows --focused-only` before/after). +- **Happy path:** `cancel` on a macOS Save dialog dismisses it; dialog disappears from `list-surfaces`. +- **Edge case:** `long-press` with `duration=0` returns `InvalidArgs`. +- **Edge case:** `show-menu` on an element that lacks context-menu support falls back to right-click; integration test verifies a menu appears or an `ActionNotSupported` is returned cleanly. +- **Error path:** `cancel` on a non-dialog element returns `ActionNotSupported`. +- **Integration:** cross-platform JSON shape identical for all four commands. + +**Verification:** +- Both platforms' CI green for `it_actions_new.rs`. + +--- + +### - [ ] Unit 7: `watch_element` — event subscription with push notifications + +**Goal:** `watch --event <kind> --ref @e<id> --timeout <ms>` returns events within 500 ms of a programmatic change. P2-O11. Replaces the polling in `system/wait.rs` for element existence when event-driven path is available. + +**Requirements:** R11 + +**Dependencies:** Unit 1 (types: `EventKind`, `ElementEvent`, trait method stub), Unit 3 (Windows adapter present). + +**Ordering note (review-refined):** Unit 7's **opening spike PR** (validation-only, no shipped code) must land **before** Unit 1 sub-PR 1h so that `watch_element` trait signature + threading model are informed by validated behavior, not guessed. Sequence: U1 sub-PRs 1a→1g land → U7 spike lands → U1 sub-PR 1h lands (trait method signatures informed by spike) → U7 implementation proceeds. + +**Files:** +- Modify: `crates/core/src/adapter.rs` (confirm `watch_element(handle, events, timeout) → Result<Vec<ElementEvent>, AdapterError>` signature; stub landed in Unit 1) +- Create: `crates/macos/src/events/{mod,observer,runloop_worker}.rs` (AXObserver + CFRunLoop on a dedicated worker thread) +- Create: `crates/windows/src/events/{mod,handler,mta_worker}.rs` (UIA event handler on a dedicated MTA thread) +- Create: `crates/core/src/commands/watch.rs` +- Modify: `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` (`wait --event <kind>` flag, with `--event` repeatable) +- Modify: `crates/core/src/commands/wait.rs` (when `--event` present, dispatch to new watch command rather than the polling path) +- Test: `crates/macos/tests/it_watch.rs` (value-changed, focus-changed, menu-opened scenarios against TextEdit / Finder) +- Test: `crates/windows/tests/it_watch.rs` (value-changed, focus-changed against Notepad / Settings) +- Test: `crates/core/src/commands/wait.rs` (unit: ref-format validation separate from event-kind validation) + +**Approach:** +- **Opening spike (research-directed):** validate the **asymmetric** threading model on both platforms: + - **macOS:** confirm that a **main-thread `CFRunLoop`** (the CLI's main thread runs `CFRunLoopRunInMode` during `watch_element`) supports `AXObserver` attach + event delivery on the same thread. Research Topic A: every production AX consumer (AXSwift, Hammerspoon, Phoenix) binds `AXObserverGetRunLoopSource` to `CFRunLoopGetMain()` — off-main attachment is a trap, not a supported configuration. Validate against Finder, TextEdit, VS Code on macOS 14/15. + - **Windows:** confirm that a worker thread on `CoInitializeEx(MTA)` hosts `AddAutomationEventHandler` / `AddPropertyChangedEventHandler` with cross-thread event delivery via mpsc to the main thread. Per Microsoft's 2025 UIA threading doc this is the supported pattern. Validate against Notepad, Settings, VS Code. + - Spike also validates **(a)** callback-panic → stop path (`catch_unwind` around the AX/UIA callback emits a synthetic `ElementEvent::CallbackPanic` and exits cleanly) and **(b)** hard-join timeout = `2 × user_timeout_ms` returning `ErrorCode::Internal`. + - Spike PR lands **before** Unit 1 sub-PR 1h so trait signatures reflect validated behavior. +- **Trait signature:** `watch_element(&self, entry: &RefEntry, spec: &WatchSpec) -> Result<Vec<ElementEvent>, AdapterError>`. Takes `&RefEntry`, not `&NativeHandle`. Each platform's implementation resolves the element on whichever thread owns its accessibility state. +- **Asymmetric threading model (research-refined — Apple and Microsoft prescribe different patterns):** + - **macOS — main-thread `AXObserver`.** Research Topic A confirmed all AX functions are main-thread-only. Implementation: + 1. Main thread: resolve `RefEntry` → `AXUIElementRef`, `AXObserverCreate(pid, callback)`, `AXObserverAddNotification` for each subscribed kind, `CFRunLoopAddSource(CFRunLoopGetMain(), AXObserverGetRunLoopSource(observer), kCFRunLoopDefaultMode)`. + 2. Main thread: spawn a tiny **signal thread** whose only job is `thread::sleep(timeout_duration)` + `CFRunLoopStop(main_loop)`. + 3. Main thread: `CFRunLoopRunInMode(kCFRunLoopDefaultMode, remaining_secs, false)` — blocks until timeout OR `CFRunLoopStop`. + 4. Callback (main thread): appends to a `RefCell<Vec<ElementEvent>>` local to `watch_element`; no mpsc needed — callback and consumer are the same thread. + 5. Teardown: `AXObserverRemoveNotification` + `CFRelease(observer)`; signal thread joins cleanly (either via its sleep expiring or a cancellation flag). + - **Windows — worker-thread MTA handler.** UIA supports cross-thread event delivery: + 1. Main thread: resolve `RefEntry` → `IUIAutomationElement` on the main-thread MTA, extract `(pid, automation_id, runtime_id)` for the worker. + 2. Main thread: spawn worker thread → worker calls `CoInitializeEx(NULL, COINIT_MULTITHREADED)`, creates its own `IUIAutomation` client, re-resolves the element by fingerprint on its thread (apartment-affine handles do not cross thread boundaries per Research Topic 4). + 3. Worker: installs `AddAutomationEventHandler` (and friends) with handlers stored as `Arc<Handler>` so the handler object outlives the final dispatched callback (Research Topic 4 — post-remove barrier pattern). + 4. Worker: handler pushes `ElementEvent` into `mpsc::Sender` wrapped in `catch_unwind`. + 5. Main thread: `recv_timeout(Duration::from_millis(timeout_ms))` on the receiver. + 6. Teardown: main calls `PostThreadMessage(worker_tid, WM_QUIT)`; worker calls `RemoveAutomationEventHandler` + no-op UIA call (post-remove barrier drains in-flight callbacks) + `CoUninitialize`. Hard-join within `2 × timeout_ms`. +- **No shared `EventWorker` trait.** The two platforms are categorically different (main-thread CFRunLoop vs worker MTA). Inlining is simpler than abstracting over incompatible shapes. Revisit at Phase 3 when AT-SPI dbus joins; likely Linux is also asymmetric, and the three-way shape may or may not unify. +- Public API is **synchronous**: `watch_element` returns `Vec<ElementEvent>` or `TIMEOUT` error. +- **`WaitStrategy` enum in `commands/wait.rs`** (learning 1 refinement): introduce `enum WaitStrategy { EventDriven(WatchSpec), Polling(PollSpec) }` that `execute()` matches over. The existing `wait --element` / `--window` / `--text` / `--menu` / `--notification` routes construct `Polling(...)`; the new `wait --event` route constructs `EventDriven(...)`. This prevents future flags (e.g., `--until-visible`) from adding yet another branch. +- Error taxonomy (learning 2): ref syntax error → `INVALID_ARGS`; ref valid but element gone → `STALE_REF`; no events within timeout → `TIMEOUT`; observer couldn't attach → `ACTION_FAILED` with `platform_detail` naming the AX/COM error; > 32 subscriptions → `RESOURCE_EXHAUSTED`; worker hard-join timeout → `INTERNAL` with platform detail. +- **Secure-field redaction in `ElementEvent.attr_snapshot` (review-added — security):** before pushing an `ElementEvent` into the mpsc channel, the worker inspects the source element. If `AXSubrole == AXSecureTextField` (macOS) or `UIA_IsPasswordProperty == true` (Windows), the `value` field on the carried `AccessibilityNode` is replaced with `<redacted>`. This prevents `watch --event value-changed` from streaming the literal keystrokes as a user types into a password field. +- **Concurrency cap (review-added):** a process-wide `AtomicUsize` counts in-flight `watch_element` calls. Cap default `32`; beyond that, return `RESOURCE_EXHAUSTED` with suggestion "Close other watch sessions or raise AGENT_DESKTOP_MAX_WATCHES". Per-call MTA apartment creation on Windows costs ~5-20ms cold; unbounded fan-out from FFI consumers can hit COM rate limits. Cap configurable via env var. + +**Execution note:** Open with the spike PR (no shipped code) before implementation units. + +**Technical design (research-refined — asymmetric by platform):** + +``` +macOS — main-thread AXObserver (research Topic A): +============================================ +main thread signal thread +----------- ------------- +watch_element(entry, spec) + │ + AXObserverCreate + AddNotification + │ + CFRunLoopAddSource(main_loop, ...) + │ + ├── spawn ────────────────────────► sleep(timeout_ms) + │ CFRunLoopStop(main_loop) + │ + CFRunLoopRunInMode(default, t, false) + │ + (callback on same thread appends to local Vec<ElementEvent>) + │ + return Vec<ElementEvent> + + +Windows — worker-thread MTA handler (research Topic 4): +============================================ +main thread worker thread +----------- ------------- +watch_element(entry, spec) + │ + ├── spawn ────────────────────────► CoInitializeEx(MTA) + │ IUIAutomation::new() on this thread + │ re-resolve element by fingerprint + │ AddAutomationEventHandler(Arc<Handler>) + │ (handler:) + │←────── mpsc::Sender ──────────── catch_unwind(|| tx.send(ElementEvent)) + │ + recv_timeout(Duration::from_ms(t)) + │ + PostThreadMessage(worker_tid, WM_QUIT) ─► RemoveAutomationEventHandler + │ post-remove barrier (no-op UIA call) + │ CoUninitialize + │ thread::exit + │ + hard-join (2 * timeout_ms) or Internal + │ + return Vec<ElementEvent> +``` + +**Patterns to follow:** +- `crates/macos/src/system/wait.rs` — current polling path that this unit replaces for element-existence scenarios. +- Learning 2 — separate error codes by failure mode; boundary-node pattern for capped subscription counts (max 32 active subscriptions per call; beyond that → `ResourceExhausted` with suggestion). + +**Test scenarios:** +- **Happy path (macOS):** Start `watch --event value-changed --ref @e<textfield> --timeout 2000`; separately, set the field's value via `set-value`; the watch command returns a single `ValueChanged` event within 500 ms (R11 metric). +- **Happy path (Windows):** Same against Notepad. +- **Happy path:** Multi-event subscription — `--event value-changed --event focus-changed` receives both kinds. +- **Edge case:** `--event unknown-kind` → `InvalidArgs` with the list of accepted kinds. +- **Edge case:** ref has bad syntax (`not-a-ref`) → `InvalidArgs` (not `StaleRef`) per learning 2. +- **Edge case:** ref valid but element gone → `StaleRef` with snapshot-refresh suggestion. +- **Error path:** timeout reached with no events → `Timeout` (not `Ok(vec![])`). The contract: the command returns `Ok` only when events fired. +- **Edge case:** > 32 subscriptions → `ResourceExhausted` with "max 32 kinds per call" suggestion. +- **Integration:** worker thread joins cleanly; `std::thread::available_parallelism()` usage does not leak threads across 100 sequential `watch_element` calls (regression harness). +- **Integration:** the legacy polling path in `system/wait.rs` still works for non-event scenarios (`wait --element` without `--event`). + +**Verification:** +- Spike PR lands first (separately reviewable — no shipped code); implementation PR cites it. +- Both platforms' CI green; 500 ms latency target met in the metric assertion. + +--- + +### - [ ] Unit 8: Text range primitives + +**Goal:** `text get-selection`, `text select-range`, `text insert-at-caret`, `text at-offset` on both platforms. Enables `Action::SelectRange` and `Action::InsertAtCaret` dispatch arms added in Unit 1. P2-O12. + +**Requirements:** R12 + +**Dependencies:** Unit 1 (types), Unit 3 (Windows adapter present) + +**Files:** +- Create: `crates/macos/src/text/{mod,selection,range,param_attrs}.rs` (parameterized attribute helpers: `AXStringForRangeParameterizedAttribute`, `AXBoundsForRangeParameterizedAttribute`, `AXRangeForLineParameterizedAttribute`; `AXValueCreate(kAXValueCFRangeType)`) +- Create: `crates/windows/src/text/{mod,text_pattern,range}.rs` (`TextPattern.GetSelection`, `TextRange.Select`, `TextRange.Move`, `TextRange.GetText`, `TextRange.GetBoundingRectangles`) +- Create: `crates/core/src/commands/text_get_selection.rs`, `text_select_range.rs`, `text_insert_at_caret.rs`, `text_at_offset.rs` +- Modify: `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` (top-level `text` subcommand with four sub-subcommands) +- Modify: `crates/macos/src/actions/dispatch.rs` + `crates/windows/src/actions/dispatch.rs` (`Action::SelectRange` → `set_text_selection`; `Action::InsertAtCaret` → `insert_text_at_caret`) +- Test: `crates/macos/tests/it_text.rs` (TextEdit: select-range + get-selection roundtrip; insert-at-caret advances caret) +- Test: `crates/windows/tests/it_text.rs` (Notepad: same) + +**Approach:** +- Use **UTF-16 code units** for `TextRange { start, length }` consistently across both platforms (matches AX `CFRange` and UIA `TextRange` native conventions). Document this at the public-contract boundary with a one-line doc comment on `TextRange`. +- macOS `get_text_selection`: read `kAXSelectedTextRangeAttribute` → `CFRange`; decode to `TextRange`. +- macOS `set_text_selection`: `AXValueCreate(kAXValueCFRangeType, &CFRange)` → `AXUIElementSetAttributeValue(kAXSelectedTextRangeAttribute, value)`. +- macOS `insert_text_at_caret`: get selection → `AXUIElementSetAttributeValue(kAXSelectedTextAttribute, string)` replaces selection with string (and advances caret to end). +- **Password-field gate (review-added — security, both platforms):** before any text-read operation (`get_text_selection`, `get_text_at`), the adapter checks the target element for password-field status. macOS: `AXSubrole == AXSecureTextField` → return `ActionNotSupported` with suggestion "Text reads are blocked on password fields; use set-value to write without reading." Windows: `UIA_IsPasswordProperty == true` OR `ControlType == PasswordEdit` → same error. **Writes (`set_text_selection`, `insert_text_at_caret`) are allowed** on password fields because typing a credential is a legitimate agent operation. The `TextRangeConfig.check_password_field: bool` (default `true`) gates this; explicit `--allow-password-read` flag opts out with a stern CLI warning, reserved for security research scenarios. +- Windows `get_text_selection`: `TextPattern.GetSelection` → first `TextRange.GetBoundingRectangles` → start/length relative to `TextPattern.DocumentRange.GetText(-1)` UTF-16 length. +- Windows `set_text_selection`: `TextPattern.DocumentRange.Clone().Move(TextUnit_Character, start).MoveEndpointByRange(End, start+length)` → `Select`. +- Windows `insert_text_at_caret`: `set_text_selection` to caret-only range → `SendInput` typing the text (UIA has no direct "insert at caret" primitive); alternatively `TextPattern.SupportedTextSelection` fallback. + +**Patterns to follow:** +- Learning 2 — `INVALID_ARGS` for bad range bounds (negative, overflow); `STALE_REF` for gone elements; `ACTION_FAILED` for elements that don't support text patterns. + +**Test scenarios:** +- **Happy path:** TextEdit with "hello world" → `text select-range @e<field> 0 5` + `text get-selection @e<field>` returns `{ start: 0, length: 5, text: "hello" }`. +- **Happy path:** `text insert-at-caret @e<field> "bye "` inserts at current caret; field value reflects it; caret advanced by 4. +- **Happy path:** `text at-offset @e<field> 0 5` returns `"hello"` without mutating selection. +- **Edge case:** `start` > document length → `InvalidArgs` with the document length in the suggestion. +- **Edge case:** `length = 0` selects nothing (caret-only); valid. +- **Edge case:** element without `TextPattern` on Windows or `AXValue` string type on macOS → `ActionNotSupported`. +- **Edge case:** Unicode surrogate pairs ("👋") — UTF-16 `length = 2` for one emoji; test verifies boundary alignment. +- **Error path:** element gone between snapshot and call → `StaleRef`. +- **Integration:** cross-platform roundtrip — same test against a multi-line field on both platforms produces identical JSON with identical offsets. + +**Verification:** +- Both platform CI integration tests green; roundtrip assertions equal on macOS and Windows. + +--- + +### - [ ] Unit 9: Modern screenshot backends + +**Goal:** Modern default on both platforms where the OS/session supports it. macOS via ScreenCaptureKit (`screencapturekit 1.5`); Windows via `windows-capture 1.5.4`. Legacy fallback (`--screenshot-backend legacy`) keeps the Phase 1 subprocess on macOS and `PrintWindow` on Windows. Cold latency <50 ms on supported modern paths. P2-O13. + +**Requirements:** R13 + +**Dependencies:** Unit 1 (`ScreenshotBackend` type, trait method `get_screenshot_with_backend`), Unit 3 (Windows adapter; legacy screenshot already landed there) + +**Files:** +- Modify: `Cargo.toml` (target-gated: macOS gains `screencapturekit = "1.5"` (research-refined — **published crates.io** canonical crate as of Q1 2026, doom-fish fork is the maintained successor and is published there; no git-SHA pin needed) + `objc2 = "0.6"` with `Foundation`/`AppKit` features; Windows gains `windows-capture = "1.5.4"` (latest stable, pins `windows = "0.62.2"` matching our workspace)) +- Create: `docs/security/screenshot-deps-audit-2026-04-18.md` (**review-added** — documents (a) why `screencapturekit 1.5` (crates.io) is trusted, (b) the weekly CI audit that compares `Cargo.lock` hash against last known good, (c) the screen-capture code paths reviewed for exfiltration risk. Committed before Unit 9 merge.) +- Modify: `.github/workflows/ci.yml` (**review-added** — weekly cron job: `cargo update --dry-run -p windows-capture -p screencapturekit` reports transitive dep shifts; alerts on unexpected movement.) +- Modify: `crates/macos/Cargo.toml`, `crates/windows/Cargo.toml` (mirror deps) +- Modify: `crates/macos/src/system/screenshot.rs` (split into `modern.rs` + `legacy.rs` if LOC pressure; wire `ScreenshotBackend::Modern` to ScreenCaptureKit `SCShareableContent.windows` + `SCScreenshotManager.captureImage`) +- Modify: `crates/windows/src/system/screenshot.rs` (split; modern via `windows-capture` + `Direct3D11CaptureFramePool`) +- Modify: `crates/core/src/commands/screenshot.rs` (wire `--screenshot-backend <modern|legacy>` flag; default Modern) +- Modify: `src/cli_args.rs` (add flag) +- Modify: `crates/macos/src/adapter.rs` + `crates/windows/src/adapter.rs` (implement `get_screenshot_with_backend`; `screenshot` legacy trait method routes to `get_screenshot_with_backend(ScreenshotBackend::Legacy)`) +- Create: `.github/workflows/ci.yml` grep guard step — fails if `objc2::` import appears outside `crates/macos/src/system/screenshot.rs` / `crates/macos/src/system/permissions.rs` (R6 mitigation per origin risks) +- Test: `crates/macos/tests/it_screenshot_modern.rs` (SSIM compare modern vs legacy against Finder window; latency assertion <50 ms) +- Test: `crates/windows/tests/it_screenshot_modern.rs` (same against Notepad) + +**Approach:** +- **Spike first** per origin risk R3: verify `windows-capture 1.5.4` API + `Capture::start` + frame-callback signature compiles with `windows 0.62.2` before merging. Verify `screencapturekit 1.5` `SCScreenshotManager.captureImage(contentFilter:config:)` signature on macOS 14/15. +- macOS modern: `SCShareableContent.current(excludingDesktopWindows: false, onScreenWindowsOnly: true)` → filter by `CGWindowID` → `SCContentFilter(desktopIndependentWindow: window)` → `SCScreenshotManager.captureImage(contentFilter:config:)`. Config: `SCStreamConfiguration` with `width`/`height` = window bounds; `pixelFormat = BGRA`. Result: `CGImage` → `CGImageDestinationCreateWithData` → PNG buffer. +- Windows modern: first check WGC/session support (`GraphicsCaptureSession::IsSupported` where available plus a real `CreateFromWindowHandle` probe). Then `GraphicsCaptureItem.CreateFromWindowHandle(HWND)` → `Direct3D11CaptureFramePool::Create(device, BGRA8, 2, item.Size)` → attach `FrameArrived` callback; start session; receive one frame; copy to CPU memory via D3D11 `Map`; encode PNG via `image` crate or `windows::Win32::Graphics::Imaging::WIC`. +- Legacy fallback: macOS continues `/usr/sbin/screencapture -R <rect> -t png`; Windows continues `PrintWindow(hwnd, hdc, PW_RENDERFULLCONTENT)`. +- `--screenshot-backend legacy` exists for WDA-protected windows (e.g. password managers) where modern APIs return a black frame. + +**Execution note:** Open with a two-PR spike — one per platform — validating API compilation + single capture against a test app. Commit spike outputs (sample PNGs) in `tests/fixtures/screenshot-*.png` as ground truth for the SSIM regression. + +**Patterns to follow:** +- `crates/macos/src/system/screenshot.rs` current structure — keep its file shape, split into modern/legacy submodules. +- Learning 4 — screenshot permission handling (Screen Recording TCC); failure → `PermDenied` with Screen-Recording-specific suggestion (distinct from AX). Pairs with Unit 11. + +**Test scenarios:** +- **Happy path (macOS):** `screenshot --app Finder` (Modern) produces a valid PNG; dimensions match Finder window bounds; file opens in Preview. +- **Happy path (Windows):** `screenshot --app Notepad` (Modern) produces a valid PNG via `windows-capture`. +- **Happy path:** `screenshot --app Notepad --screenshot-backend legacy` produces a valid PNG via `PrintWindow`. +- **Integration:** cold latency — `screenshot --app Finder` Modern completes in <50 ms on macOS; <50 ms on Windows when WGC is supported by the runner/session (R13 metric). Measured across 10 runs; median reported. +- **Integration:** SSIM — Modern vs Legacy outputs against the same window match with SSIM ≥ 0.95 (not pixel-identical because color spaces differ, but structurally identical). +- **Edge case:** Modern backend unavailable (pre-macOS 12.3, pre-Win10 1903, Session 0, Server Core, locked/secure desktop, or runner without WGC support) → automatic fall-through to Legacy when possible with a warning in `platform_detail`; otherwise `PLATFORM_NOT_SUPPORTED`. +- **Edge case:** WDA-protected window → Modern returns black frame; user re-runs with `--screenshot-backend legacy`; legacy succeeds. +- **Error path:** denied Screen Recording on macOS → Modern returns `PermDenied` with Screen-Recording-specific suggestion (Unit 11's tri-state makes the distinction from AX possible). +- **Integration:** grep guard CI step — adding `use objc2::…` outside the two allowed files fails CI with a clear message. + +**Verification:** +- Both platforms' CI green; latency regression asserted; SSIM assertion green; grep guard active. + +--- + +### - [ ] Unit 10: New surfaces — Toolbar, Spotlight, Dock, MenuBarExtras, Windows shell surfaces + +**Goal:** macOS: add `Toolbar`, `Spotlight`, `Dock`, `MenuBarExtras` surfaces; ship tray commands (mirror of U3b). Windows: add structured shell surfaces for `Toolbar`, `Taskbar`, `SystemTray`, `SystemTrayOverflow`, `StartMenu`, `ActionCenter`, and `QuickSettings` where the current Windows build/session exposes them. P2-O14 and P2-O18. + +**Requirements:** R14 + +**Dependencies:** Unit 3 (Windows adapter), Unit 3b (Windows tray commands already merged; this unit backfills macOS) + +**Files:** +- Modify: `crates/core/src/adapter.rs` (`SnapshotSurface` enum gains `Toolbar`, `Spotlight`, `Dock`, `MenuBarExtras`, `Taskbar`, `SystemTray`, `SystemTrayOverflow`, `StartMenu`, `ActionCenter`, `QuickSettings`) +- Modify: `crates/macos/src/tree/surfaces.rs` (detect Toolbar via `AXRole == AXToolbar` or `AXUnifiedTitleAndToolbar`; Spotlight via `Spotlight.app` PID; Dock via `Dock.app` PID; MenuBarExtras via `SystemUIServer` + `ControlCenter` + per-app `AXExtrasMenuBar`) +- Modify: `crates/windows/src/tree/surfaces.rs` (Taskbar/SystemTray via `Shell_TrayWnd`, overflow via `NotifyIconOverflowWindow`, Start/ActionCenter/QuickSettings via shell-surface roots in `list_surfaces`; the open-surface and tray command dispatchers already landed in U3b) +- Modify: `crates/macos/src/adapter.rs` (implement `list_tray_items`, `click_tray_item`, `open_tray_menu` against MenuBarExtras items) +- Modify: `crates/core/src/commands/list_surfaces.rs` (no change — the new `SnapshotSurface` variants flow through) +- Modify: `src/cli_args.rs` (snapshot `--surface` accepts the new kinds) +- Test: `crates/macos/tests/it_surfaces.rs` (`snapshot --surface toolbar` on Safari; `list-surfaces` includes Spotlight / Dock / MenuBarExtras pids) +- Test: `crates/windows/tests/it_surfaces.rs` (`snapshot --surface toolbar` on Edge; `list-surfaces` includes present Windows shell surfaces) + +**Approach:** +- Spotlight surface: launch Spotlight via `Cmd+Space` equivalent if needed (Spotlight process is ephemeral); `NSRunningApplication.applications(withBundleIdentifier: "com.apple.Spotlight")` → pid → AX subtree. +- Dock surface: `NSRunningApplication.applications(withBundleIdentifier: "com.apple.dock")` → pid → AX subtree; children are `AXDockItem` entries. +- MenuBarExtras surface: union of `SystemUIServer`, `ControlCenter`, and each running app's `AXExtrasMenuBar`. +- Windows shell surfaces: walk `Shell_TrayWnd` UIA children + `NotifyIconOverflowWindow`; surface roots opened by `open-system-surface` are snapshotted immediately through their UIA roots. + +**Patterns to follow:** +- `crates/macos/src/tree/surfaces.rs` Menu / Sheet / Popover detection. +- Learning 1 — `SurfaceDetectionConfig` shared shape so macOS and Windows surface detectors don't diverge by accident. + +**Test scenarios:** +- **Happy path:** `snapshot --surface toolbar --app Safari` returns toolbar children (back, forward, URL field, bookmarks) with refs. +- **Happy path:** `list-surfaces` on macOS includes `{type: "spotlight"}`, `{type: "dock", item_count: N}`, `{type: "menubar_extras", item_count: M}`. +- **Happy path:** `list-surfaces` on Windows includes present shell surfaces such as `{type: "system_tray", item_count: K}`, `{type: "taskbar", item_count: M}`, `{type: "start_menu"}`, `{type: "action_center"}`, and `{type: "quick_settings"}`. +- **Happy path:** `snapshot --surface toolbar --app Edge` on Windows returns Edge's toolbar children. +- **Edge case:** Spotlight not running → Spotlight surface missing from `list-surfaces` (not an error). +- **Edge case:** Dock hidden (auto-hide enabled) → Dock surface present but `item_count` reflects visible items. + +**Verification:** +- Both platforms' CI green for surfaces. + +--- + +### - [ ] Unit 11: Permission tri-state on macOS + +**Goal:** `check_permissions` on macOS returns the tri-state struct introduced in Unit 1, fully populated for Accessibility, Screen Recording, and Automation. `permissions` command output shows all three. Errors distinguish `PermDenied` (AX), `AutomationPermissionDenied`, and Screen-Recording-denied screenshot failures. P2-O17. + +**Requirements:** R17 + +**Dependencies:** Unit 1 (tri-state types, new error codes), Unit 9 (Modern screenshot uses Screen Recording) + +**Files:** +- Modify: `crates/macos/src/system/permissions.rs` (extend from the current ~55 L; add `CGPreflightScreenCaptureAccess` + `CGRequestScreenCaptureAccess` reads; add `AEDeterminePermissionToAutomateTarget` probe) +- Modify: `crates/macos/src/adapter.rs` (`check_permissions` returns tri-state) +- Modify: `crates/macos/src/system/screenshot.rs` (Modern backend returns `PermDenied` with Screen-Recording-specific suggestion when `CGPreflightScreenCaptureAccess` is false) +- Modify: `crates/macos/src/system/app_ops.rs` (`close_app` that requires AppleEvents returns `AutomationPermissionDenied` when `AEDeterminePermissionToAutomateTarget` says denied) +- Modify: `crates/core/src/commands/permissions.rs` (output shape matches tri-state) +- Test: `crates/macos/tests/it_permissions.rs` (on a macOS runner without Screen Recording granted: `screenshot` returns `PermDenied` with the right suggestion) +- Test: `crates/macos/tests/it_permissions_automation.rs` (verify `AutomationPermissionDenied` fires on `close-app` against an app lacking Automation grant) + +**Approach:** +- `CGPreflightScreenCaptureAccess` returns bool; `CGRequestScreenCaptureAccess` triggers the TCC prompt; expose the prompt behind an explicit `--request-permission` flag — do not auto-prompt during `check_permissions`. +- **TTY-gate on `--request-permission` (review-added — security):** the flag is permitted **only** when `atty::is(atty::Stream::Stdin)` returns true. Automated agent pipelines (stdin piped / not a TTY) reject with `ErrorCode::InvalidArgs` and suggestion "Permission requests require interactive stdin; run the tool manually once to grant." Prevents agent-consent-bypass where a malicious task payload instructs an automated agent to grant itself OS permissions. +- `AEDeterminePermissionToAutomateTarget(const AEAddressDesc *target, AEEventClass theAEEventClass, AEEventID theAEEventID, Boolean askUserIfNeeded)` — call with `askUserIfNeeded = false` during probing; the returned `OSStatus` maps to the tri-state (`noErr` → `Granted`, `errAEEventNotPermitted` → `Denied`, `procNotFound` → `Unknown`). +- Tri-state output JSON: + ```json + { + "accessibility": "Granted", + "screen_recording": { "state": "denied", "suggestion": "Open System Settings > Privacy & Security > Screen & System Audio Recording" }, + "automation": { "state": "not_determined" } + } + ``` +- Error mapping: screenshot failure with Screen Recording denied → `ErrorCode::PermDenied` + suggestion naming **Screen Recording** specifically (not the generic AX suggestion). `close_app` failure with Automation denied → `ErrorCode::AutomationPermissionDenied`. + +**Patterns to follow:** +- Existing `crates/macos/src/system/permissions.rs:check_permissions` shape. +- Learning 4 — tri-state matches the optional-fingerprint pattern. + +**Test scenarios:** +- **Happy path:** `permissions` on a fully-granted macOS runner returns all three = `Granted`. +- **Happy path:** on a Screen-Recording-denied runner, `permissions` returns `screen_recording.state = "denied"` with the Screen Recording suggestion; `accessibility.state` remains `Granted`. +- **Happy path:** `screenshot --app Finder` with Screen Recording denied returns `PermDenied` + Screen-Recording suggestion (not the generic AX one). +- **Happy path:** `close-app --app TargetApp` with Automation denied returns `AutomationPermissionDenied`. +- **Edge case:** `Unknown` state → the `permissions` JSON still has the field (no `suggestion` key in that branch). +- **Edge case:** `--request-permission` flag triggers `CGRequestScreenCaptureAccess`; subsequent `permissions` reflects the user's grant. +- **Integration:** cross-platform parity — the `permissions` command on Windows returns `accessibility = "Granted"` (always granted on Windows after UAC elevation) + `Unknown` for the other two. + +**Verification:** +- macOS CI green; Windows CI exercises the JSON shape (tri-state across both platforms). + +--- + +### - [ ] Unit 12: `DeliverFiles` + `ForceClick` per-platform (formerly "FileDrop") + +**Goal:** `DeliverFiles(Vec<PathBuf>)` (renamed from `FileDrop` per research) and `ForceClick` on macOS and Windows. Linux returns `ActionNotSupported` on `ForceClick` (Phase 3 non-goal). Balance of P2-O9. + +**Rename rationale (research-driven):** `NSDraggingSession` is NOT headless-compatible on macOS — it requires an `NSApplication` event loop, a window-server mouse-event stream, and foreground activation. Research Topic 1 confirmed this is not a gap to work around but a categorical incompatibility. The headless-first invariant (§Headless-First Invariant) forbids `NSDraggingSession` as a primary path. The action is renamed `DeliverFiles` to reflect the contract: "files appear at the destination" — the mechanism is platform-appropriate, not always a literal drag event. + +**Requirements:** R9 (remaining) + +**Dependencies:** Unit 1 (variants declared — `FileDrop` enum variant renamed to `DeliverFiles` in 1d), Unit 6 (simple variants landed) + +**macOS delivery strategy (research Topic 1 — four-tier fallback, headless-first):** +1. **Tier 1 — app-native URL scheme (per-app registry).** A small table in `crates/macos/src/actions/deliver_files.rs` maps known bundle IDs to their URL scheme / CLI protocols (e.g., `com.microsoft.VSCode` → `vscode://file/…`, `com.tinyspeck.slackmacgap` → slack slash-command). Only used when the target app advertises an official protocol; never guessed. +2. **Tier 2 — `NSWorkspace.open(urls:withApplicationAt:configuration:completionHandler:)` with `activates: false` (PRIMARY path for most apps).** Headless, non-focus-stealing, Apple-supported since macOS 10.15. Files open in the target app. Works for Finder, Preview, TextEdit, Safari, most native apps, and Electron apps that implement standard file-open protocol. +3. **Tier 3 — `NSPasteboard.general` write of `public.file-url` + `CGEventPostToPid(cmd+v)` to target PID.** Simulates paste-into-focused-view. Requires the target app to have a focused paste-capable view. Used for Electron text editors, note apps. Preserves the system clipboard by saving/restoring. +4. **Tier 4 — `osascript -e 'tell application "X" to open {POSIX file "/path"}'`.** AppleScript Apple Events. Requires Automation TCC grant (Unit 11 tri-state permission covers this). Fallback for apps with no URL scheme and no pasteboard response. + +Selection logic: `deliver_files.rs` tries tiers in order, returns on first success. Tier choice logged via `tracing::debug!` for the user to understand which path fired. `NSDraggingSession` is **NEVER** invoked. + +**Windows delivery strategy:** +- **Tier 1 — app-native URI / command handoff.** Use documented app URI handlers or shell verbs when the target app advertises them. Never guess private protocols. +- **Tier 2 — filesystem destination.** For Explorer windows and known filesystem destinations, use `IFileOperation::CopyItems` / `MoveItems` as the semantic path. This is the default for folders because it avoids fake drag state entirely. +- **Tier 3 — `CF_HDROP` clipboard paste.** Populate clipboard file-drop format, target the destination with an explicit paste action, then restore the prior clipboard. This is used only when the destination accepts paste semantics. +- **Tier 4 — `IDataObject + DoDragDrop` spike/fallback.** Implement only after a spike proves it can be driven without violating focus/cursor policy for the target class. It is never the default headless path. + +**ForceClick strategy unchanged from origin plan** — macOS via `kCGMouseEventPressure` on `CGEventCreateMouseEvent`; Windows via `SendInput` with pen-input flags. Linux → `ActionNotSupported`. + +**Opening spikes (first sub-PRs of Unit 12):** validate macOS Tier 2 (`NSWorkspace.open` with `activates: false`) against 3 targets — Finder folder, VS Code workspace, Safari tab — confirming (a) files land correctly, (b) no focus steal, (c) agent-desktop process remains headless. Validate Windows Tiers 2/3 against Explorer, Notepad/wordpad-style file-open targets, and one Electron app. Only after that spike can Tier 4 OLE drag be attempted for targets that require drag semantics. + +**Path-validation rules (review-added — security, both platforms):** +- All input paths pass through `std::fs::canonicalize` before any pasteboard / `IDataObject` write. +- Default: reject paths outside `$HOME` and `/tmp` (macOS), outside `%USERPROFILE%` and `%TEMP%` (Windows). +- Reject paths containing `..` segments **after** canonicalization (defense in depth). +- `--allow-system-paths` flag overrides the scope check with a CLI warning; never enabled by default, never settable via env var (explicit intent required). +- Empty path list → `InvalidArgs`. +- Non-existent source path → `InvalidArgs` with the missing path in the message. +- Threat model documented in `docs/solutions/logic-errors/deliver-files-path-safety-2026-04-18.md` (created in U14 docs pass). + +**Files:** +- Create: `crates/macos/src/actions/deliver_files.rs` (4-tier headless fallback from §Unit 12 strategy; `NSWorkspace.open` is Tier 2 primary path — no `NSDraggingSession`) +- Create: `crates/macos/src/actions/deliver_files_registry.rs` (per-app URL scheme table for Tier 1) +- Create: `crates/macos/src/actions/force_click.rs` (`kCGMouseEventPressure = 1.0` + `kCGEventMouseSubtypeTabletPoint` on `CGEventCreateMouseEvent`) +- Create: `crates/windows/src/actions/deliver_files.rs` (URI/shell delivery, `IFileOperation`, `CF_HDROP` clipboard paste, and policy-gated OLE fallback after spike) +- Create: `crates/windows/src/actions/force_click.rs` (`SendInput` with `PEN_FLAGS_BARREL` pen input flags) +- Modify: `crates/macos/src/actions/dispatch.rs` + `crates/windows/src/actions/dispatch.rs` (add arms) +- Modify: `crates/linux/src/adapter.rs` (ensure `execute_action` returns `ActionNotSupported` for `ForceClick` with documented-platform-divergence message — this is the stub; real implementation is Phase 3) +- Modify: `crates/core/src/commands/` (commands for `deliver-files`, `force-click`) +- Modify: `src/cli.rs`, `src/cli_args.rs`, `src/dispatch.rs` +- Test: `crates/macos/tests/it_deliver_files.rs` (2 files delivered to a Finder folder via Tier 2; verify files appear; verify NO focus steal on agent-desktop process) +- Test: `crates/macos/tests/it_force_click.rs` (force-click a word in TextEdit; verify the definition popover appears via `list-surfaces`) +- Test: `crates/windows/tests/it_deliver_files.rs` (files delivered to an Explorer folder via `IFileOperation`; verify files appear; verify NO focus steal; clipboard paste path restores clipboard; OLE fallback tests are gated behind explicit policy/spike fixtures) +- Test: `crates/windows/tests/it_force_click.rs` (pen input into a test app; verify pressure was delivered) + +**Approach:** +- macOS `ForceClick`: `CGEventCreateMouseEvent(..., kCGEventMouseDown, kCGMouseButtonLeft, ...)` → `CGEventSetIntegerValueField(event, kCGMouseEventPressure, 1)` → `CGEventSetIntegerValueField(event, kCGMouseEventSubtype, kCGEventMouseSubtypeTabletPoint)` → post; then `MouseUp`. +- macOS `DeliverFiles`: see the **4-tier headless fallback strategy** above (URL scheme → `NSWorkspace.open` with `activates: false` → pasteboard + `Cmd-V` → AppleScript). `NSDraggingSession` / `NSFilePromiseProvider` are explicitly rejected — they require foreground activation + `NSApp.run` event loop, which violate the Headless-First Invariant. +- Windows `ForceClick`: `SendInput` with `INPUT_PEN` structures; set `PEN_FLAGS_BARREL` and `pressure = 1024` (max). +- Windows `DeliverFiles`: implement the semantic tiers in order: URI/shell handoff, `IFileOperation` for Explorer/filesystem destinations, `CF_HDROP` clipboard paste with save/restore, and only then policy-gated `IDataObject + DoDragDrop` for target classes whose integration spike proves no unintended focus/cursor side effects. + +**Execution note:** `DeliverFiles` macOS Tier-2 and Windows Tier-2/Tier-3 spikes are mandatory. OLE drag is treated as a fallback requiring proof, not assumed safe by architecture. + +**Patterns to follow:** +- `crates/macos/src/actions/extras.rs` as the template for action extras that bypass the standard AX dispatch. +- Learning 1 — `ActionDispatchConfig` so platform-specific quirks don't leak into the core trait. + +**Test scenarios:** +- **Happy path (macOS):** `deliver-files @e<folder_ref> /tmp/a.txt /tmp/b.txt` produces a.txt and b.txt inside the folder (verified via `ls`); no focus change observed on active frontmost app. +- **Happy path (Windows):** same against Explorer. +- **Happy path (macOS):** `force-click @e<word_ref>` in TextEdit → definition popover appears in `list-surfaces`. +- **Edge case:** `deliver-files` with non-existent source path → `InvalidArgs` with the missing path in the message. +- **Edge case:** `force-click` on an element that doesn't respond to pressure (e.g. a regular button) → action succeeds but no side-effect; document this as expected behavior. +- **Edge case:** Linux `force-click` → `ActionNotSupported` with platform-divergence note in `platform_detail`. +- **Error path:** `deliver-files` with 0 paths → `InvalidArgs`. +- **Error path:** destination element not a drop target → `ActionFailed` with suggestion. + +**Verification:** +- Both platform CI integration tests green. Linux stub returns `ActionNotSupported` cleanly. + +--- + +### - [ ] Unit 13: Windows CI + release pipeline (x86_64 + aarch64) + +**Goal:** `windows-latest` runs build, clippy, unit, contract, and non-interactive tests on every PR; interactive UIA/shell tests run on a labeled Windows desktop runner; Windows CLI binaries (x86_64, aarch64) attached to Phase 2 release; npm `postinstall` picks up Windows branches. P2-O6, P2-O7. + +**Requirements:** R6, R7 + +**Dependencies:** None hard (runs parallel to U3…U12); must land **before** Unit 3's integration tests can validate on a Windows runner. In practice: a draft PR for this unit opens alongside Unit 3. + +**Files:** +- Modify: `.github/workflows/ci.yml` (add `test-windows` job: `runs-on: windows-latest`; mirrors the `test` job — clippy, `cargo test --lib --workspace`, `cargo build --profile ci`, `cargo build --profile release-ffi -p agent-desktop-ffi`, `cargo tree` isolation check) +- Modify: `.github/workflows/ci.yml` (add optional `test-windows-interactive` job guarded by label/runner availability: `runs-on: [self-hosted, windows, desktop, interactive]`; runs UIA/shell integration tests for Explorer, Start, taskbar, Action Center, Quick Settings, Notepad, and screenshot WGC. Hosted `windows-latest` must not be required to expose an unlocked Explorer desktop.) +- Modify: `.github/workflows/ci.yml` (the existing `test` job stays on `macos-latest`; add a matrix if sharing logic becomes tidy) +- Modify: `.github/workflows/release.yml` (CLI `build` matrix gains `x86_64-pc-windows-msvc` + `aarch64-pc-windows-msvc` on `windows-latest`. **aarch64 archive suffixed `-experimental`** (review-refined — shipping a build-only artifact with no test coverage deserves explicit labeling): `agent-desktop-v0.2.0-aarch64-pc-windows-msvc-experimental.zip`. FFI matrix gains `aarch64-pc-windows-msvc` row (same suffix). Archive format `zip` for Windows matches existing FFI Windows row. **MSVC ARM64 toolchain setup step added** (review-refined — cross-compile from windows-latest x64 host requires the MSVC ARM64 build tools component, not present in the default Desktop C++ workload): step invokes `Install-Module VSSetup` / `vs_installer modify` to add `Microsoft.VisualStudio.Component.VC.Tools.ARM64` before `cargo build --target aarch64-pc-windows-msvc`, then sources `vcvarsall.bat arm64` via `VsDevCmd.bat`.) +- Modify: `rust-toolchain.toml` (`targets` list gains `x86_64-pc-windows-msvc`; `aarch64-pc-windows-msvc` stays opt-in via `rustup target add` in the workflow step) +- Modify: `npm/scripts/postinstall.js` (add `win32-x64` branch selecting `agent-desktop-v<version>-x86_64-pc-windows-msvc.zip`. **aarch64 Windows falls back to x86_64 auto-selection** (review-refined — arm64 archive is flagged experimental; npm does not auto-select it until a runner validates). Explicit opt-in via `AGENT_DESKTOP_PREFER_ARM64=1` env var for users who want to test the experimental build. Decompress via native node `yauzl` (smaller footprint than `adm-zip`, no optionalDeps).) +- Modify: `npm/package.json` (optional: declare `os`/`cpu` matrix includes win32) +- Modify: `npm/bin/agent-desktop` (wrapper picks `.exe` on Windows) +- Test: `.github/workflows/ci.yml` — Windows CI green on the reference branch before Unit 3 lands. +- Test: `npm/scripts/postinstall.test.js` (mock win32 + arm64 environments; assert correct archive name selected) + +**Approach:** +- Clone the existing `test` job into a new `test-windows` job targeting `windows-latest` for compile/unit/contract coverage. Add a separate interactive desktop job for UIA/shell tests; it is required before marking Windows adapter GA but is not assumed to be available on GitHub-hosted runners. Skip binary-size check on Windows initially (MSVC toolchain + dynamic CRT produces slightly larger binaries; either raise the cap to 20 MB on Windows or skip — recommend skip with a TODO for Phase 5 binary-size work). +- The ARM64 Windows CLI binary is **build-only** for now. When GitHub Actions promotes the ARM runner to GA, a follow-up PR wires testing; the release job publishes the binary regardless. +- `npm/scripts/postinstall.js` mirrors existing macOS logic; adds `win32` branches. + +**Patterns to follow:** +- Existing `release.yml` FFI matrix already includes `x86_64-pc-windows-msvc` — copy its structure for the CLI matrix row. +- Existing `ci.yml` `test` job shape. + +**Test scenarios:** +- **Happy path:** PR with a trivial `crates/core/src/` change triggers `test-windows` and it passes. +- **Happy path:** PR with Windows shell changes triggers `test-windows-interactive` when the label/runner is available; otherwise the PR must include mocked/unit coverage and a manual evidence artifact before merge. +- **Happy path:** release-please creating a v0.2.0 tag publishes x86_64 + aarch64 Windows CLI `.zip` archives alongside existing macOS and Linux artifacts; checksum file includes them. +- **Happy path:** `npm install @lahfir/agent-desktop` on a Windows x64 machine downloads the right archive and the binary runs. +- **Integration:** `cargo tree -p agent-desktop-core` isolation check runs on Windows and passes. +- **Edge case:** Windows-specific test failure surfaces clearly in the PR check list. +- **Error path:** missing `windows-latest` runner minutes → workflow fails fast with a non-retriable error (document runbook in Unit 14 docs). + +**Verification:** +- Windows CI job green on a reference PR. +- Release workflow dry-run against a test tag produces the expected artifact set. + +--- + +### - [ ] Unit 14: Documentation + skills + `phases.md` sync + +**Goal:** Ship Phase 2 docs. `skills/agent-desktop-windows/SKILL.md` created; core skill updated to three-platform; `skills/agent-desktop-ffi/` reflects `ad_abi_version` + `ad_set_log_callback`; README platform table + Windows install + permissions; `phases.md` stale-reference cleanup. Skill Maintenance Addendum compliance. Final unit; runs after U1..U13 are merged. + +**Requirements:** Skill Maintenance Addendum; release-readiness + +**Dependencies:** U1..U13 (all implementation complete) + +**Files:** +- Create: `skills/agent-desktop-windows/SKILL.md` (sibling to macOS content) +- Create: `skills/agent-desktop-windows/references/uia.md` (UIA control types, patterns, pattern-first dispatch order) +- Create: `skills/agent-desktop-windows/references/windows-permissions.md` (UAC, UIA access, integrity-level mismatches, WGC/session support, Focus Assist / notification listener permission, and unsupported locked/secure desktop behavior) +- Create: `skills/agent-desktop-windows/references/windows-shell-surfaces.md` (Start menu/search, taskbar, system tray/overflow, Action Center, Quick Settings, virtual desktop detection, mixed-DPI coordinate caveats, and when to use `open-system-surface`) +- Create: `skills/agent-desktop-windows/references/chromium.md` (Windows-flavored version of `electron-compat.md` + `--force-electron-a11y`) +- Modify: `skills/agent-desktop/SKILL.md` (three-platform command surface; link to macOS + Windows sibling skills; note tray + notifications on Windows) +- Modify: `skills/agent-desktop/references/commands-*.md` (add `watch`, `text *`, `open-system-surface`, `list-tray-items`, `click-tray-item`, `open-tray-menu`, `long-press`, `show-menu`, `window-raise`, `cancel`, `force-click`, `deliver-files`; note new flags `--force-electron-a11y`, `--screenshot-backend`, `--event`, `--request-permission`) +- Modify: `skills/agent-desktop-ffi/SKILL.md` (document `ad_abi_version()`, `ad_set_log_callback`, registry-driven wrapper surface, `AD_ABI_VERSION_MAJOR`) +- Modify: `skills/agent-desktop-ffi/references/ownership.md` + `references/threading.md` (note worker-thread use in `watch_element`; `ad_set_log_callback` lifetime) +- Create: `crates/ffi/README.md` (pre-1.0 FFI policy statement per KD17) +- Modify: `README.md` (platform support matrix; Windows install; permission pre-flight; link to Windows skill) +- Modify: `docs/phases.md` (strike "Gap Analysis — 2026-04-17 Research at the bottom" references; fix stale Windows dependency pins to `0.62.2`; remove `Watch(WatchSpec)` from P2-O9 enumeration; update shipped command count; add Phase 2 "Shipped" status block once v0.2.0 tags) +- Modify: `CHANGELOG.md` (release-please will generate, but add a human-curated summary of v0.2.0 breaking changes: `ErrorCode` now `#[non_exhaustive]`, `PermissionReport` tri-state, `ad_abi_version` exported + policy) +- Create: `docs/migrations/0.1-to-0.2.md` (consumer migration guide: JSON schema changes, FFI ABI changes, new error codes) +- Create: `docs/solutions/best-practices/electron-compat-cross-platform-2026-04-18.md` (document-review refinement — moved here from Unit 4; ports private-memory `electron-compat.md` so Windows + future Linux contributors see the depth-skip rules) +- Keep backward-compat shim: `scripts/update-ffi-header.sh` (thin wrapper) alongside the new `scripts/update-ffi.sh` so existing references, skills, and contributor muscle memory don't break (document-review refinement) +- Modify: `skills-lock.json` (register `agent-desktop-windows`) + +**Approach:** +- Use `/skill-creator` for every SKILL.md touched (per MEMORY.md Skill Maintenance rule). +- `phases.md` surgical edits — do not rewrite narrative, just fix the stale claims and add a "Shipped 2026-MM-DD" line to Phase 2 §Status. +- Consumer migration guide covers: `ErrorCode` additions (consumers parsing SCREAMING_SNAKE_CASE must accept unknown codes since `#[non_exhaustive]`), `PermissionReport` JSON shape change, any renames. + +**Patterns to follow:** +- Existing `skills/agent-desktop/SKILL.md` structure and references layout. +- Existing release-please-generated CHANGELOG conventions. + +**Test scenarios:** +- **Test expectation: none** — documentation-only unit; validated via manual review and by running `skills/` publish dry-run (`clawhub sync --root skills/ --dry-run`). +- **Verification gates (not tests):** + - `scripts/update-ffi-header.sh` shows no drift. + - `skills-lock.json` references the new Windows skill. + - `docs/phases.md` grep for `"Gap Analysis — 2026-04-17 Research at the bottom"` returns zero matches. + - `docs/phases.md` grep for `windows = "0.58"` returns zero matches. + - `README.md` platform table includes Windows x64 + arm64. + +**Verification:** +- Manual review of all skill files and README on the release PR. +- CI green; skills publish step dry-run succeeds. + +--- + +## System-Wide Impact + +- **Interaction graph:** `watch_element` introduces worker threads inside the adapter — breaks the single-threaded Phase 1 assumption. All shared state between callbacks and the main thread goes through `mpsc` channels (no shared mutable state). **Deepening-pass decision:** the `watch_element` trait signature takes `&RefEntry` (not `&NativeHandle`), so the worker re-resolves the element on its own thread and the `NativeHandle` `PhantomData<*const ()>` `!Send`/`!Sync` invariant is preserved intact. The existing `unsafe impl Send for NativeHandle` / `unsafe impl Sync for NativeHandle` at `crates/core/src/adapter.rs:93-94` (Phase 1 single-threaded justification) does NOT need reassessment in Phase 2. +- **Error propagation:** `ErrorCode` gains 4 new variants (`#[non_exhaustive]` makes this forward-compatible for consumers that match exhaustively and accept a default arm). Consumers of the FFI `AdResult` enum see 4 new discriminants; the `const _: () = assert!(…)` parity gate keeps them in lockstep. New errors (`AutomationPermissionDenied`, Screen-Recording-specific `PermDenied`) flow through existing `AdapterError::with_suggestion` / `with_platform_detail` helpers. +- **State lifecycle risks:** `DeliverFiles` macOS uses `NSWorkspace.open(activates: false)` (no delegate lifetime to manage — async completion via closure, `Retained<T>` wrapper for any callback handles). On Windows, `IFileOperation`, clipboard save/restore, and optional `IDataObject` fallback each need explicit lifetime ownership; `IDataObject` must span `DoDragDrop` only inside the policy-gated fallback. `watch_element` on macOS uses main-thread `CFRunLoopRunInMode` (no worker thread lifecycle); on Windows the worker thread is hard-joined inside `watch_element` (no leak). `ad_set_log_callback` installs a global `OnceCell`; removing the callback is not supported (second call errors). +- **API surface parity:** Every new trait method ships with a `not_supported()` default implementation so Linux (Phase 3) compiles without changes. The Linux adapter explicitly keeps `ForceClick` returning `ActionNotSupported` as permanent divergence. +- **Integration coverage:** Cross-platform JSON identity fixture (`tests/fixtures/calculator.*.json`, `tests/fixtures/vscode.*.json`) ensures structural parity is checked, not just individual unit behavior. `watch_element` + `ad_set_log_callback` + `ad_abi_version` each require their own FFI integration harness. +- **Unchanged invariants:** + - Ref format `@e{n}` and refmap file location at `~/.agent-desktop/last_refmap.json` (0o600). + - JSON envelope shape (`{version, ok, command, data, error}`). + - Binary size limit 15 MB for the CLI (Windows CI may skip pending Phase 5 size work; documented explicitly). + - Core isolation — `cargo tree -p agent-desktop-core` still free of platform crates (CI checks on macOS AND Windows). + - Single-command-per-file + 400-LOC rule (Unit 5 splits `element.rs` because it's at the cap). + - No `unwrap()` in non-test code; zero-warning clippy. + +## Risks & Dependencies + +| ID | Risk | Mitigation | Owner Unit | +|----|------|------------|------------| +| R1 | Calendar slip — 15 units is a large PR count | Parallelize where dependency graph allows; each unit is independently reviewable; honest calendar-months sizing up front | — | +| R2 | `watch_element` architecture unproven (AXObserver off main thread, UIA handler lifetime) | Unit 7 opens with a named spike against three apps per platform; spike outcome gates the rest of U7 | U7 | +| R3 | `windows-capture` newer majors may move the API surface | Pin `windows-capture = "=1.5.4"`; verify API compiles against 1.5.4 before merging U9 spike PR; re-evaluate newer majors post-Phase 2 | U9 | +| R4 | Registry migration (U2) affects every existing command — high blast radius | Migrate one command category at a time (observation → interaction → system → clipboard → notifications → batch); each sub-PR keeps `cargo test --workspace` green | U2 | +| R5 | MSRV bump to 1.82 may break downstream consumers on pinned toolchains | Release notes flag MSRV; pre-1.0 explicitly unstable; acceptable | U1 | +| R6 | `objc2` introduction spreads beyond screenshot/permissions | CI grep guard: `rg "objc2::" crates/macos/src/ \| grep -v "system/(screenshot\|permissions).rs" && exit 1` (landed in U9) | U9 | +| R7 | FFI ABI churn across 17 objectives needs one coordinated `ad_abi_version()` bump | Unit 1 exports `ad_abi_version()` as **sub-PR 1a** with anchor `MAJOR = 1`; the first struct-layout-breaking sub-PR (1g — `PermissionReport` tri-state) atomically bumps to `MAJOR = 2`; every subsequent FFI-affecting change in U1–U12 asserts via CI grep that the version has incremented when its own changes touch `crates/ffi/src/error.rs` or `crates/ffi/src/generated/` | U1 | +| R8 | Windows runner flakiness against real apps | Split Windows CI: `test-windows-unit` (blocks merge) + `test-windows-ui` (nightly + pre-release); Unit 13 implements the split when GitHub Actions runner minutes warrant | U13 | +| R9 | `ForceClick` on Linux has no native path (Phase 3 gap) | Documented; Linux adapter returns `ActionNotSupported` — legitimate per-platform divergence, not a bug | U12 | +| R10 | Cross-compile-from-macOS iteration cost for Windows-heavy phase | Windows CI runs on every PR touching `crates/windows/` or cross-platform trait methods; if iteration cost exceeds 2000 CI minutes/PR-cycle, spin up a local Windows VM | U3 / U13 | +| R11 | `AccessibilityNode` field explosion — nested `StableSelectors` preserves wire shape | Planning picked `#[serde(flatten)]` on an inline `StableSelectors` with per-field `skip_serializing_if` (see §Open Questions §Resolved); JSON wire shape preserved | U1 | +| R12 | `phases.md` references a "Gap Analysis — 2026-04-17 Research" section that doesn't exist | Fixed in U14 alongside other phases.md syncs | U14 | +| R13 | Codegen mechanism for FFI wrappers | **Resolved by research** — Unit 2 uses `build.rs` filesystem enumeration of `crates/core/src/commands/*.rs` (deterministic, cdylib-safe, zero linker magic). `inventory`/`linkme`/`xtask` all rejected per Research Topic B. Spike PR still opens Unit 2 to validate codegen on `click` alone before migrating 52 more. | U2 | +| R14 | Electron rules currently live in private memory (`electron-compat.md`) | Unit 4 ports those rules to `docs/solutions/best-practices/electron-compat-cross-platform-2026-04-18.md` as the first task | U4 | +| R15 | `SendInput` on Windows doesn't honor `blocked_combos` by itself — the safety check is ours | Unit 3's input/keyboard.rs calls `is_blocked(combo, adapter.blocked_combos())` before dispatching; test covers this | U3 | +| R16 | `UIA_HtmlIdProperty` / `UIA_HtmlClassProperty` only work on WebView2 elements | Unit 5 returns empty Vec for non-WebView2 nodes; documented in selectors reference | U5 | +| R17 | **FFI wrapper rename gap** — renaming a command without updating its filename or the `descriptor()` function could slip past wrapper drift check | `crates/core/tests/cli_registry_parity.rs` cross-checks that every clap subcommand name has a matching `CommandDescriptor` and vice versa; `build.rs` filesystem enumeration makes drift impossible because the file listing IS the source of truth | U2 | +| R18 | **400-LOC breaches compound in Phase 2** — `actions/chain_steps.rs` already at 407 L today; `actions/dispatch.rs` at 349 L will breach adding 4 arms in U6 | Unit 6 opens with a behavior-preserving refactor sub-PR that pre-splits both files; documented in U6 pre-work | U6 | +| R19 | **`NativeHandle` `!Send`/`!Sync` invariant vs Unit 7 worker thread** | Deepening pass resolves by passing `&RefEntry` (not `&NativeHandle`) to `watch_element`; worker re-resolves on its own thread, preserving the invariant | U1 / U7 | +| R20 | **Unit 7 callback panic → hung worker thread leak** (critical for long-running FFI consumers) | U7 spike validates `catch_unwind` wrapper around AX/UIA callback; hard-join timeout in `watch_element` itself (`2 × timeout_ms`) with `Internal` error on refuse-to-exit | U7 | +| R21 | **`AD_ABI_VERSION_MAJOR` bump ordering race** — sub-PR 1g is the only actual ABI-breaking sub-PR in Unit 1 (struct-layout change on `PermissionReport`); 1b adds `#[non_exhaustive]` without variant changes (non-breaking) and 1c adds variants under `#[non_exhaustive]` (breaking for C consumers per adversarial review — see P0 findings) | Document-review refinement: `ad_abi_version()` ships at sub-PR 1a as anchor `MAJOR = 1`; sub-PR 1g atomically bumps to `MAJOR = 2` with the layout change; CI grep asserts the bump happens on every sub-PR that touches layout files | U1 | + +## Alternative Approaches Considered + +1. **Sub-phase split (2a/2b).** Rejected in brainstorm §D1 before this plan opened. Accumulates unstated deferrals; the 15-unit PR decomposition managed via this plan is lower-risk than a sub-release cut. +2. **`Option<StableSelectors>` with `#[serde(flatten)]` wrapper.** Rejected during planning (§Open Questions §Resolved). The inline struct with per-field `skip_serializing_if` preserves JSON wire shape with no behavioral change; the Option wrapper adds a layer of indirection that doesn't buy anything. +3. **Persistent modifier-state file for key-down/key-up.** Rejected. Adds state to a stateless CLI; the whole-combo safety check in `blocked_combos()` is sufficient because every blocked combo includes at least one non-modifier key. +4. **Shared bounded worker-thread pool for U7 and U9.** Rejected. Different lifetimes (observer loops vs single-shot captures); different threading models (MTA vs no specific apartment). Revisit with Phase 4 daemon. +5. **`windows 0.58` as originally named in the early draft.** Rejected. Patched to `windows 0.62.2` in origin §D15 because `windows-capture 1.5.x` requires 0.62+ and the workspace pins the current compatible release. +6. **`inventory` OR `linkme` for the command registry.** BOTH rejected (research-driven). Research Topic B found neither survives link-GC reliably across ld64, ld-prime, GNU ld, lld, MSVC link.exe for cdylib consumers. Replaced with `build.rs` filesystem enumeration — deterministic, no linker magic. +7. **Auto-commit regenerated FFI wrappers from `build.rs` back into `crates/ffi/src/generated/`.** Rejected per learning 3 — auto-heal masks CI drift. The committed copy is the ABI contract; `cargo xtask gen-ffi` refreshes on demand. +7a. **Codegen via `build.rs` macro expansion inside `crates/ffi/src/lib.rs`.** Rejected (deepening pass) — has no committed artifact to drift-check against, which violates learning 3 and means rename-detection / shape-drift caught only at runtime. `xtask` path (chosen) keeps the committed file as the ABI contract. +7b. **Codegen via a proc-macro crate that expands `deterministic registry iteration` at the `ffi` crate's top level.** Rejected for the same reason as 7a — no committed artifact. +8. **`screencapturekit 0.3` from the early draft.** Rejected for `1.5` per origin §D15 — the 0.3 series is older and lacks `SCScreenshotManager` stable shape. +9. **Widen `WindowInfo.pid` to `i64` for Windows DWORD safety.** Rejected per KD3 — breaks JSON and FFI ABI for zero practical benefit; narrow-at-boundary is adequate. + +## Phased Delivery + +Calendar-month ceilings (hard, not commitments — if breached, invoke the Deferral Order below). + +### Phase 0 — v0.1.14 prep release (review-added, ~1 week) + +Before Phase 2 proper opens, ship a minimal non-breaking release that gives downstream consumers FFI-stability primitives without a major bump. Scope: +- `#[non_exhaustive]` on `ErrorCode` (no variant additions yet) +- `ad_abi_version()` exported returning `AD_ABI_VERSION_MAJOR = 1` (anchor Phase 1 value) +- `ad_init(expected_major: u32) -> AdResult` enforced version-negotiation handshake +- `AD_RESULT_UNKNOWN = -99` sentinel exported +- `crates/ffi/README.md` documenting the pre-1.0 FFI policy (KD17) + +Consumers adopt these and update their integration tests before v0.2.0 breaks the PermissionReport layout. Splits the 4-way break (MSRV + non_exhaustive + tri-state + ABI bump) into 2+2, giving adopters breathing room. + +**Gate:** v0.1.14 tag pushed; FFI consumers on the main integration list (if any) have adopted `ad_init()`. + +### Phase A — Foundation (~4 weeks) — Unit 1 + Unit 2 + Unit 2.5 + Unit 2.6 +U1 lands across 10 sub-PRs (1a–1j). U2 (registry migration, single-concern) across 6 category sub-PRs. U2.5 (`ad_set_log_callback` with redaction) and U2.6 (Phase 1.5 FFI backfill) land as small follow-ups. U7's opening spike PR lands between sub-PRs 1g and 1h (so trait signatures are informed by spike outcome). **Gate:** `cargo test --workspace` green on macOS; `ad_init` handshake working; parity const assertion intact; registry count matches clap command enumeration under all CI profiles. + +### Phase B — Windows foundation (~5 weeks) — Unit 3 + Unit 3a + Unit 3b; Unit 13 in parallel +U3 implements the Windows adapter across 4 sub-PRs (tree / actions+input / clipboard+app+window / permissions+screenshot-legacy). U3a and U3b follow. U13 opens in parallel to keep Windows CI running against U3 draft branches. **Gate:** Windows CI green; cross-platform JSON fixture test (R2) meets the review-refined metric (jaccard ≥ 0.85, identifier equality, ±15% ref count). + +### Phase C — Feature parity (~6 weeks) — Units 4–12, parallelizable +Units 4, 5, 6, 7, 8, 9, 10, 11, 12 all depend only on U1, U2, U3. They open in parallel worktrees. + +**Merge order (review-refined — value-dense first, not risk-cheapest first):** +1. **U5** (stable-selector fields) — R8 directly benefits agent loops +2. **U7** (watch_element) — R11 replaces 100ms polling with sub-500ms push +3. **U4** (Windows Electron compat) — R15 unlocks VS Code / Slack / Cursor +4. **U8** (text range primitives) — R12 enables in-place editing flows +5. **U6** (action variants LongPress/ShowMenu/WindowRaise/Cancel) — completes the Action surface +6. **U11** (permission tri-state on macOS) — R17 unblocks Modern screenshot +7. **U9** (modern screenshot) — R13, depends on U11 for Screen Recording probe +8. **U10** (new surfaces) — R14, polish +9. **U12** (DeliverFiles + ForceClick) — remaining Action variants, highest-risk spike + +**Gate:** every P2-O* metric green in CI integration tests. + +### Phase D — Release (~1 week) — Unit 14 +U14 updates docs, strikes stale phases.md references, creates the Windows skill, publishes the FFI pre-1.0 policy, ports electron-compat to docs/solutions. release-please cuts v0.2.0. **Gate:** tag pushed; GitHub Release artifacts include all Windows + macOS CLI and FFI archives (aarch64 Windows marked `-experimental`); npm publish succeeds. + +### Total ceiling: **16 weeks** + +### Deferral Order (if ceiling breached) + +Defer last-to-first in this order (lowest-impact first): +1. **U10** (new surfaces: Spotlight / Dock / MenuBarExtras / Windows shell surfaces) → v0.2.1 +2. **U3b** (Windows tray commands) → v0.2.1 +3. **U12** (DeliverFiles + ForceClick) → v0.2.1 or v0.3.0 (DeliverFiles spikes decide) +4. **U4** (Electron compat on Windows) → v0.2.1 — but prefer keeping, since this blocks VS Code / Slack adoption on Windows + +R8, R11, R12, R13, R16, R17 are **never deferred** — they are the core agent-value wins of Phase 2. + +## Success Metrics + +- **R8 metric (review-refined):** Primary — real-app integration: 10-step Slack sidebar navigation on macOS and Windows completes without `STALE_REF`-induced re-snapshot in ≥80% of runs (10 runs minimum). Secondary — `identifier` field populated on ≥70% of interactive nodes across VS Code + Slack + Calculator fixture set. The earlier "+20pp on curated fixture" is retained as a diagnostic, not a gate. +- **R11 metric:** `watch --event value-changed` returns within 500 ms on both platforms. +- **R13 metric:** `screenshot --app Finder` (Modern) cold latency <50 ms median over 10 runs on both platforms. +- **R15 metric:** VS Code snapshot with `--force-electron-a11y` exposes ≥100 refs at default depth on both platforms. +- **R16 metric:** Adding a new command (`hello-world` test fixture) requires only creating one file under `crates/core/src/commands/`; CLI, FFI wrappers, and schemas auto-register; asserted by an integration test in Unit 2. +- **FFI ABI stability:** `ad_abi_version()` returns `AD_ABI_VERSION_MAJOR = 2` from Phase 2 onward; `crates/ffi/README.md` documents the bump policy. +- **Binary size:** macOS CLI release binaries remain under 15 MB; Windows CLI initially unbounded (size check skipped on Windows in Unit 13 with TODO). + +## Documentation Plan + +- `skills/agent-desktop-windows/SKILL.md` — created in Unit 14 with `/skill-creator`. +- `skills/agent-desktop/SKILL.md` — updated for three-platform support in Unit 14. +- `skills/agent-desktop-ffi/SKILL.md` — `ad_abi_version` + `ad_set_log_callback` noted in Unit 14. +- `crates/ffi/README.md` — new file in Unit 2 / Unit 14 with the pre-1.0 FFI ABI policy. +- `docs/migrations/0.1-to-0.2.md` — created in Unit 14; consumer migration guide. +- `docs/phases.md` — stale-reference cleanup in Unit 14. +- `README.md` — three-platform install + permission pre-flight in Unit 14. +- `docs/solutions/best-practices/electron-compat-cross-platform-2026-04-18.md` — created in Unit 4 (ports private memory). +- `CHANGELOG.md` — human-curated v0.2.0 breaking-change summary in Unit 14 (release-please generates the base). +- `MEMORY.md` — update the user's project memory after v0.2.0 ships (Phase 2 status, Windows adapter, registry migration). + +## Operational / Rollout Notes + +- **Breaking release:** v0.2.0. Downstream FFI consumers MUST call `ad_abi_version()` at load time and compare against their expected major. Example in `skills/agent-desktop-ffi/references/build-and-link.md`. +- **Toolchain:** MSRV bumps to 1.82. Pre-commit hook and CI cache keys refresh automatically on first PR after the bump. +- **Permissions pre-flight:** a new README section explains Screen Recording TCC grant on macOS before running `screenshot`; Windows documents UIA integrity boundaries, WGC support checks, UAC/elevation mismatch, and shell-surface unsupported cases. +- **Rollback:** v0.1.x remains installable via `npm install @lahfir/agent-desktop@0.1`. ABI breakage means mixing 0.1 CLI with 0.2 FFI is undefined — migration guide says so. +- **Monitoring:** post-release, watch `STALE_REF` rate in field (user-reported) and compare against the Unit 5 baseline fixture. Phase 3 plan deepens with Linux AT-SPI coverage. +- **Follow-up PRs after v0.2.0 ships:** + - `windows-capture 2.0` upgrade evaluation (pending 2.x stability). + - GitHub Actions `windows-11-arm` GA → wire aarch64 Windows testing matrix (today: build-only). + - Binary size budget for Windows CLI (Phase 5 production-readiness work). + +## Swarm & Parallelization Strategy + +Phase 2 is 16 weeks single-threaded. Most of Phase C and large parts of Phase A can run concurrently via Claude Code agent teams (spawn via Agent tool with `isolation: "worktree"`) or parallel worktrees managed by the user. This section lists what can safely run in parallel and what cannot. + +### Hard serial dependencies (DO NOT parallelize) + +These must land in strict order — parallelizing them creates merge conflicts on shared types or breaks intermediate builds: + +| Dependency | Reason | +|---|---| +| v0.1.14 prep → Phase 2 start | Consumers need `ad_init()` available before ABI breaks | +| U1 sub-PRs 1a → 1b → 1c → 1d → 1e → 1f → 1g | Each sub-PR mutates `crates/core/src/error.rs` or `node.rs` in sequence; parallel edits conflict on the parity `const` assertion and serde derives | +| U7 spike → U1 sub-PR 1h → U7 implementation | Trait signature depends on spike outcome | +| U1 sub-PR 1g → U1 sub-PR 1i | `PermissionReport` layout change must precede the `AD_ABI_VERSION_MAJOR` bump test that reads it | +| U2 → U2.5, U2.6 | `ad_set_log_callback` and backfill are `CommandDescriptor` entries; registry must exist first | +| U2 → U3 | Windows adapter's new commands register via `deterministic registry metadata`; registry must exist | +| U3 → U3a, U3b, U4 | Sub-units extend the Windows adapter surface | +| U11 → U9 | Modern screenshot needs tri-state permission probe to distinguish Screen-Recording-denied from AX-denied | +| All units → U14 | Docs update reflects shipped behavior | + +### Parallelizable swarm fan-out points + +#### Swarm Point 1 — Phase A (sub-PR 1i onward) + +After Unit 1 sub-PR 1g has landed, these can fan out concurrently: + +| Worker | Work | Isolation | +|---|---|---| +| A | U1 sub-PR 1h (trait method stubs) | main branch, serial | +| B | U1 sub-PR 1i (`event.rs`, `text_range.rs`, `screenshot_backend.rs`, 4 config structs) | worktree | +| C | U1 sub-PR 1j (CLI arm reservations with `#[clap(hide = true)]`) | worktree | +| D | U7 spike PR (validation-only, `crates/macos/src/events/spike.rs` throwaway) | worktree, **blocks sub-PR 1h** | + +Sub-PRs 1h, 1i, 1j can land in any order once 1g is in main, because they touch disjoint files. + +#### Swarm Point 2 — Phase A (after U2 merges) + +U2.5 and U2.6 are small and independent. Fan out 2 workers: + +| Worker | Work | Estimated PR size | +|---|---|---| +| A | U2.5 (`ad_set_log_callback` + redaction layer) | ~400 LOC | +| B | U2.6 (Phase 1.5 FFI backfill: `ad_execute_by_ref` + descriptor confirms) | ~200 LOC | + +#### Swarm Point 3 — Phase B (Windows foundation) + +U3's 4 sub-PRs serialize internally (tree → actions → clipboard → permissions), but U13 (Windows CI + release pipeline) runs entirely in parallel against U3's draft branches: + +| Worker | Work | +|---|---| +| A | U3 tree sub-PR (UITreeWalker, roles map, element wrapper) | +| B | U3 actions sub-PR — **starts after A merges** | +| C | U13 sub-PRs (CI job, release.yml matrix, npm postinstall) — **parallel to A/B/C/D** | +| D | `docs/security/screencapturekit-fork-audit-2026-04-18.md` audit (can start anytime) | + +#### Swarm Point 4 — Phase C fan-out (highest parallelism) + +After U3 merges, 9 workers can land concurrently. This is the densest parallelization point. **Recommended team composition: 4–6 workers maximum** (not 9 — reviewer bandwidth is the bottleneck). + +| Worker | Recommended units | Isolation | Total LOC estimate | +|---|---|---|---| +| A | U5 (stable-selector population) | worktree | ~600 | +| B | U7 (watch_element — spike validated) | worktree | ~1200 | +| C | U4 (Windows Electron compat) | worktree | ~500 | +| D | U8 (text range primitives) | worktree | ~900 | +| E | U6 + U11 (action variants + macOS tri-state permissions) | worktree | ~700 | +| F | U9 (modern screenshot — after E's U11) + U10 (new surfaces) | worktree | ~1000 | + +Worker E bundles U6 + U11 because both are small and independent. Worker F starts U9 only after U11 merges (serial dependency per Phase C merge order). + +U12 (DeliverFiles + ForceClick) lands **after** F's U9 merges, with its own opening spikes. Treat U12 as single-worker. + +#### Swarm Point 5 — Phase D (Documentation) + +U14 fans out across skill references and docs: + +| Worker | Work | +|---|---| +| A | `skills/agent-desktop-windows/SKILL.md` + references | +| B | `docs/migrations/0.1-to-0.2.md` + CHANGELOG curation | +| C | `docs/phases.md` cleanup + `crates/ffi/README.md` policy doc | +| D | `docs/solutions/best-practices/electron-compat-cross-platform-2026-04-18.md` port | + +All four are independent; merge in any order. + +### Spawning workers with Claude Code `Agent` tool + +``` +Agent({ + description: "U5 stable-selector implementation", + subagent_type: "general-purpose", + isolation: "worktree", + prompt: "Implement Unit 5 of docs/plans/2026-04-18-001-feat-phase2-windows-crossplatform-plan.md — stable-selector field population on macOS and Windows. Read the full unit definition including Files, Approach, Patterns to follow, Test scenarios, Verification. Land each of the test scenarios as a test case. Keep crates/macos/src/tree/element.rs under 400 LOC (split into element.rs + element_selectors.rs per the Files list). Return when all tests pass and cargo clippy is clean." +}) +``` + +Spawn 4–6 in one message for true concurrent execution. Each worker operates in an isolated worktree so merge conflicts surface at PR review, not mid-write. + +### Swarm anti-patterns (avoid) + +- **Do not parallelize sub-PRs within U1.** They all touch `crates/core/src/{error,action,node,adapter,refs}.rs` in overlapping regions; parallel edits conflict on the parity assertion. +- **Do not fan out U3's 4 sub-PRs.** `crates/windows/src/adapter.rs` is the contended file; serialize. +- **Do not start U9 before U11.** Screen Recording permission probe is a dependency. +- **Do not start U12 before U9.** DeliverFiles depends on Modern screenshot's TCC story for shared `objc2` surface area; also U12's spikes need U9's dependency tree. +- **Do not assign two workers to the same unit.** Units are atomic scope; splitting within a unit requires manual coordination that defeats the swarm. + +--- + +## Progressive Commit Checklist + +Every box below maps to **one PR-sized commit**. Dependency order is top-to-bottom within each phase. Where `[parallel with: …]` appears, spawn a swarm worker. + +### Phase 0 — v0.1.14 prep + +- [ ] C0.1: add `#[non_exhaustive]` to `ErrorCode` (no variant additions) — `feat(ffi): prepare error enum for non-exhaustive evolution` +- [ ] C0.2: export `ad_abi_version()` returning 1, add `AD_RESULT_UNKNOWN = -99`, add `ad_init(expected_major)` handshake — `feat(ffi): add abi version + init handshake` +- [ ] C0.3: publish `crates/ffi/README.md` FFI policy — `docs(ffi): document pre-1.0 abi policy` +- [ ] C0.4: release v0.1.14 — `chore: release 0.1.14` + +### Phase A — Foundation (Unit 1 + U2 + U2.5 + U2.6) + +- [ ] C1.1 (sub-PR 1a): abi_version.rs + ad_init + AD_RESULT_UNKNOWN sentinel (if not in 0.1.14) — serial +- [ ] C1.2 (sub-PR 1b): `#[non_exhaustive]` on `ErrorCode` (if not in 0.1.14) — serial +- [ ] C1.3 (sub-PR 1c): 4 new `ErrorCode` variants + matching `AdResult::Err*` — serial +- [ ] C1.4 (sub-PR 1d): 8 new `Action` variants — serial +- [ ] C1.5 (sub-PR 1e): `StableSelectors` + `#[serde(flatten)]` on `AccessibilityNode` — serial +- [ ] C1.6 (sub-PR 1f): `RefEntry.identifier: Option<String>` — serial +- [ ] C1.7 (sub-PR 1g): `PermissionReport` tri-state + `AD_ABI_VERSION_MAJOR` bumps to 2 atomically — serial, **gates downstream** +- [ ] C1.8 (U7 spike): validate `AXObserver` non-main-thread behavior — `feat(spike): validate ax observer threading` [parallel with: C1.9, C1.10] +- [ ] C1.9 (sub-PR 1h): trait method stubs (signature informed by C1.8) — parallel with C1.10 +- [ ] C1.10 (sub-PR 1i): supporting types + 4 shared configs — parallel with C1.9 +- [ ] C1.11 (sub-PR 1j): reserve CLI arms with `#[clap(hide = true)]` — parallel with C1.9, C1.10 +- [ ] C2.1: U2 registry migration spike on `click` command only — `feat(ffi): registry migration spike (click)` [gates C2.2–C2.7] +- [ ] C2.2: U2 observation category migration — `refactor(ffi): migrate observation commands to registry` +- [ ] C2.3: U2 interaction category migration — `refactor(ffi): migrate interaction commands` +- [ ] C2.4: U2 system category migration — `refactor(ffi): migrate system commands` +- [ ] C2.5: U2 clipboard category migration — `refactor(ffi): migrate clipboard commands` +- [ ] C2.6: U2 notifications category migration — `refactor(ffi): migrate notification commands` +- [ ] C2.7: U2 batch migration + xtask + drift check + link-GC mitigation — `refactor(ffi): complete registry migration with xtask` +- [ ] C2.5.1: U2.5 log_callback + redaction layer — `feat(ffi): ad_set_log_callback with redaction` [parallel with C2.6.1] +- [ ] C2.6.1: U2.6 Phase 1.5 backfill (execute_by_ref + descriptor confirms) — `feat(ffi): backfill phase 1.5 wrappers` [parallel with C2.5.1] + +### Phase B — Windows foundation (U3 + U3a + U3b; U13 parallel) + +- [ ] C3.1: U3 tree — `feat(windows): uia tree walker + role map` +- [ ] C3.2: U3 actions — `feat(windows): uia action dispatch with smart chain` [after C3.1] +- [ ] C3.3: U3 input — `feat(windows): sendinput keyboard/mouse + win32 clipboard` [after C3.1] +- [ ] C3.4: U3 system — `feat(windows): app lifecycle + window ops + legacy screenshot` [after C3.1] +- [ ] C3.5: U3a Windows notifications — `feat(windows): toast/action-center notifications` [after C3.2] +- [ ] C3.6: U3b Windows shell surfaces + tray — `feat(windows): shell surfaces and system tray commands` [after C3.2, parallel with C3.5] +- [ ] C13.1: U13 Windows CI job — `ci: add windows-latest test job` [parallel with C3.*] +- [ ] C13.2: U13 release matrix + postinstall — `ci: release pipeline for x86_64 + aarch64 windows` [parallel with C3.*] +- [ ] C13.3: U13 MSVC arm64 toolchain setup — `ci: install arm64 msvc build tools` [after C13.2] + +### Phase C — Feature parity (U4–U12, heavily parallel) + +**Swarm fan-out at this point — spawn 4–6 workers.** + +- [ ] C5: U5 stable-selector population — `feat(tree): populate stable selector fields` [parallel with C7, C4, C8, C6, C11] +- [ ] C7: U7 watch_element — `feat(events): watch_element with push notifications` [parallel with C5, C4, C8, C6, C11] +- [ ] C4: U4 Windows Electron compat — `feat(windows): electron/webview2 depth-skip` [parallel with C5, C7, C8, C6, C11] +- [ ] C8: U8 text range primitives — `feat(text): text range primitives with password-field gate` [parallel with C5, C7, C4, C6, C11] +- [ ] C6: U6 action variants (LongPress/ShowMenu/WindowRaise/Cancel) + pre-work chain_steps.rs + dispatch.rs split — `refactor(macos): split actions files; feat(actions): 4 new variants` [parallel with C5, C7, C4, C8, C11] +- [ ] C11: U11 macOS permission tri-state — `feat(macos): screen recording + automation tri-state` [parallel with C5, C7, C4, C8, C6; gates C9] +- [ ] C9.1: U9 screencapturekit fork audit — `docs(security): audit screencapturekit doom-fish fork` [parallel with all Phase C, gates C9.2] +- [ ] C9.2: U9 modern screenshot — `feat(screenshot): modern backend via ScreenCaptureKit + windows-capture` [after C11 + C9.1] +- [ ] C10: U10 new surfaces (Toolbar/Spotlight/Dock/MenuBarExtras + tray commands on macOS) — `feat(surfaces): add toolbar/spotlight/dock/menubar-extras/tray` [parallel with C9.2] +- [ ] C12.1: U12 DeliverFiles macOS/Windows spikes — `feat(spike): validate deliver-files semantic paths` [after C9.2, gates C12.2] +- [ ] C12.2: U12 DeliverFiles + ForceClick implementations — `feat(actions): deliver-files (headless) + force-click with path validation` [after C12.1] + +### Phase D — Documentation + Release (U14) + +- [ ] C14.1: `skills/agent-desktop-windows/SKILL.md` + references — `docs(skill): add windows skill` [parallel with C14.2–C14.5] +- [ ] C14.2: `docs/migrations/0.1-to-0.2.md` + CHANGELOG curation — `docs: v0.2.0 migration guide` [parallel] +- [ ] C14.3: `docs/phases.md` cleanup + `crates/ffi/README.md` policy updates — `docs: phase 2 hygiene sweep` [parallel] +- [ ] C14.4: port `electron-compat.md` private memory → `docs/solutions/best-practices/` — `docs(solutions): cross-platform electron compat` [parallel] +- [ ] C14.5: `docs/solutions/logic-errors/deliver-files-path-safety-2026-04-18.md` + `docs/solutions/logic-errors/watch-element-thread-safety-2026-04-18.md` — `docs(solutions): phase 2 threat models` [parallel] +- [ ] C14.6: release v0.2.0 — `chore: release 0.2.0` + +### Progressive commit discipline + +- Each box = one `feat:` / `fix:` / `refactor:` / `docs:` / `ci:` conventional commit, single concern. +- After every box merges to `main`, the preceding deferred tests / CI workflow re-runs and must stay green. +- If a swarm worker's commit would conflict with another worker's in-flight commit, the later-to-open PR rebases — never force-push an earlier PR. +- Sub-PR 1g (the `AD_ABI_VERSION_MAJOR = 1 → 2` bump) is the one-way door; everything after it is v0.2.0. + +--- + +## Sources & References + +- **Origin document:** [docs/brainstorms/2026-04-18-phase2-windows-crossplatform-brainstorm.md](../brainstorms/2026-04-18-phase2-windows-crossplatform-brainstorm.md) +- **Prior plan (superseded):** [docs/plans/2026-02-25-feat-windows-adapter-phase2-plan.md](./2026-02-25-feat-windows-adapter-phase2-plan.md) — covers a narrower P2 scope that this plan supersedes. +- **Related plan:** [docs/plans/2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md](./2026-04-16-001-fix-ffi-safety-abi-correctness-plan.md) — FFI safety baseline this plan builds on. +- **Phases reference:** [docs/phases.md §Phase 2](../phases.md) (cleaned up in Unit 14). +- **Institutional learnings referenced:** + - [docs/solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md](../solutions/best-practices/deduplicate-ref-allocator-via-config-struct-2026-04-14.md) + - [docs/solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md](../solutions/logic-errors/progressive-snapshot-review-contract-2026-04-16.md) + - [docs/solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md](../solutions/best-practices/deterministic-build-artifact-marker-2026-04-16.md) + - [docs/solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md](../solutions/best-practices/identity-fingerprint-against-os-reorder-2026-04-16.md) +- **External references:** + - UIA patterns: https://learn.microsoft.com/en-us/dotnet/framework/ui-automation/ui-automation-control-patterns-overview + - `uiautomation` crate: https://crates.io/crates/uiautomation + - `windows` crate: https://crates.io/crates/windows + - `windows-capture` crate: https://crates.io/crates/windows-capture + - ScreenCaptureKit: https://developer.apple.com/documentation/screencapturekit + - `screencapturekit` crate (doom-fish fork): https://crates.io/crates/screencapturekit + - `objc2` crate: https://crates.io/crates/objc2 + - `inventory` crate: https://crates.io/crates/inventory + - `schemars` crate: https://crates.io/crates/schemars + - MS Agent Framework MCP transport: https://learn.microsoft.com/en-us/ai/agent-framework (stdio sufficient through 1.0) diff --git a/docs/plans/2026-06-02-001-feat-playwright-grade-reliability-plan.md b/docs/plans/2026-06-02-001-feat-playwright-grade-reliability-plan.md new file mode 100644 index 0000000..0e8ae37 --- /dev/null +++ b/docs/plans/2026-06-02-001-feat-playwright-grade-reliability-plan.md @@ -0,0 +1,366 @@ +--- +title: "feat: add Playwright-grade desktop reliability" +type: feat +status: active +date: 2026-06-02 +--- + +# feat: add Playwright-grade desktop reliability + +## Summary + +Build a pragmatic reliability foundation for Agent Desktop by borrowing the parts of Playwright that make automation dependable: late target resolution, strict matching, actionability waits, retrying assertions, useful failure traces, and session-safe refs. The work keeps Agent Desktop's role clear: Playwright remains the browser automation tool; Agent Desktop becomes a reliable native desktop automation tool whose behavioral contract can be shared by macOS, Windows, and Linux adapters. + +--- + +## Problem Frame + +Agent Desktop already has strong macOS primitives: snapshot-scoped refmaps, rich `RefEntry` identity data, path and bounds based stale-ref recovery, semantic AX action chains, and policy-gated physical fallback. The remaining reliability gap is at the public contract layer. A ref can still behave like a concrete handle that is resolved once and acted on immediately, while Playwright's most reliable surface keeps a target as intent until action time, checks that the target is unique and actionable, and leaves enough trace data to debug failures. + +This plan strengthens that contract without building a test runner, a daemon, a visual trace viewer, or a custom selector language. It adds the smallest cross-platform foundation needed for future Windows and Linux adapters to inherit the same semantics instead of inventing platform-local behavior. + +--- + +## Validation Baseline and Success Criteria + +Before implementation, capture a small baseline of representative flaky desktop workflows. The baseline should include the command sequence, expected behavior, observed failure, and which acceptance example or conformance test will prove the failure is fixed. If no production incident is available for a category, use a fixture-backed reproduction rather than inventing scope. + +Minimum success for the first shipping slice is: + +- A moved-but-unique ref resolves late and acts safely without a manual refresh. +- A duplicated target fails as ambiguous without mutating the desktop. +- A disabled or non-actionable target can be waited on with a structured timeout reason. +- Existing one-shot `get` and `is` behavior remains cheap and non-retrying. +- Existing semantic macOS click and text flows continue to pass. + +--- + +## Requirements + +### Target Resolution + +- R1. Ref-consuming commands must resolve through a late-bound target contract that can re-find an element from snapshot identity, source surface, path, and semantic fields before acting. +- R2. Single-target actions must be strict by default: zero matches is stale or missing, multiple matches is ambiguous, and only exactly one candidate proceeds. +- R3. Stale-ref handling must remain fail-closed. Automatic recovery is allowed only when the resolver finds one high-confidence candidate in the same scoped app/window/surface. + +### Action Reliability + +- R4. Ref actions must run a shared actionability preflight before mutation: resolved uniquely, evaluated for visibility, enabledness, and stability when native evidence is available, allowed to proceed on unknown checks only when core policy says the action remains safe, and policy-compatible. +- R5. Actionability failures must return structured reasons that agents can use for recovery, not only generic `ACTION_FAILED`. +- R6. Existing headless-first policy must stay intact: semantic AX/UIA/AT-SPI actions remain primary, and focus or cursor movement happens only through explicit policy paths. + +### Waiting, Assertions, and Diagnostics + +- R7. `wait` must support retrying state and actionability predicates so agents can wait for "enabled", "visible", "value changed", "text present", or "actionable" without manual polling loops. +- R8. One-shot `get` and `is` commands must remain available and cheap; retrying behavior belongs in `wait` predicates rather than a full test-runner abstraction. +- R9. Optional trace output must capture resolver attempts, actionability checks, action-chain steps, post-state, and failure context in a machine-readable artifact. + +### Interface and Platform Parity + +- R10. Current CLI and FFI ref-consuming action surfaces must use the same core ref, resolver, actionability, and error semantics so reliability does not depend on the calling interface. Future MCP work must inherit that core path when MCP is implemented. +- R11. Concurrent agents must be able to avoid global latest-snapshot collisions through an explicit session-scoped ref namespace while default no-session behavior stays unchanged. +- R12. The work must define a reusable adapter conformance harness for strict resolution, stale-ref recovery, actionability, retrying waits, and session isolation. Windows and Linux adapters must pass that harness before claiming parity. + +--- + +## Key Technical Decisions + +- **Keep refs, add locator semantics behind them:** Do not replace `@eN` refs or introduce a new user-facing selector DSL. Extend the core ref pipeline so a ref carries enough locator-like identity for strict late resolution. +- **Strictness is the default:** If a recovery search finds multiple matching candidates, the command fails with a first-class ambiguity error and candidate summaries. Guessing is worse than a retryable failure because desktop actions can mutate real user state. +- **Actionability is shared core policy, with adapter evidence only where needed:** Core owns the vocabulary, result shape, and per-action gate rules. The first pass should build the report from the resolved target, policy, live node fields, bounds, and existing adapter methods. Add new adapter trait surface only when a named check cannot be expressed through existing methods. +- **Extend `wait`, do not build an `expect` framework yet:** Playwright-style retrying assertions are valuable, but a standalone test runner would be YAGNI. Add predicate modes to `wait` and leave `get` / `is` as one-shot commands. +- **Trace JSONL before trace viewer:** Start with structured trace events written to a caller-provided path or failure-retained default. A UI viewer, zip bundle, or screencast can wait until real trace usage proves the need. +- **Session namespace before daemon:** Session-scoped refs solve multi-agent contamination without committing to a future persistent daemon. The default CLI behavior can remain compatible while MCP and advanced callers opt into session IDs. +- **Conformance before adapter expansion:** The plan does not implement Windows or Linux automation. It defines an adapter-facing harness and capability matrix those adapters must satisfy later so platform work does not diverge. + +--- + +## High-Level Technical Design + +```mermaid +flowchart TB + Snapshot["snapshot builds tree + RefMap"] --> Entry["RefEntry + locator identity"] + Entry --> Resolve["strict late resolver"] + Resolve -->|0 matches| Stale["STALE_REF / missing"] + Resolve -->|many matches| Ambiguous["AMBIGUOUS_TARGET"] + Resolve -->|1 match| Actionable["actionability preflight"] + Actionable -->|not ready| Waitable["structured waitable reason"] + Actionable -->|ready| Execute["adapter action chain"] + Execute --> Post["post-state + optional trace event"] + Wait["wait predicates"] --> Resolve + Trace["trace sink"] -. records .-> Resolve + Trace -. records .-> Actionable + Trace -. records .-> Execute +``` + +The resolver remains ref-rooted and snapshot-aware. The new behavior is that ref actions no longer treat the loaded `RefEntry` as enough evidence by itself. They resolve the entry strictly, classify failure, run actionability, then execute through the existing adapter action path. + +--- + +## Scope Boundaries + +### Included + +- Core resolver contract and strict matching result types. +- macOS implementation of the new resolver and actionability checks using existing AX data. +- Retry-capable `wait` predicate modes. +- Trace event sink sufficient for debugging resolver/actionability failures. +- Session-scoped ref storage namespace. +- FFI parity for `RefEntry` fields that CLI resolution already uses. +- Shared conformance tests and documentation updates. + +### Deferred to Follow-Up Work + +- Full MCP server implementation. +- MCP parity acceptance gates, once MCP exists, requiring MCP commands to use the same core resolver/session path. +- Persistent daemon, warm caches, and event-driven invalidation. +- Visual trace viewer, screencast recording, and trace zip bundles. +- OCR or vision fallback for inaccessible custom-rendered UIs. +- New public selector language beyond existing `find` inputs and refs. +- Windows and Linux adapter implementation beyond conformance expectations. + +--- + +## Execution Strategy + +Ship this in slices so the foundation proves value before wider surface area is added: + +1. **Slice 1: CLI/macOS reliability core.** Implement U1-U3 first: strict late resolution, shared actionability, and retrying wait predicates. This is the minimum shippable reliability improvement. +2. **Slice 2: diagnostics and session isolation.** Implement U4 only when Slice 1 exposes failures that need trace artifacts, or when current callers need concurrent no-`--snapshot` isolation. Session IDs must be explicit; no daemon or global process context. +3. **Slice 3: interface and platform hardening.** Implement U5 when FFI consumers need ref-consuming mutations to match CLI behavior. Implement U6 before Windows/Linux adapter work claims parity. + +Do not start a later slice just because it is listed here. Each slice must either protect correctness already introduced by earlier units or answer a current consumer need. + +--- + +## Acceptance Examples + +- AE1. Given a ref whose element moved within the same source window, when a click command runs, then the resolver re-finds the unique matching element and the action proceeds. +- AE2. Given two matching "Save" buttons in the same window, when a click command targets a stale ref whose identity now matches both, then the command fails with an ambiguity error and does not click either. +- AE3. Given a disabled button, when `wait` asks for it to become actionable, then the command retries until it becomes enabled or returns a timeout with the last actionability reason. +- AE4. Given two agents using different session IDs, when each omits `--snapshot`, then each loads its own latest snapshot and cannot act on the other's latest refs. +- AE5. Given a failed action with trace enabled, when the command exits, then the trace records resolver attempts, candidate count, actionability result, action-chain steps attempted, and final error. +- AE6. Given the same macOS window is discovered through focused, main, and `AXWindows` roots, when strict resolution counts candidates, then native identity deduplication prevents a false ambiguity result. + +--- + +## Implementation Units + +### U1. Strict Late Resolution Contract + +- **Goal:** Add a core target-resolution contract that preserves ref ergonomics while making strict late resolution explicit. +- **Requirements:** R1, R2, R3, R10 +- **Dependencies:** None +- **Files:** + - `crates/core/src/refs.rs` + - `crates/core/src/resolved_element.rs` + - `crates/core/src/error.rs` + - `crates/core/src/commands/helpers.rs` + - `crates/macos/src/tree/resolve.rs` + - `crates/macos/src/tree/resolve_identity.rs` + - `crates/macos/src/tree/resolve_bounds.rs` + - `crates/macos/src/tree/resolve_tests.rs` + - `crates/core/src/commands/helpers_tests.rs` +- **Approach:** Introduce a small core resolution result model: unique target, no target, ambiguous target, and timed out. Add a dedicated ambiguity error code so agents can distinguish "refresh needed" from "target is no longer unique." Reuse the existing `RefEntry` fields rather than creating a new selector object unless implementation needs a tiny helper struct to group matching inputs. macOS keeps its current path fast path and broad search fallback, deduplicates candidates by native element identity, and returns candidate classification instead of first-success-only semantics. +- **Patterns to follow:** Existing scoped-root search in `crates/macos/src/tree/resolve.rs`; identity matching in `crates/macos/src/tree/resolve_identity.rs`; stale-ref error shape in `crates/core/src/error.rs`. +- **Test scenarios:** + - A valid ref with matching path and identity resolves to exactly one handle. + - A ref whose path no longer resolves but whose identity uniquely matches elsewhere in the same source window resolves successfully. + - A ref whose identity matches multiple candidates returns an ambiguity classification and no action is executed. + - A ref found through duplicate macOS roots is deduplicated to one native candidate before strict count classification. + - A syntactically invalid ref remains `INVALID_ARGS`, not stale or ambiguous. + - A valid ref with no candidates returns `STALE_REF` with the existing refresh suggestion. +- **Verification:** Ref-consuming commands can distinguish stale, ambiguous, timeout, and unique resolution without changing their public JSON envelope. + +### U2. Shared Actionability Preflight + +- **Goal:** Add a small actionability layer before ref actions mutate the desktop. +- **Requirements:** R4, R5, R6 +- **Dependencies:** U1 +- **Files:** + - `crates/core/src/action.rs` + - `crates/core/src/actionability.rs` + - `crates/core/src/commands/helpers.rs` + - `crates/macos/src/actions/mod.rs` + - `crates/macos/src/actions/actionability.rs` + - `crates/macos/src/actions/dispatch.rs` + - `crates/macos/src/actions/chain.rs` + - `crates/macos/src/actions/chain_steps_tests.rs` + - `crates/core/src/commands/ref_policy_tests.rs` +- **Approach:** Define an `ActionabilityReport` in core with a compact set of checks: unique target, visible or unknown, enabled or unknown, stable or unknown, editable when text input is requested, and policy allowed. Build the first implementation from the resolved target, policy, live state, bounds, available actions, and existing adapter calls. Keep `force` out of the first implementation unless a current command needs it; explicit physical/focus policy already handles most escape hatches. +- **Actionability gate matrix:** + + | Action group | Required evidence | Unknowns allowed | Blocking states | + | --- | --- | --- | --- | + | Click/select/toggle | Unique target, policy allowed, semantic action available or physical policy allowed | Visibility/stability unknown when a semantic native action is available | Disabled false, zero-size bounds when bounds exist, policy would require forbidden focus/cursor movement | + | Text input/set value | Unique target, policy allowed, settable value or editable text evidence | Visibility unknown for headless set-value | Non-editable false, missing settable value when keyboard fallback is policy-denied | + | Scroll | Unique target or window, policy allowed, scroll action or allowed physical fallback | Enabled/stability unknown | No scroll action and physical fallback denied | + | Keyboard/app press | App/window target, policy allowed, focus policy satisfied | Element visibility when the command is app-scoped | Required focus cannot be established under policy | + +- **Patterns to follow:** `InteractionPolicy` in `crates/core/src/action.rs`; post-state verification in `crates/macos/src/actions/post_state.rs`; chain timeout handling in `crates/macos/src/actions/chain.rs`. +- **Test scenarios:** + - A disabled element reports `enabled=false` and a structured actionability failure before action dispatch. + - An element with zero-size bounds reports not visible when bounds are available. + - A text command against a non-editable target fails actionability before keyboard or clipboard fallback. + - A headless command that would require cursor movement remains policy denied. + - A command with unknown visibility but valid semantic AX action can proceed when no stronger evidence exists. +- **Verification:** Existing semantic click and text paths still work, but failures include actionability reasons before falling back to generic action failure. + +### U3. Retrying Wait Predicates + +- **Goal:** Make Playwright-style retrying assertions available through `wait` without building a separate test framework. +- **Requirements:** R7, R8 +- **Dependencies:** U1, U2 +- **Files:** + - `src/cli_args_system.rs` + - `src/dispatch.rs` + - `crates/core/src/commands/wait.rs` + - `crates/core/src/commands/wait_tests.rs` + - `crates/core/src/search_text.rs` + - `src/batch.rs` +- **Approach:** Extend `wait` with predicate modes for ref state, ref actionability, live value/text, and match count. Keep `get` and `is` one-shot so simple reads stay cheap and predictable. Predicate results should include elapsed time and the last observed value or actionability reason. Do not add a generic expression language. +- **Wait predicate contract:** + + | Predicate | Required input | Snapshot behavior | Timeout output | + | --- | --- | --- | --- | + | `exists` | ref or find query | Fixed `--snapshot` stays fixed; omitted snapshot may refresh through existing throttle | last candidate count and last resolver error | + | `state` | ref plus state name/value | Re-resolves the ref on each poll | last observed state map and actionability reason if related | + | `actionable` | ref plus optional action kind | Re-runs strict resolution and U2 gate on each poll | last failed check and policy reason | + | `value` | ref plus equals/contains operand | Re-resolves and reads live value on each poll | last observed value | + | `text` | text or query operand | Uses existing text search refresh behavior | last match count and matched surface when present | + | `match-count` | find query plus count relation | Uses the same fixed/latest snapshot rule as `exists` | last observed count | + + Invalid predicate/input combinations return `INVALID_ARGS`, not timeout. +- **Patterns to follow:** Existing `wait_for_element`, `wait_for_text`, and `LatestRefCache` in `crates/core/src/commands/wait.rs`. +- **Test scenarios:** + - Waiting for an element to become enabled polls live state and succeeds when the state changes. + - Waiting for actionability reports the last failed check on timeout. + - Waiting for text still saves a fresh snapshot when it finds a match. + - Waiting against a fixed `--snapshot` does not silently switch to a newer snapshot. + - Waiting without `--snapshot` refreshes latest snapshot metadata only at the existing throttle interval. +- **Verification:** Agents can replace manual sleep-and-poll loops with a single structured `wait` call for state and actionability. + +### U4. Trace Events and Session-Scoped RefStore + +- **Goal:** Add the smallest useful diagnostics and isolation layer for concurrent agent runs. +- **Requirements:** R9, R11 +- **Dependencies:** U1, U2 +- **Files:** + - `src/cli.rs` + - `src/cli_args.rs` + - `src/batch.rs` + - `src/dispatch.rs` + - `src/main.rs` + - `crates/core/src/trace.rs` + - `crates/core/src/refs_store.rs` + - `crates/core/src/refs_lock.rs` + - `crates/core/src/commands/helpers.rs` + - `crates/core/src/commands/snapshot.rs` + - `crates/core/src/snapshot_ref.rs` + - `crates/core/src/commands/wait.rs` + - `crates/core/src/commands/status.rs` + - `crates/core/src/refs_tests.rs` + - `crates/core/src/commands/snapshot_tests.rs` +- **Approach:** Add opt-in trace writing as structured JSONL events, not a viewer. Add a session namespace to `RefStore` so callers can isolate `latest_snapshot_id` and snapshot directories by session. Preserve current default behavior when no session is provided. +- **Session contract:** Add a global `--session <id>` CLI option and equivalent batch field. Validate IDs as short filesystem-safe tokens. Map storage to `~/.agent-desktop/sessions/{session_id}/snapshots/...` while preserving the current default path for no-session commands. Thread the session through snapshot creation, drill-down, helper resolution, wait, status, and every omitted-`--snapshot` latest lookup. Do not use process-global session state. +- **Patterns to follow:** Private file writes and lock handling in `crates/core/src/refs.rs` and `crates/core/src/refs_store.rs`; response envelope handling in `src/main.rs`. +- **Test scenarios:** + - Two session IDs can each save and load independent latest snapshots. + - Omitting a session ID preserves the current default latest-snapshot path. + - A failed ambiguous resolution writes trace events for candidate count and final error when tracing is enabled. + - Trace output never appears in stdout JSON envelopes. + - Trace writing failure does not turn a successful desktop action into a failed action unless the caller explicitly requested strict trace output. +- **Verification:** Concurrent agent workflows have an isolation mechanism, and failed actions can be diagnosed from trace events without rerunning with verbose stderr. +- **Delivered:** Added `CommandContext`, global `--session`, batch item `session`, session-scoped `RefStore`, opt-in `--trace`, `--trace-strict`, trace events on snapshot/ref/action paths, and tests for session isolation and trace behavior. + +### U5. FFI Ref-Action Parity + +- **Goal:** Bring FFI ref resolution and ref-consuming mutations up to the same reliability standard as the CLI. +- **Requirements:** R10 +- **Dependencies:** U1, U2 +- **Files:** + - `crates/ffi/src/types/ref_entry.rs` + - `crates/ffi/src/actions/resolve.rs` + - `crates/ffi/src/actions/execute.rs` + - `crates/ffi/src/types/action.rs` + - `crates/ffi/src/types/action_result.rs` + - `crates/ffi/src/actions/resolve_tests.rs` + - `crates/ffi/tests/c_abi_actions.rs` + - `crates/ffi/tests/c_abi_layout.rs` + - `crates/ffi/include/agent_desktop.h` + - `scripts/update-ffi-header.sh` + - `skills/agent-desktop-ffi/SKILL.md` +- **Approach:** Fill the FFI `AdRefEntry` conversion gap for the fields CLI resolution already uses: value, description, bounds, source app/window/title, surface, root ref, path, and path mode. Do not require `states` for resolution parity unless the CLI resolver actually consumes them; if state data becomes necessary for actionability, make that dependency explicit in U2. Add a ref-action entrypoint or shared command wrapper that accepts ref identity plus action/policy and runs strict resolution, actionability, optional trace context, and execution in one path. Keep the existing native-handle execute API documented as low-level if it remains. +- **Patterns to follow:** FFI string conversion in `crates/ffi/src/convert/string.rs`; ABI layout tests in `crates/ffi/tests/c_abi_layout.rs`; existing header update script. +- **Test scenarios:** + - FFI resolution with value and description can match a target that name-only matching cannot. + - FFI resolution with bounds hash rejects a candidate with the wrong bounds. + - Invalid UTF-8 in optional identity fields returns `INVALID_ARGS`. + - FFI ref-consuming actions run the same preflight as CLI ref-consuming actions. + - The low-level native-handle action path, if retained, is documented as bypassing ref reliability semantics. + - C ABI layout tests reflect any struct changes. + - Header drift check catches stale `agent_desktop.h` after type changes. +- **Verification:** CLI and FFI consumers no longer get materially different stale-ref or actionability behavior for the same target identity. +- **Delivered:** Expanded `AdRefEntry` to carry the full ref identity envelope, switched FFI resolution to strict resolution, added `ad_execute_ref_action_with_policy`, updated the committed C header, and added ABI/conversion/actionability tests. + +### U6. Cross-Platform Conformance and Documentation + +- **Goal:** Lock the reliability contract before Windows and Linux adapters implement their platform-specific details. +- **Requirements:** R12 +- **Dependencies:** U1, U2, U3, U4, U5 +- **Files:** + - `crates/core/src/refs_test_support.rs` + - `crates/core/src/commands/ref_policy_tests.rs` + - `crates/core/src/commands/wait_tests.rs` + - `tests/conformance/README.md` + - `tests/fixtures/README.md` + - `tests/fixtures/skeleton-tree.json` + - `README.md` + - `skills/agent-desktop/SKILL.md` + - `skills/agent-desktop/references/workflows.md` + - `docs/phases.md` + - `docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md` +- **Approach:** Add core and fixture-level conformance tests for resolver strictness, actionability reason shapes, retrying wait predicates, trace event shape, and session isolation. Define an adapter conformance harness that each real platform adapter can opt into with adapter-provided fixtures. Publish a small capability matrix for required, optional, unsupported, and unknown checks. Update docs to describe refs as snapshot-scoped handles backed by strict late resolution, not stable durable element handles. +- **Patterns to follow:** Existing solution docs under `docs/solutions/best-practices/`; existing fixture documentation in `tests/fixtures/README.md`; skill reference organization under `skills/agent-desktop/references/`. +- **Test scenarios:** + - Mock adapter fixtures prove zero, one, and many candidate resolution outcomes. + - Mock adapter fixtures prove actionability timeout includes the last failed reason. + - Fixture snapshots cover a moved-but-unique target and an ambiguous duplicate target. + - Adapter conformance fixtures prove required checks can pass on a real adapter before that adapter claims parity. + - Documentation examples show stale, ambiguous, and waitable recovery paths. + - Windows and Linux stub adapters continue returning supported structured errors until implemented. +- **Verification:** A future Windows or Linux adapter has explicit behavioral tests to satisfy before claiming parity. +- **Delivered:** Added `tests/conformance/README.md`, a reliability solution note, README reliability contract docs, and cross-platform expectations for stale, ambiguous, actionable, session, trace, and FFI behavior. + +--- + +## Risks and Mitigations + +- **Risk: Overbuilding a selector framework.** Mitigation: keep the first pass ref-backed and internal; no new public selector DSL unless future usage proves it. +- **Risk: Ambiguity errors feel worse than best-effort clicking.** Mitigation: include candidate summaries and recovery suggestions. For desktop mutation, fail-closed is the safer default. +- **Risk: Actionability checks become platform-inconsistent.** Mitigation: core defines the report vocabulary; adapters report supported, unsupported, or unknown checks explicitly. +- **Risk: Trace output bloats normal CLI usage.** Mitigation: trace is opt-in and never mixed into stdout envelopes. +- **Risk: Session namespaces complicate simple scripts.** Mitigation: default behavior remains unchanged when no session is specified. +- **Risk: FFI struct changes affect consumers.** Mitigation: keep changes tied to existing FFI ABI layout tests and header drift checks; document the reliability rationale. +- **Risk: Later slices expand before the core path proves value.** Mitigation: U4-U6 are gated by the Execution Strategy; U1-U3 must ship and validate the core behavior first. + +--- + +## Documentation and Operational Notes + +- Update README and skill docs to explain three target states: fresh ref, stale ref, and ambiguous target. +- Document coordinate and physical commands as lower-confidence fallbacks compared with accessibility-backed refs. +- Add a solution note after implementation so future adapter work understands why strict late resolution and actionability are core semantics, not macOS-only details. +- Keep Playwright comparisons in docs conceptual. Avoid implying Agent Desktop automates browsers or replaces Playwright. + +--- + +## Sources and Research + +- Playwright docs: locators, strictness, actionability, auto-waiting, retrying assertions, trace viewer, browser contexts, codegen, and CLI behavior. +- Installed Playwright CLI observed locally as `Version 1.58.0`; command surface included `open`, `codegen`, `install`, `screenshot`, `pdf`, and `show-trace`. +- Current Agent Desktop ref and resolver code: `crates/core/src/refs.rs`, `crates/core/src/refs_store.rs`, `crates/core/src/commands/helpers.rs`, `crates/macos/src/tree/resolve.rs`, `crates/macos/src/tree/resolve_identity.rs`, and `crates/macos/src/tree/resolve_bounds.rs`. +- Current action reliability code: `crates/core/src/action.rs`, `crates/macos/src/actions/chain.rs`, `crates/macos/src/actions/dispatch.rs`, `crates/macos/src/actions/post_state.rs`, and `crates/macos/src/actions/chain_web_steps.rs`. +- Current one-shot and wait behavior: `crates/core/src/commands/get.rs`, `crates/core/src/commands/is_check.rs`, and `crates/core/src/commands/wait.rs`. +- FFI parity gap: `crates/ffi/src/actions/resolve.rs`. +- Cross-platform adapter seam: `crates/core/src/adapter.rs`, `crates/windows/src/adapter.rs`, and `crates/linux/src/adapter.rs`. diff --git a/docs/plans/2026-06-24-001-feat-ffi-completion-plan.md b/docs/plans/2026-06-24-001-feat-ffi-completion-plan.md new file mode 100644 index 0000000..a95c058 --- /dev/null +++ b/docs/plans/2026-06-24-001-feat-ffi-completion-plan.md @@ -0,0 +1,478 @@ +--- +title: "feat: Complete the FFI surface and cross-platform parity contract" +status: active +type: feat +date: 2026-06-24 +depth: deep +origin: docs/phases.md (P2-O16, ad_abi_version §613/§632) +decisions: + - Codegen migration included, sequenced last (after entrypoints are proven) + - First external consumer = Python (ctypes) smoke harness in CI + - Cross-platform = contract + gates only; no Windows/Linux adapter implementation +--- + +# feat: Complete the FFI surface and cross-platform parity contract + +## Summary + +`crates/ffi` already ships a real C ABI — a cdylib plus a committed cbindgen header (`crates/ffi/include/agent_desktop.h`) exposing ~60 `ad_*` functions over the core engine. This plan closes the remaining gaps so the FFI is *fully done* on macOS and *ready to light up* on Windows/Linux with zero new FFI code: + +1. A load-time **ABI-version handshake** (`ad_abi_version`, `ad_init`) so a consumer detects a header/dylib mismatch instead of corrupting memory. +2. The missing **command-backed entrypoints** — `ad_snapshot` (full refmap → `@e` refs), `ad_execute_by_ref`, `ad_wait`, `ad_version`, `ad_status`. +3. **`ad_set_log_callback`** forwarding `tracing` output to a consumer callback. +4. A **Python ctypes smoke harness** in CI — the first real external consumer. +5. A **`build.rs` codegen migration** that replaces the hand-written command-backed wrappers with one generated `ad_<name>` per command, sharing its file-scan with a CI exhaustiveness guard so CLI↔FFI parity becomes automatic. + +It deliberately does **not** implement the Windows or Linux adapters. Instead it bakes the parity *contract* — a CI header-drift gate, per-target `release-ffi` builds, and verified `PLATFORM_NOT_SUPPORTED` passthrough — so those adapters expose the same `ad_*` surface for free when they land. + +--- + +## Problem Frame + +The FFI is the in-process path for non-Rust hosts (Python agents, Swift apps, Go services) to drive desktop automation without spawning the CLI or parsing JSON over a pipe. Today it has three classes of gap: + +- **Safety:** no runtime way for a consumer to check that the dylib it loaded matches the header it compiled against. The only guard is comparing `ad_*_size()` against the hand-written `AD_*_SIZE` macros — partial and easy to skip. +- **Completeness:** the ref-based observe→act loop that defines the CLI (snapshot → `@e5` ref → action) is not reachable in one call. `ad_get_tree` returns a tree with **no refs** (`ref_id` is always null); there is no `ad_snapshot`, no `ad_execute_by_ref`, no `ad_wait`. There is no `ad_version`/`ad_status`, and `dlopen` consumers cannot see debug output. +- **Maintainability + proof:** the ~60 wrappers are hand-maintained, so CLI/FFI parity is enforced by review, not by construction (drift risk). And nothing outside the repo consumes the ABI — it is built and layout-tested but unproven from another language. + +This plan resolves all three while keeping core untouched in spirit: the FFI calls `core` through the `PlatformAdapter` trait, so cross-platform parity is an architectural property, not new per-platform code. + +--- + +## Scope Boundaries + +### In scope +- ABI-version handshake (`ad_abi_version`, `ad_init`, `AD_ABI_VERSION_MAJOR`). +- New command-backed entrypoints: `ad_snapshot`, `ad_execute_by_ref`, `ad_wait`, `ad_version`, `ad_status`. +- `ad_adapter_create_with_session` constructor (session plumbing for refmap persistence — KTD5) and a `stub-adapter` cargo feature for CI/passthrough testing (KTD10). +- `ad_set_log_callback` + a `tracing_subscriber` layer. +- Python ctypes smoke harness wired into CI. +- `build.rs` codegen for the command-backed family + a CI exhaustiveness guard + a header-drift gate. +- Cross-platform parity gates: header-drift CI check, per-target `release-ffi` build verification, `PLATFORM_NOT_SUPPORTED` passthrough tests. + +### Deferred to Follow-Up Work +- Codegen of the **typed-struct, adapter-direct family** (`ad_find`, `ad_get`, `ad_get_tree`, `ad_execute_action`, `ad_resolve_element`). These need a per-parameter marshaling refactor; this plan leaves them hand-written and only codegens the command-backed family (see KTD2). +- A Swift native-host example (Python is the CI consumer this round). +- Progressive-traversal args (`--skeleton`, `--root @ref`) on `ad_snapshot` — the first cut exposes the full-window snapshot only; the skeleton/drill-down parity is a fast-follow. +- ✅ **DONE (commit `9bf4731` on `chore/ffi-header-toolchain`/PR #67):** Restored the 3 ABI header doc comments lost in the cbindgen regen — added `///` docs on the Rust source (`error.rs` `ad_last_error_details` privacy note + `AdResult` forward-compat note; `actions/execute.rs` behavioral descriptions for the `ad_execute_action*` family) and regenerated the header. Done directly on the integration branch (not a separate post-merge PR) since the fold already required the branch to be live. +- ✅ **DONE (commit `2aebce2`):** DRY — folded `wait.rs`'s local `app_error_to_adapter_error` into the shared `commands::app_error_to_adapter`; one canonical `AppError→AdapterError` conversion across every ffi command. +- **Won't-fix (noted):** `AdWaitArgs::count` is `usize`; on a hypothetical 32-bit target it would shift the `AD_WAIT_ARGS_SIZE=112` layout pin. Only 64-bit targets are supported and the per-platform `const` assert catches it at compile time — revisit only if 32-bit ever ships. + +### Out of scope (different product phase) +- **Windows adapter implementation** — Phase 2. +- **Linux adapter implementation** — Phase 3. +- Any new `Action`/`ErrorCode` variants or new CLI commands (those arrive with the cross-platform phases). + +--- + +## Requirements + +- **R1** — A consumer can detect header/dylib incompatibility at load time via `ad_abi_version()` + `ad_init(expected_major)`, before any adapter call. *(origin: docs/phases.md §613)* +- **R2** — `ad_snapshot` produces the CLI-format snapshot envelope with `@e` refs and persists the refmap, so a consumer can drive the ref-based observe→act loop. +- **R3** — `ad_execute_by_ref` drives a ref action through the **full strict-resolution ladder** (refmap load → strict resolve → `STALE_REF`/`AMBIGUOUS_TARGET` → live actionability → dispatch → handle release) with **CLI-parity policy** (headless default). +- **R4** — `ad_version`, `ad_status`, and `ad_wait` expose CLI-equivalent behavior over the ABI. +- **R5** — `ad_set_log_callback` forwards `tracing` output to a consumer-supplied callback, thread-safely, without writing to stdout, and never failing a mutation on a trace error. +- **R6** — A Python ctypes harness loads the dylib, validates `ad_abi_version` and every `ad_*_size()` against the header, drives the new entrypoints, and runs as a CI gate. +- **R7** — The command-backed wrappers are generated by `build.rs` from the per-file command set, with a CI exhaustiveness guard that fails when a command file has no FFI wrapper, and per-command `InteractionPolicy` preserved. +- **R8** — The FFI is cross-platform-ready: a CI header-drift gate, per-target `release-ffi` builds, and verified `PLATFORM_NOT_SUPPORTED` passthrough — so Windows/Linux adapters expose the same `ad_*` surface with zero new FFI code. + +--- + +## Key Technical Decisions + +- **KTD1 — Expose the ABI version as a runtime getter *and* a cbindgen-emitted constant.** `ad_abi_version() -> u32` is the only mechanism that works for `dlopen` consumers (the caller compiled against its own header copy and can detect a mismatch only at runtime) — confirmed standard by SQLite/libgit2/Botan. Pair it with `AD_ABI_VERSION_MAJOR` emitted *by cbindgen* from a `pub const AD_ABI_VERSION_MAJOR: u32` via `[const] allow_static_const` (→ `static const uint32_t …`), or an `after_includes` `#define` if a preprocessor `#if` form is wanted — **not** hand-maintained. (`[defines]` maps `cfg`→`#ifdef` and does not emit const values.) `ad_init(expected_major)` failing closed is stronger than the field norm (SQLite asserts, libgit2 leaves it to callers) and right for embedding in agents. +- **KTD2 — Codegen targets only the command-backed JSON-returning family (Family B).** These commands return `Result<Value, AppError>`, but their call sites are **not** uniform: `version::execute()` takes no args/adapter, `status::execute_with_report_with_context(adapter, &report, &ctx)` needs a precomputed `PermissionReport`, and the standard form is `execute(args, adapter, &ctx)`. So the generator emits a **per-command call site** (not a single `execute` fn-pointer table), sharing only the output convention (KTD9) and the policy table (KTD6). The typed-struct adapter-direct family (Family A) needs bespoke marshaling and stays hand-written. +- **KTD3 — Introduce a minimal `CommandDescriptor` for Family B; the codegen and the exhaustiveness guard share one command universe** so generator and guard cannot diverge. The universe is the set of `pub mod` command declarations in `crates/core/src/commands/mod.rs` (cross-checked against the `Commands` enum / dispatch arms) — **not** a `commands/*.rs` glob, which would pull in helper/sub-modules (`helpers.rs`, `wait_mode.rs`, `point_resolve.rs`, `*_tests.rs`) and emit phantom wrappers. No runtime registry, no `inventory`/`linkme` (link-GC unreliable for cdylib per docs/phases.md §631). +- **KTD4 — Generated FFI source is committed at a fixed path** (mirroring the committed-header contract), with a CI regenerate-and-diff drift check — not `$OUT_DIR`-only, whose hash-randomized path makes drift checks unreliable. *(learning: deterministic-build-artifact-marker.)* +- **KTD5 — `ad_snapshot` uses a default `CommandContext`; session is opt-in.** `AdAdapter` gains an optional `session_id`; a `NULL`/absent session means the sessionless default context. This is the minimum needed to persist the refmap. +- **KTD6 — Per-command `InteractionPolicy` stays per-command.** The generator must read a per-command policy table (`type_text` → `focus_fallback`, everything else → `headless`); it must never centrally pick a default. FFI headless default mirrors the CLI. *(learnings: keep-ffi-action-policy-aligned-with-cli, preserve-command-policy-semantics.)* +- **KTD7 — The log callback is thread-safe, install-once, best-effort.** `tracing` events fire from arbitrary threads. Install the subscriber once via `OnceLock`; store the swappable callback pointer in an `AtomicPtr` (lock-free, reentrancy-safe) wrapped in a `Send + Sync` newtype — not a `Mutex` (Mutex only if the install allocates under the lock). The pointer's ABI is `unsafe extern "C"` (not `C-unwind`) so a foreign unwind aborts rather than corrupts Rust state. Invocations are best-effort; a trace failure never fails the originating command. *(external: libgit2/Botan install-once; Rust `OnceLock`/`AtomicPtr` idiom.)* +- **KTD8 — Envelope-version discipline.** `ad_abi_version`/`ad_version` are additive (no `ENVELOPE_VERSION` bump). Only bump `ENVELOPE_VERSION` (with a `BREAKING CHANGE:` footer) if `ad_status` alters always-present top-level fields. Tests assert through the `ENVELOPE_VERSION` constant, never a string literal. *(learning: envelope-version-bump-contract.)* +- **KTD9 — Command-backed entrypoints emit the full CLI envelope.** `commands::{name}::execute(...)` returns only the *data payload*; the `{version, ok, command, data}` envelope is applied by `output::Response::ok(command, data)` (the binary's `finish()` path, `pub` in `crates/core/src/output.rs`). Every command-backed `ad_*` must build the `Response` via `Response::ok`/`Response::err` and serialize *that*, not the raw `Value` — otherwise FFI output diverges from the CLI (e.g. `version` would ship `{version,target,os}` instead of the enveloped form U2's test asserts). +- **KTD10 — A `stub-adapter` cargo feature** swaps `build_adapter()` for a not-supported adapter, so the Python CI harness (U9) and the passthrough tests (U10) can exercise the `PLATFORM_NOT_SUPPORTED` path on a macOS runner without AX permission and without a real Windows/Linux adapter. + +--- + +## High-Level Technical Design + +### ABI-version handshake (load-time) + +```mermaid +sequenceDiagram + participant C as C / Python consumer + participant L as libagent_desktop_ffi + C->>L: dlopen() + C->>L: ad_abi_version() + L-->>C: u32 (packed major) + C->>C: compare to AD_ABI_VERSION_MAJOR (from header) + alt incompatible + C->>C: refuse to proceed (no adapter calls) + else compatible + C->>L: ad_init(expected_major) + L-->>C: AD_RESULT_OK (or ErrInvalidArgs on mismatch) + C->>L: ad_adapter_create() → ... → ad_adapter_destroy() + end +``` + +### Two-family wrapper split (governs the codegen boundary) + +```mermaid +flowchart TD + subgraph A["Family A — typed-struct, adapter-direct (stays hand-written)"] + A1["ad_find / ad_get / ad_get_tree<br/>ad_execute_action / ad_resolve_element"] + A1 --> AM["bespoke C-struct ↔ Rust marshaling per parameter"] + end + subgraph B["Family B — command-backed, JSON-returning (codegen target)"] + B1["ad_snapshot / ad_version / ad_status<br/>ad_wait / ad_execute_by_ref"] + B1 --> BM["uniform: guard_non_null → trap_panic →<br/>commands::name::execute(args, adapter, ctx) →<br/>serialize Value → string_to_c(out)"] + BM --> CG["build.rs walks commands/*.rs →<br/>emits one ad_name per command-backed command"] + CG --> G["CI exhaustiveness guard shares the same file-scan"] + end +``` + +The codegen and the guard read the **same** `crates/core/src/commands/*.rs` set, so a command file that lacks a wrapper fails CI (R7). Family A is explicitly excluded from the walk by a per-command descriptor opt-in. + +--- + +## Output Structure + +``` +crates/ffi/ +├── src/ +│ ├── abi_version.rs # NEW ad_abi_version, ad_init, AD_ABI_VERSION_MAJOR +│ ├── commands/ # NEW command-backed entrypoints (hand-written first, then generated) +│ │ ├── snapshot.rs +│ │ ├── version.rs +│ │ ├── status.rs +│ │ ├── execute_by_ref.rs +│ │ └── wait.rs +│ ├── types/wait.rs # NEW (U7) AdWaitArgs repr(C) struct +│ ├── log_callback.rs # NEW ad_set_log_callback + tracing layer +│ ├── descriptor.rs # NEW CommandDescriptor + per-command policy table (codegen phase) +│ └── generated/ffi_commands.rs # NEW (codegen phase) committed generated wrappers +├── build.rs # MODIFIED add codegen step (codegen phase) +├── include/agent_desktop.h # MODIFIED via scripts/update-ffi-header.sh +└── tests/ + └── c_abi_*.rs # MODIFIED size + lifecycle + parity tests for new surface +tests/ffi-python/ # NEW ctypes smoke harness +.github/workflows/ci.yml # MODIFIED header-drift gate + python harness job +``` + +--- + +## Delivery & PR Strategy + +**One PR per unit, against `main`, organized into 5 dependency waves. Parallel *within* a wave, serial *across* waves.** Not every unit depends on the others — the independent ones build and review concurrently; a dependent unit waits only for *its* specific dependency to **merge**, never for the whole plan. (Stacking is rejected — it adds rebase cascades for no benefit on a dependency chain.) + +**Dependency waves** (derived from each unit's `Dependencies`): + +| Wave | Units (run in parallel) | Unblocked when | +|------|-------------------------|----------------| +| W1 | U1, U2, U3, U8 | immediately — 4 concurrent worktrees | +| W2 | U4, U5, U7 | U3 merged | +| W3 | U6 | U4 merged | +| W4 | U9, U10 | U9: U1+U2+U4 merged · U10: all entrypoints (U1–U8) merged | +| W5 | U11 | U9 + all command-backed entrypoints merged | + +Each unit is its own PR/worktree (smallest reviewable diff, max parallelism). Trivial independent units in the *same* wave MAY be bundled into one PR, never across waves. **U8 and U11 always ship alone** (highest miss-risk). + +**Per-unit pipeline — local review gate BEFORE the remote PR:** + +1. **Worktree:** `git worktree add ../ad-ffi-u<N> -b feat/ffi-u<N>-<slug> main` (off the *latest merged* `main`). `ce-worktree` automates this. +2. **Build:** the builder agent (`ce-work`) implements the unit in that worktree and commits locally. **No push.** +3. **Review (separate agent):** a reviewer audits the worktree diff (`git diff main...HEAD`) — e.g. `ce-code-review mode:agent` (reports findings, does not push). +4. **Fix:** builder or you apply fixes in the same worktree. +5. **Your visibility:** the worktree is a real local checkout — `cd ../ad-ffi-u<N>`, read the diff, run it, fix anything. Nothing is remote yet. +6. **Promote:** when satisfied → `git push -u origin <branch>` → `gh pr create --base main`. +7. **Merge:** CI runs on the remote PR → green → squash-merge → `git worktree remove ../ad-ffi-u<N>` + delete branch. + +**How parallel + reviewable coexist:** within a wave, each unit runs steps 1–7 in its **own worktree at the same time** — you review N worktrees concurrently and they merge independently as each goes green. When a wave's units are all merged, the next wave branches off the updated `main`. So the build is parallel within a wave, dependency order is honored across waves, **nothing merges unreviewed, and nothing is built against unmerged code.** + +**Per-PR gate (before step 6):** `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`, `cargo test --workspace`, `cargo test -p agent-desktop-ffi` all green; regenerate + commit the header if any `ad_*`/`repr(C)` changed; conventional-commit title; no AI attribution. + +--- + +## Implementation Units + +> Phases group the units; dependency order is explicit per unit. The codegen migration (U11) is intentionally last so every wrapper is proven by hand before being mechanized. **Delivery is one PR per unit across 5 dependency waves (parallel within a wave) — see Delivery & PR Strategy.** + +### Progress (as of 2026-06-24) + +**Phase A — COMPLETE as 9 stacked PRs against `main` (none merged; review-fix pass in flight).** + +| Unit | PR | Branch | Base | CI | +|------|----|--------|------|----| +| Foundation (KTD1 self-maintaining header guards + cbindgen pin) | #67 | `chore/ffi-header-toolchain` | `main` | ✅ green | +| U1 ABI handshake | #68 | `feat/ffi-abi-handshake` | #67 | ✅ green | +| U2 `ad_version` | #69 | `feat/ffi-version` | #67 | ✅ green | +| U3 session context | #70 | `feat/ffi-session-context` | #67 | ✅ green | +| U8 `ad_set_log_callback` | #71 | `feat/ffi-log-callback` | #67 | ✅ green | +| U4 `ad_snapshot` | #72 | `feat/ffi-snapshot` | #70 | ✅ green | +| U5 `ad_status` | #73 | `feat/ffi-status` | #70 | ✅ green | +| U7 `ad_wait` | #74 | `feat/ffi-wait` | #70 | ✅ green | +| U6 `ad_execute_by_ref` | #75 | `feat/ffi-execute-by-ref` | #72 | ✅ green | + +- Each unit built in its own git worktree, full CI-mirrored local gate run per unit before push. +- `/ce-code-review` pass landed on every branch (1st round); a 2nd per-worktree review-fix swarm is running (triage → validate → fix → re-gate → force-push). +- **Merge order (dependency waves):** #67 → {#68,#69,#70,#71} → {#72,#73,#74} → #75. Restack siblings after each merge (shared `lib.rs`/header/`Cargo.toml`). + +**Phase B (U9 Python smoke, U10 parity gates) — NOT STARTED.** +**Phase C (U11 codegen migration) — NOT STARTED.** + +### Phase A — ABI safety + entrypoints + +*Delivery: units span waves W1–W3 — per-unit PRs, parallel within each wave (W1: U1/U2/U3/U8 · W2: U4/U5/U7 · W3: U6). See Delivery & PR Strategy.* + +### U1. ABI-version handshake + +**Goal:** Consumers can detect a header/dylib mismatch at load. +**Requirements:** R1. +**Dependencies:** none. +**Files:** `crates/ffi/src/abi_version.rs` (new), `crates/ffi/src/lib.rs` (module decl), `crates/ffi/cbindgen.toml` (add `[const] allow_static_const`), `crates/ffi/include/agent_desktop.h` (regenerate), `crates/ffi/tests/c_abi_lifecycle.rs`, `crates/ffi/tests/c_header_compile.rs`. +**Approach:** `ad_abi_version() -> u32` returns a packed major version (start at `1`). `ad_init(expected_major: u32) -> AdResult` returns `Ok` when compatible, `ErrInvalidArgs` + `set_last_error` otherwise. Emit `AD_ABI_VERSION_MAJOR` from a `pub const` via cbindgen `[const] allow_static_const` (KTD1) — not hand-written — so the header and the getter share one source. No struct → no size guard; document the version-bump rule in a header comment. +**Patterns to follow:** existing `ad_*_size()` runtime-getter pattern; `set_last_error_static` for the mismatch message; `trap_panic` wrapper. +**Test scenarios:** +- `ad_abi_version()` returns the current major (`== AD_ABI_VERSION_MAJOR`). +- `ad_init(current_major)` → `AD_RESULT_OK`. +- `ad_init(current_major + 1)` and `ad_init(0)` → `ErrInvalidArgs`, and `ad_last_error_message()` is non-null. +- `c_header_compile.rs` still compiles with the new `#define`. +**Verification:** A consumer reading `ad_abi_version()` before any adapter call can branch on compatibility; mismatched-major `ad_init` fails closed. + +### U2. `ad_version` entrypoint (establishes the JSON-string output pattern) + +**Goal:** Expose `version` over the ABI; lock the command-backed output convention reused by U4–U7. +**Requirements:** R4. +**Dependencies:** none. +**Files:** `crates/ffi/src/commands/version.rs` (new), `crates/ffi/src/lib.rs`, header, `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** `ad_version(out: *mut *mut c_char) -> AdResult`. `guard_non_null!(out)`, `trap_panic`, call `commands::version::execute()`, **wrap the payload via `output::Response::ok("version", value)` and serialize the `Response`** (KTD9) — not the raw `Value` — then `string_to_c` → `*out`. No adapter, no context. On error: zero `*out`, `set_last_error`, return code. This `Response::ok` wrapping is the shared output convention U4/U5/U7 reuse. Document `ad_free_string(*out)` ownership. +**Patterns to follow:** `crates/ffi/src/convert/string.rs` (`string_to_c`, 1 MB cap); the entry template from research (`guard_non_null!` outside `trap_panic`). +**Test scenarios:** +- `ad_version(&out)` → `OK`; `out` parses as JSON with `data.version`/`data.target`/`data.os`. +- `ad_version(NULL)` → `ErrInvalidArgs`, no write. +- After success, `ad_free_string(out)` then `ad_last_error_code()` is unchanged (success doesn't clear prior errno per the documented lifetime). +**Verification:** `out` matches `agent-desktop version` JSON byte-for-byte (envelope `version` from `ENVELOPE_VERSION`). + +### U3. `CommandContext` + session plumbing on `AdAdapter` + +**Goal:** Give context-taking commands (`snapshot`/`status`/`wait`) a `CommandContext`, with opt-in session for refmap persistence. +**Requirements:** R2, R4 (enabler). +**Dependencies:** none. +**Files:** `crates/ffi/src/adapter.rs` (add `session_id: Option<String>` to `AdAdapter`; optional `ad_adapter_create_with_session(session: *const c_char)` or a setter), `crates/core/src/context.rs` (confirm minimal constructor), `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** Add `session_id: Option<String>` to `AdAdapter` (default `None`). Add a **separate constructor** `ad_adapter_create_with_session(session: *const c_char)` — not a setter, which would introduce mutable state that can race with an in-flight `ad_snapshot`. Tri-state decode the session (`NULL` = sessionless, `""` distinct, invalid UTF-8 → `ErrInvalidArgs`). Build a `CommandContext` at each call boundary from `session_id` via `CommandContext::new(...)` (confirm the exact arg list at `crates/core/src/context.rs:16` — session, trace path, trace-strict). Keep `ad_adapter_create()` working unchanged (sessionless). +**Patterns to follow:** tri-state `try_c_to_string` C-string decode (learning: identity-fingerprint §FFI rule); existing `AdAdapter`/`build_adapter()` in `crates/ffi/src/adapter.rs`. +**Execution note:** Confirm `CommandContext` minimal construction against `crates/core/src/context.rs:16` before wiring U4/U5/U7. +**Test scenarios:** +- `ad_adapter_create()` yields a sessionless adapter; context has `session_id() == None`. +- session-string constructor with `"agent-a"` → context `session_id() == Some("agent-a")`. +- invalid-UTF-8 session bytes → `ErrInvalidArgs`, no adapter leaked. +**Verification:** `ad_snapshot`/`ad_status` can construct a valid `CommandContext`; sessionless default path works for the smoke harness. + +### U4. `ad_snapshot` — full refmap pipeline + +**Goal:** One call yields the CLI-format snapshot with `@e` refs and a persisted refmap. +**Requirements:** R2. +**Dependencies:** U3. +**Files:** `crates/ffi/src/commands/snapshot.rs` (new), `crates/ffi/src/lib.rs`, header, `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** `ad_snapshot(adapter, args..., out: *mut *mut c_char) -> AdResult`. `require_main_thread()`, `guard_non_null!`, `trap_panic`. Call `crates/core/src/snapshot.rs::run_with_context()` with a `CommandContext` from `AdAdapter.session_id` (U3) so `RefStore::for_session(...).save_new_snapshot(...)` writes `~/.agent-desktop/snapshots/{id}/refmap.json`. Serialize the full envelope (`{version, ok, command:"snapshot", data:{app, ref_count, tree, ...}}`) to `*out`. Decide the minimal arg surface (app/surface/max_depth/interactive_only/compact) — flat scalars or a small size-pinned `AdSnapshotArgs` struct. +**Patterns to follow:** `run_with_context` (NOT `transform_tree`, which is `ad_get_tree`'s ref-less path); `convert/string.rs`; struct-size pinning (3 layers) if a struct is introduced. +**Test scenarios:** +- `ad_snapshot` against `MockAdapter` → `OK`; `out` has `data.ref_count >= 1` and tree nodes carry `@e` refs. +- A follow-up `ad_execute_by_ref` (U6) resolves a ref from this snapshot (integration). +- `NULL` adapter / `NULL` out → `ErrInvalidArgs`, no write. +- `PLATFORM_NOT_SUPPORTED` passthrough: stub adapter → envelope `code == "PLATFORM_NOT_SUPPORTED"`. +- Covers refmap persistence: the sessionless default context still persists the refmap and the envelope carries a `snapshot_id` (the refmap is saved regardless of whether a session is set). +**Verification:** Output matches `agent-desktop snapshot` JSON shape; refmap file exists and is loadable by a subsequent ref action. + +### U5. `ad_status` entrypoint + +**Goal:** Expose adapter health + permission state over the ABI. +**Requirements:** R4. +**Dependencies:** U3. +**Files:** `crates/ffi/src/commands/status.rs` (new), `crates/ffi/src/lib.rs`, header, `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** `ad_status(adapter, out) -> AdResult`. Call `adapter.inner.permission_report()` inline, then `commands::status::execute_with_report_with_context(&*adapter.inner, &report, &ctx)`. Serialize to `*out`. Per KTD8, if the status envelope's always-present top-level fields differ from the CLI status shape, bump `ENVELOPE_VERSION` with a `BREAKING CHANGE:` footer; otherwise no bump. +**Patterns to follow:** `crates/core/src/commands/status.rs:11`; envelope-version assertion via `ENVELOPE_VERSION`. +**Test scenarios:** +- `ad_status` against `MockAdapter` → `OK`; `out` has the status fields and envelope `version == ENVELOPE_VERSION`. +- `NULL` adapter/out → `ErrInvalidArgs`. +- Stub adapter permission nuance: document + assert that `ad_check_permissions` returns `ErrPermDenied (-1)` (not `-8`) on stubs, while other entrypoints return `-8`. +**Verification:** Matches `agent-desktop status`; envelope version asserted through the constant. + +### U6. `ad_execute_by_ref` — strict-resolution ref action + +**Goal:** Drive a ref action (`@e5` + action) with full CLI parity. +**Requirements:** R3. +**Dependencies:** U4 (needs a refmap to resolve against). +**Files:** `crates/ffi/src/commands/execute_by_ref.rs` (new), `crates/ffi/src/lib.rs`, header, `crates/ffi/tests/c_abi_lifecycle.rs`, `crates/ffi/tests/c_abi_actions.rs`. +**Approach:** `ad_execute_by_ref(adapter, ref_id: *const c_char, action: *const AdAction, out) -> AdResult`. Tri-state decode `ref_id`, then: (1) load `RefStore::for_session(ctx.session_id()).load(...)` to get the `RefEntry`; (2) build the `ActionRequest` with the **action's CLI base policy** (per KTD6: `TypeText` → `focus_fallback`, every other action → `headless`); (3) call `ref_action::execute_entry(adapter, &entry, request)` — the same `pub(crate)` core path the CLI uses (already called from `crates/ffi/src/actions/execute.rs`), which traverses strict resolve → `STALE_REF`/`AMBIGUOUS_TARGET` → live actionability preflight → dispatch → handle release. An explicit `AdPolicyKind` parameter may *elevate* to headed but must never downgrade an action below its CLI base policy. Reuse the existing `AdAction` C struct (already size-pinned). +**Patterns to follow:** `crates/ffi/src/actions/` existing ref-action wrappers; learnings keep-ffi-action-policy-aligned-with-cli + playwright-grade-desktop-reliability (the 7-step ladder). +**Test scenarios:** +- Valid `@e` ref from a U4 snapshot + click action → `OK`; effect observed (integration via MockAdapter scripted resolution). +- Stale/removed ref → `ErrStaleRef`. +- Ambiguous twins → `ErrAmbiguousTarget` with candidate summaries in `details`. +- `NULL`/invalid-UTF-8 `ref_id` → `ErrInvalidArgs` (null ≠ empty ≠ invalid). +- Policy parity: a `TypeText` action with no policy arg defaults to `focus_fallback` (behaving identically to `agent-desktop type`); other actions default to `headless`; an explicit `AD_POLICY_KIND_HEADED` elevates to headed. +- Covers AE(ref-action strict resolution). +**Verification:** Behavior matches `agent-desktop click @e5` including fail-closed on stale/ambiguous and headless-by-default. + +### U7. `ad_wait` entrypoint + +**Goal:** Expose `wait` (predicates + async appearance) over the ABI. +**Requirements:** R4. +**Dependencies:** U3. +**Files:** `crates/ffi/src/commands/wait.rs` (new), `crates/ffi/src/types/wait.rs` (new C struct), `crates/ffi/src/lib.rs`, header, `crates/ffi/tests/c_abi_layout.rs`, `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** Define a flat `AdWaitArgs` `repr(C)` struct mirroring `WaitModeArgs` (7) + `WaitPredicateArgs` + `timeout_ms` + `app` (~14 fields fully flattened; `Option` modeled as nullable pointers / sentinel). Apply the **3-layer size pinning** (Rust `const` assert + header `_Static_assert` + `c_abi_layout.rs` test) and an `ad_wait_args_size()` getter (mandatory for ctypes). Decode + call `commands::wait::execute(...)`. Heaviest marshaling — do last in this phase. **Document that `ad_wait` blocks the calling thread up to `timeout_ms`** (it sleeps internally); consumers must not call it on a thread they need responsive — on macOS the main-thread requirement compounds this (R-F). +**Patterns to follow:** `crates/ffi/src/types/action.rs` size-guard reference; learning ffi-repr-c-struct-size-pinning. +**Test scenarios:** +- `wait` with a `ms` mode → `OK` after the delay. +- predicate `actionable`/`visible`/`value` → resolves true against a scripted MockAdapter. +- timeout with unmet predicate → `ErrTimeout`, `details` carries last observed state. +- `AdWaitArgs` size/alignment/field-offset assertions pass; `ad_wait_args_size()` equals `sizeof`. +- `NULL` args/out → `ErrInvalidArgs`. +**Verification:** Matches `agent-desktop wait`; struct layout pinned at all three layers. + +### U8. `ad_set_log_callback` + tracing layer + +**Goal:** `dlopen` consumers can capture debug output. +**Requirements:** R5. +**Dependencies:** none (independent; can land anytime in Phase A). +**Files:** `crates/ffi/src/log_callback.rs` (new), `crates/ffi/src/lib.rs`, `crates/ffi/Cargo.toml` (add `tracing-subscriber` dep), `crates/core/src/trace.rs` (extract shared redaction fn), header, `crates/ffi/tests/c_abi_lifecycle.rs`. +**Approach:** `ad_set_log_callback(cb: Option<extern "C" fn(level: i32, msg: *const c_char)>) -> AdResult`. Add `tracing-subscriber` to `crates/ffi/Cargo.toml` (only `tracing` is transitively present). Store the swappable callback pointer in an `AtomicPtr` behind a `Send + Sync` newtype (KTD7) — `tracing` fires from arbitrary threads. **Install the subscriber exactly once** (`OnceLock`/`Once` on first registration): `set_global_default` is per-process, so subsequent calls only swap the guarded pointer, never re-install. **Redaction:** `sanitize_trace_value` is wired to the file writer, not a subscriber — extract its key-based redaction into a shared `crates/core/src/trace.rs` fn and apply it in the layer's `on_event` before formatting, so the `SENSITIVE_KEYS` fields never reach the callback. Never write to stdout; a callback/trace failure never fails a command. Passing `NULL` unregisters (swaps the pointer to `None`). +**Patterns to follow:** `crates/core/src/trace.rs` subscriber hookup; redaction rules (don't leak secrets to the callback — reuse trace sanitization). +**Execution note:** Resolve the thread/subscriber model against `trace.rs` before fixing the callback signature. +**Test scenarios:** +- Register a callback; a subsequent failing call delivers at least one event with a level + non-null message. +- Callback fired from a non-caller thread does not panic across the boundary (spawn a thread that emits a tracing event). +- `NULL` unregisters; no further callbacks. +- Re-registering a callback (and `NULL` then re-register) swaps the pointer without re-installing the subscriber or erroring. +- Secret-bearing fields (the `SENSITIVE_KEYS` set) are redacted in the **callback output** — asserted against the actual delivered message, not the trace file. +**Verification:** A consumer sees structured debug output; no stdout pollution; mutations still succeed when the callback errors. + +### Phase B — Proof + cross-platform parity gates + +*Delivery: wave W4 — U9 and U10 in parallel, off `main` after the entrypoints (U1–U8) merge.* + +### U9. Python ctypes smoke harness (first external consumer) + +**Goal:** Prove the ABI works from a non-Rust host and gate it in CI. +**Requirements:** R6. +**Dependencies:** U1, U2, U4 (minimum: version + abi + snapshot). +**Files:** `tests/ffi-python/smoke.py` (new), `tests/ffi-python/README.md` (new), `.github/workflows/ci.yml` (new job). +**Approach:** `ctypes.CDLL` loads the `release-ffi` dylib. **First call** is `ad_abi_version()`, asserted `== AD_ABI_VERSION_MAJOR`. Validate each `ad_*_size()` against the header sizes, then drive the **AX-independent** surface (`ad_version`, sizes, the handshake) — these need no permission and form the always-green CI gate. For the adapter path, build with the `stub-adapter` feature (KTD10) whose `build_adapter()` returns a not-supported adapter; the harness then drives `ad_adapter_create` → `ad_snapshot` and asserts a clean `PLATFORM_NOT_SUPPORTED` envelope. **Do not** assert `OK` from `ad_snapshot` on a CI runner without AX permission — the real adapter returns `PERM_DENIED (-1)` there; the real-adapter happy path is covered by the local E2E harness, not this CI gate. Declare `restype`/`argtypes` for every symbol. CI builds `--profile release-ffi -p agent-desktop-ffi` (plus `--features stub-adapter` for the adapter leg). +**Patterns to follow:** `crates/ffi/tests/common/mod.rs` extern declarations → ctypes equivalents (`c_int`, `c_char_p`, `c_void_p`); learning ffi-repr-c-struct-size-pinning (ctypes must validate sizes at import). +**Test scenarios:** +- Library loads; `ad_abi_version()` matches header major. +- Every `ad_*_size()` equals the header's `AD_*_SIZE`. +- `ad_version` returns parseable JSON with `data.version`. +- With `--features stub-adapter`, `ad_adapter_create` → `ad_snapshot` → `ad_free_string` → `ad_adapter_destroy` returns a `PLATFORM_NOT_SUPPORTED` envelope — no crash, no leak. (Real-adapter `OK` is exercised by the E2E harness, not this CI gate.) +- Missing-symbol / wrong-arity call surfaces a clear Python error (guards header/binary drift). +**Verification:** CI job is green and fails loudly if a symbol, size, or return contract drifts. + +### U10. Cross-platform parity gates + +**Goal:** Make the FFI ready for Windows/Linux adapters with zero new FFI code, enforced by CI. +**Requirements:** R8. +**Dependencies:** U1–U8 (gates cover the full surface). +**Files:** `.github/workflows/ci.yml` (header-drift gate), `scripts/update-ffi-header.sh` (reused), `crates/ffi/tests/c_abi_lifecycle.rs` (passthrough tests), `crates/ffi/include/agent_desktop.h` (doc the permission nuance). +**Approach:** +- **Header-drift gate:** a CI step installs a pinned cbindgen (e.g. `0.29.x`) and runs `cbindgen crates/ffi --config crates/ffi/cbindgen.toml --output crates/ffi/include/agent_desktop.h --verify` — cbindgen's native `--verify` exits non-zero when the committed header would differ (cleaner than regen + `git diff`). Pin the cbindgen version so a generator upgrade can't false-positive. +- **Panic-trap guard:** run the FFI integration tests (or a dedicated step) under `--profile release-ffi`, or assert that `release-ffi` keeps `panic = "unwind"` — otherwise a flip to `panic = "abort"` silently defeats every `catch_unwind` trap in the shipped dylib, and the default `test` profile wouldn't catch it. +- **Per-target builds:** confirm `release.yml`'s `build-ffi` matrix already covers macOS×2 + Linux×2 + Windows×1; add a smoke `cargo build --profile release-ffi` per target if not gated on PR. +- **Passthrough tests:** for every new entrypoint, call it against a `not_supported()` adapter path and assert the JSON envelope carries `"code": "PLATFORM_NOT_SUPPORTED"` with a non-empty `suggestion`. Document the `ad_check_permissions` → `ErrPermDenied (-1)` exception (stub `permission_report()` returns `Denied`, not absent). +**Patterns to follow:** learning playwright-grade-desktop-reliability (core owns contract, adapters supply evidence); `error_code_to_result` mapping in `crates/ffi/src/error.rs`. +**Test scenarios:** +- Header-drift gate fails on an intentionally stale header (verified once locally), passes when regenerated. +- Each new `ad_*` entrypoint against a stub/not-supported adapter → `PLATFORM_NOT_SUPPORTED` envelope (except `ad_check_permissions` → documented `-1`). +- `release-ffi` builds on all five targets. +**Verification:** The same `ad_*` surface compiles and returns structured not-supported errors on non-macOS today; when a real adapter lands, the surface lights up unchanged. + +### Phase C — Codegen migration (last) + +*Delivery: wave W5 — U11 alone, off `main` after W4 merges.* + +### U11. `build.rs` codegen for the command-backed family + exhaustiveness guard + +**Goal:** Replace the hand-written command-backed wrappers (U2/U4/U5/U6/U7) with generated ones so CLI↔FFI parity is automatic. +**Requirements:** R7. +**Dependencies:** U2, U4, U5, U6, U7 (must be proven by hand first), U9 (harness proves equivalence post-migration). +**Files:** `crates/ffi/src/descriptor.rs` (new `CommandDescriptor` + per-command policy table), `crates/ffi/build.rs` (codegen step), `crates/ffi/src/generated/ffi_commands.rs` (new, committed), `crates/ffi/tests/codegen_exhaustiveness.rs` (new guard), `.github/workflows/ci.yml` (codegen drift gate), `crates/ffi/include/agent_desktop.h` (preserve hand-written `#define`/`_Static_assert` block). +**Approach:** Introduce a minimal `CommandDescriptor` (name, arg-decode, **per-command call-site template**, `policy`) for the command-backed family — *not* a single `execute` fn pointer, since the call sites differ (KTD2: `version` no-arg, `status` precomputes `permission_report`, standard `execute(args,adapter,ctx)`). The generator emits, per command, the matching call site + the KTD9 `Response::ok` wrapping + the KTD6 policy, into a **committed** `generated/ffi_commands.rs` (KTD4), in a deterministic (alphabetical) emit order. The command universe is the `commands/mod.rs` pub-mod set (KTD3), never a `*.rs` glob. Preserve the hand-written header augmentation by splitting the header into a cbindgen section and a manual `#define`/`_Static_assert` section the script concatenates. Add a CI step that reruns codegen and diffs the committed output. Add `codegen_exhaustiveness.rs`: it shares the same `mod.rs`-derived universe and fails when a command-backed module has no generated wrapper, and pins each command→policy mapping. +**Patterns to follow:** learnings exhaustiveness-guards-over-catch-alls (machine-derived command universe + per-case pins), deterministic-build-artifact-marker (committed output, stable drift path), preserve-command-policy-semantics (no central policy). +**Execution note:** Characterize first — assert the hand-written wrappers' outputs are equivalent before and after migration. Because `snapshot_id`/timestamps are non-deterministic (`new_snapshot_id()` = time+random+counter), the harness **masks volatile fields** (snapshot_id, any timestamp) to a sentinel before diffing; equivalence holds on the masked form. +**Test scenarios:** +- Generated `ad_<name>` output equals the hand-written wrapper (volatile fields masked) for `version`/`snapshot`/`status` (characterization). +- The `mod.rs`-derived universe excludes helper modules (`helpers`, `wait_mode`, `point_resolve`) — no phantom `ad_*` wrapper is generated for them. +- Exhaustiveness guard fails when a new command file is added without a descriptor/wrapper. +- Policy pins: `type`-family → `focus_fallback`; all others → `headless`. +- Codegen drift gate fails on a stale committed `generated/ffi_commands.rs`. +- The hand-written `AD_*_SIZE`/`_Static_assert` header block survives regeneration. +**Verification:** Adding a new command-backed CLI command auto-produces its `ad_<name>` (or fails CI), with no hand-edited wrapper and no policy drift. + +--- + +## Cross-Platform Extension (the Windows/Linux question) + +**Answer: yes — by construction, with zero new FFI code.** The FFI calls `core`, which dispatches through the `PlatformAdapter` trait. The Windows and Linux crates today carry an empty `impl PlatformAdapter for {Platform}Adapter {}`, so all ~25 trait methods inherit `Err(AdapterError::not_supported(...))` → `ErrorCode::PlatformNotSupported` → `AdResult::ErrPlatformNotSupported (-8)` at the boundary. Every `ad_*` call therefore already returns a structured not-supported error on those platforms. When Phase 2 (Windows) and Phase 3 (Linux) implement the trait, the **same** `ad_*` surface lights up — no per-platform FFI wrappers (docs/phases.md §1034 confirms Phase 3 adds none; §993 notes a Windows FFI cdylib already ships). + +This plan makes that future safe rather than assumed: +- **U10 header-drift gate** keeps the one committed header correct across all targets. +- **U10 per-target `release-ffi` builds** prove the cdylib compiles for macOS/Linux/Windows every release. +- **U10 passthrough tests** prove the not-supported envelope is correct *now*, before any adapter exists. +- **One nuance to document:** `ad_check_permissions` returns `ErrPermDenied (-1)` on stub platforms (the default `permission_report()` returns `Denied`, not absent), while every other entrypoint returns `-8`. Cross-platform callers should treat both as "unavailable here." +- **One caveat (R-G):** passthrough is automatic for adapter-method calls, but `ad_snapshot`'s `RefStore` persistence uses `std::fs` + advisory locking, whose semantics differ on Windows. That path is validated when the Windows adapter lands — it is not covered by `not_supported()` passthrough. + +--- + +## Risks & Dependencies + +- **R-A — Codegen wipes the hand-written header block.** The `AD_*_SIZE`/`_Static_assert` macros are manual augmentation cbindgen doesn't emit; naive regeneration loses them. *Mitigation:* KTD4 committed output + the U11 split-or-emit strategy + a test that asserts the block survives. +- **R-B — Generator flattens per-command policy.** A catch-all default would silently break `type`'s `focus_fallback`. *Mitigation:* KTD6 explicit policy table + U11 per-case policy pins (learnings). +- **R-C — Log-callback cross-thread unsafety.** `tracing` fires off-thread; a naive global pointer is a data race / use-after-free. *Mitigation:* KTD7 guarded pointer, best-effort, `NULL` unregister; resolve against `trace.rs` first. +- **R-D — `ad_snapshot` refmap persistence.** `run_with_context` saves the refmap unconditionally via `RefStore::for_session(ctx.session_id())` — the **sessionless default context still persists** it under the default namespace (KTD5). The only failure mode is calling a tree path that bypasses `run_with_context`. *Mitigation:* U3 (context plumbing) before U4; never bypass `run_with_context`. +- **R-E — CI lacks cbindgen.** The drift gate needs cbindgen on the runner. *Mitigation:* U10 installs it; if unavailable, gate degrades to the existing `c_header_compile.rs` (catches missing decls, not stale ones) and the risk is noted. +- **R-F — `ad_wait` blocks the calling thread.** It sleeps up to `timeout_ms`; a consumer calling it on a UI/main thread freezes that thread, and macOS's main-thread requirement compounds it. *Mitigation:* U7 documents that callers must run `ad_wait` off any thread they need responsive. +- **R-G — `RefStore` file-locking is not cross-platform-uniform.** `ad_snapshot`'s refmap persistence uses `std::fs` + advisory locking; Windows semantics differ (NTFS vs POSIX). The "zero new FFI code" claim holds for adapter-method passthrough but **not** automatically for `RefStore` paths. *Mitigation:* validate `RefStore` on Windows as part of the Phase 2 adapter work; flagged here so it isn't assumed solved. +- **Dependency:** `CommandContext::new` arg list (`crates/core/src/context.rs:16`) confirmed present; exact minimal construction verified in U3 before U4/U5/U7. + +--- + +## Sources & Research + +- **docs/phases.md** — P2-O16 (FFI registry migration + parity expansion, §687), `ad_abi_version` gap (§613, §632), no-`inventory` decision (§631), Phase 3 adds no FFI wrappers (§1034), Windows FFI cdylib already ships (§993). +- **Repo research (this session):** no command registry (hand-match in `src/dispatch/mod.rs`); two wrapper families; header regenerated by `scripts/update-ffi-header.sh` (not in build graph), no CI drift check; `ad_get_tree` is ref-less, `snapshot.rs::run_with_context` is the refmap path; `ad_abi_version` best as a runtime fn; stub passthrough automatic (`-8`) with the `ad_check_permissions` `-1` nuance; `release.yml build-ffi` covers 5 targets. +- **Institutional learnings (`docs/solutions/best-practices/`):** ffi-repr-c-struct-size-pinning (3-layer pinning + mandatory `ad_*_size()` for ctypes), keep-ffi-action-policy-aligned-with-cli (headless default, CI parity gate), preserve-command-policy-semantics (no central policy in shared dispatch), exhaustiveness-guards-over-catch-alls (machine-derived command universe + per-case pins), envelope-version-bump-contract (`ENVELOPE_VERSION` discipline), identity-fingerprint (tri-state C-string decode), playwright-grade-desktop-reliability (strict-resolution ladder + core-owns-contract), deterministic-build-artifact-marker (committed codegen output for stable drift checks). +- **External research (cbindgen docs via Context7 + industry prior art, 2024–2026):** validated the core FFI choices against current practice and named exemplars — **all confirmed**, with refinements folded in (cbindgen-emitted `AD_ABI_VERSION_MAJOR` per KTD1; `cbindgen --verify` drift gate; `OnceLock`/`AtomicPtr` callback per KTD7; the release-ffi panic-trap guard in U10). Exemplars: SQLite / libgit2 / Botan (version macro + runtime getter — `ad_init` fail-closed is *stronger* than their norm); libgit2 / Botan / wgpu-native (opaque handle + create/destroy + paired free); libgit2 / pact_ffi / ffi_helpers / Botan (thread-local errno last-error); libsodium (`*_size()` getters + ctypes size checks); cbindgen `--verify` (committed-header drift gate). Sources: cbindgen 0.29 docs, Rust Nomicon/Reference (panic, `C-unwind`), Effective Rust Item 34, pact_ffi, ffi_helpers, and the libgit2 / SQLite / Botan / libsodium / wgpu-native API references. + +--- + +## Builder Notes (for ce-work) + +High-signal execution guidance not fully captured in the units. **The repo is the source of truth — verify against current code; line numbers here drift.** + +**Gates — every unit passes before "done":** +- `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings` (zero warnings), `cargo test --workspace`. +- FFI also: `cargo test -p agent-desktop-ffi` — the `c_abi_layout.rs` / `c_abi_lifecycle.rs` / `c_header_compile.rs` tests MUST stay green; extend them, never break ABI layout. +- Match each unit's test scenarios to real evidence (diff + passing test); do **not** call a unit done from the diff alone. + +**FFI gotchas that will bite:** +- Build/run the cdylib with `--profile release-ffi` (panic=unwind). Default `release` is panic=abort → silently breaks every `catch_unwind`/`trap_panic` (U10 panic-trap guard). +- After touching ANY exported `ad_*` symbol or `repr(C)` struct: run `scripts/update-ffi-header.sh`, commit the regenerated `crates/ffi/include/agent_desktop.h` (committed contract; cbindgen is NOT in the build graph). +- New adapter-touching entrypoint: call `require_main_thread()` before any `adapter.inner.*` (macOS); put `guard_non_null!` OUTSIDE the `trap_panic` closure; zero out-params on error; document `ad_free_*` ownership. +- Every new `repr(C)` struct: 3-layer pin — Rust `const` assert + `c_abi_layout.rs` test + `ad_*_size()` getter (ctypes consumer needs the getter). Mirror `crates/ffi/src/types/action.rs`. + +**Verified facts — do NOT re-derive or assume:** +- `version::execute()` (and the other command fns) return the DATA payload only — wrap via `output::Response::ok(command, data)` (KTD9) or FFI output won't match the CLI envelope. +- `status` → `execute_with_report_with_context(adapter, &report, &ctx)` (precompute `adapter.inner.permission_report()`). +- `type_text` base policy is `focus_fallback`, not headless (U6 / KTD6). +- Command universe = the `pub mod` set in `crates/core/src/commands/mod.rs`, NOT a `commands/*.rs` glob (helper/`wait_*`/`*_tests` files exist there). +- `crates/ffi/Cargo.toml` lacks `tracing-subscriber` — add `tracing-subscriber.workspace = true` for U8. +- `build_adapter()` always builds the real macOS adapter → CI needs the `stub-adapter` feature (KTD10); never assert `OK` from `ad_snapshot` on a permission-less runner (it returns `PERM_DENIED -1`, not `-8`). + +**Confirm-before-coding (verify against code first):** +- `CommandContext::new` exact arg list — `crates/core/src/context.rs`. +- `ad_set_log_callback` subscriber/thread model — `crates/core/src/trace.rs` (extract the shared redaction fn there). + +**Sequencing is strict:** U3 → U4/U5/U7; U4 → U6. **Codegen (U11) is LAST** — hand-write, prove, and characterize the entrypoints (with volatile-field masking) before mechanizing; do not start U11 early. + +**Repo conventions:** 400 LOC/file hard limit; one command/struct per file; zero `unwrap()` in non-test code; no inline comments (only `///`); conventional commits, **no `Co-Authored-By` / AI attribution**; the pre-commit hook runs fmt/clippy/test — don't bypass it. + +**This plan stays local — never `git add` / commit anything under `docs/plans/`.** diff --git a/docs/plans/2026-06-28-macos-core-hardening-validated.json b/docs/plans/2026-06-28-macos-core-hardening-validated.json new file mode 100644 index 0000000..1d644e9 --- /dev/null +++ b/docs/plans/2026-06-28-macos-core-hardening-validated.json @@ -0,0 +1,1496 @@ +{ + "workList": [ + { + "id": "macos-tree-02", + "file": "crates/macos/src/tree/resolve_search.rs", + "line": "60", + "severity": "P2", + "category": "deadline / cancellation", + "title": "Role-attribute fetch in element_at_path runs with app-level timeout, not remaining-deadline timeout, on intermediate path nodes", + "description": "In element_at_path, each loop iteration calls ensure_before_deadline(deadline)? (line 59) and then immediately calls copy_string_attr(¤t, kAXRoleAttribute) (line 60). At this point current is a child element cloned from the previous iteration. No element-specific messaging timeout has been set on it; the first AX IPC call will use the application-level 2.0 s timeout set by element_for_pid. set_messaging_timeout is called inside resolve_children (line 199), which runs after the role fetch. For a deadline with little remaining budget (e.g., 10 ms), ensure_before_deadline passes, then the role fetch blocks for up to 2.0 s before the AX IPC resolves, overshooting the deadline by up to ~2 s per path step.", + "why_it_matters": "A deep path (e.g., 10 steps) can overshoot a tight deadline by up to 20 s in an extreme case. Practically, every path step that starts within 2 s of the deadline will overshoot. The timeout propagation pattern (prepare_for_read -> set_messaging_timeout -> AX call) that is used correctly in resolve_roots.rs is not applied to child elements in element_at_path.", + "suggested_fix": "Before copy_string_attr at line 60, call set_messaging_timeout(¤t, remaining_before_deadline(deadline)?). This mirrors the resolve_children pattern and bounds the role fetch to the remaining deadline budget.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed real. In element_at_path (resolve_search.rs:50-68), line 59 calls ensure_before_deadline then line 60 calls copy_string_attr(¤t, kAXRoleAttribute) without setting a messaging timeout on current. The set_messaging_timeout call is inside resolve_children at line 199, which is called at line 61 AFTER the role fetch. Child elements inherit the application-level 2.0s timeout set by element_for_pid, so the role fetch at line 60 can block for up to 2.0s past any tight deadline. The correct pattern (prepare_for_read before first IPC call) used in resolve_roots.rs is missing here.", + "fix_approach": "Before the copy_string_attr call at line 60, call set_messaging_timeout(¤t, remaining_before_deadline(deadline)?) to bound the role fetch to the remaining budget. This mirrors the resolve_children pattern.", + "fix_files": [ + "crates/macos/src/tree/resolve_search.rs" + ], + "area": "macos-tree", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Before the copy_string_attr call at line 60, call set_messaging_timeout(¤t, remaining_before_deadline(deadline)?) to bound the role fetch to the remaining budget. This mirrors the resolve_children pattern." + }, + { + "id": "macos-tree-06", + "file": "crates/macos/src/tree/resolve_deadline.rs", + "line": "25-28", + "severity": "P3", + "category": "deadline / cancellation", + "title": "sleep_before_retry sleeps the entire remaining budget when less than 75 ms remain, making the next retry a no-op", + "description": "sleep_before_retry calls std::thread::sleep(remaining.min(Duration::from_millis(75))). When the remaining budget is, say, 40 ms, the function sleeps 40 ms. The subsequent attempt in resolve_element_with_timeout begins with zero budget, hits ensure_before_deadline immediately, and returns Err(Timeout). The final retry provides no additional search coverage.", + "why_it_matters": "With the default 5 s timeout and 4 attempts, each retry waits up to 75 ms. If all three intermediate sleeps consume the budget, the fourth attempt is wasted. More importantly, if the deadline is nearly exhausted after the first attempt (e.g., a slow initial tree walk), the remaining attempts are silently skipped via immediate timeout, with no log or counter to reveal why.", + "suggested_fix": "Only sleep if sufficient time remains for a meaningful retry: 'if let Ok(remaining) = remaining_before_deadline(deadline) { if remaining > Duration::from_millis(100) { std::thread::sleep(remaining.min(Duration::from_millis(75))); } }'. Alternatively, use a fixed inter-retry delay that is not derived from the remaining budget.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed real. sleep_before_retry (resolve_deadline.rs:25-28) uses remaining.min(75ms), so if fewer than 75ms remain, it sleeps the entire remaining budget. The caller in resolve_element_with_timeout (resolve.rs:55, 62, 80) then starts the next iteration, immediately hits ensure_before_deadline, and returns Timeout. The final retry provides no search coverage. With a 5s total budget and 4 attempts this matters mainly when the first pass consumes nearly all the budget (slow tree walk). The impact is wasted retry attempts with no log; P3 is correct.", + "fix_approach": "Only sleep when sufficient budget remains for a meaningful attempt: if remaining > Duration::from_millis(100) { std::thread::sleep(remaining.min(Duration::from_millis(75))); }. Alternatively, use a fixed inter-retry delay that is not derived from the remaining budget.", + "fix_files": [ + "crates/macos/src/tree/resolve_deadline.rs" + ], + "area": "macos-tree", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Only sleep when sufficient budget remains for a meaningful attempt: if remaining > Duration::from_millis(100) { std::thread::sleep(remaining.min(Duration::from_millis(75))); }. Alternatively, use a fixed inter-retry delay that is not derived from the remaining budget." + }, + { + "id": "macos-tree-07", + "file": "crates/macos/src/tree/builder.rs", + "line": "82-117", + "severity": "P3", + "category": "builder traversal", + "title": "build_subtree bypasses ancestor cycle-detection at ABSOLUTE_MAX_DEPTH, generating a spurious leaf for a cycle node", + "description": "When raw_depth >= ABSOLUTE_MAX_DEPTH (50), build_subtree returns a leaf AccessibilityNode immediately, before reaching the 'let ptr_key = el.0 as usize; if !ancestors.insert(ptr_key) { return None; }' cycle-detection block at line 118. If an AX element at exactly depth 50 is an ancestor of the current traversal path, it will generate a leaf instead of being suppressed. Additionally the element is not inserted into ancestors at this depth, so a hypothetical sibling path that reaches the same element via a different route also builds a leaf rather than being deduplicated. Since all children are vec![] for this leaf, no recursion occurs and no actual infinite loop results.", + "why_it_matters": "A cyclic AX element at the depth cap produces a phantom tree node (with children_count reporting 0 via the cap branch, while the real count may be non-zero) instead of being cleanly suppressed. This is a semantic inconsistency: nodes above the cap are correctly cycle-guarded; the cap boundary node itself is not.", + "suggested_fix": "Move the ptr_key check before the raw_depth >= ABSOLUTE_MAX_DEPTH guard, or add 'if !ancestors.contains(&(el.0 as usize))' before building the capped leaf, and insert + remove around the early return to maintain the invariant.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed real. build_subtree (builder.rs:83-117) returns a leaf when raw_depth >= ABSOLUTE_MAX_DEPTH before reaching the cycle-detection block at line 119. An element at exactly depth 50 that is also an ancestor in the current traversal path generates a phantom leaf with children: vec[] and possibly a wrong children_count (from count_children) instead of being suppressed. No infinite recursion results since the leaf has no children. Requires a cyclic AX tree (corrupted app), so reachable only in theory. P3 is correct.", + "fix_approach": "Move the ptr_key cycle check before the raw_depth >= ABSOLUTE_MAX_DEPTH guard, or add if ancestors.contains(&(el.0 as usize)) { return None; } as a short-circuit before building the capped leaf. Insert and remove ptr_key around the early return to maintain the invariant for sibling paths.", + "fix_files": [ + "crates/macos/src/tree/builder.rs" + ], + "area": "macos-tree", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Move the ptr_key cycle check before the raw_depth >= ABSOLUTE_MAX_DEPTH guard, or add if ancestors.contains(&(el.0 as usize)) { return None; } as a short-circuit before building the capped leaf. Insert and remove ptr_key around the early return to maintain the invariant for sibling paths." + }, + { + "id": "macos-tree-08", + "file": "crates/macos/src/tree/builder.rs", + "line": "55", + "severity": "P3", + "category": "null-pointer / AX element identity", + "title": "window_element_for returns the AXApplication element when no matching window is found", + "description": "At line 55, if no window candidate is found in the windows list, the function returns 'app' — the AXApplication element obtained from element_for_pid. The caller expects an element whose subtree represents a window, but AXApplication has role AXApplication, not AXWindow. The caller (build_subtree and similar) will begin traversal at the application root, producing a tree that silently includes all windows instead of failing or narrowing to the intended one.", + "why_it_matters": "Callers that expect a window-scoped tree will receive an application-scoped tree, leading to oversized snapshots, incorrect element paths, and potential false-positive matches during resolution. There is no error signal to the caller that the window was not found.", + "suggested_fix": "Return an Option<AXElement> instead of AXElement and let callers decide how to handle a missing window. Alternatively, return the first AXWindow-role element unconditionally rather than the app element, and document that the returned element may not match win_title exactly.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed real. window_element_for (builder.rs:55) returns the AXApplication element (role AXApplication) when no AXWindow-role candidates are found. Three callers — adapter.rs:51, window_ops.rs:26, app_ops.rs:81 — pass the returned element directly to build_subtree without checking the role. The caller receives a silent application-scoped tree instead of a window-scoped one. This is reachable when an app has no AXWindow-role elements (uncommon but possible, e.g. background agents or apps using custom window roles). The failure is silent with no error signal. P3 is correct.", + "fix_approach": "Return Option<AXElement> instead of AXElement and propagate None to callers to let them emit a proper error. Alternatively, if a window element is mandatory, return Ok(first_candidate) for the fallback and document the relaxed semantics, but add a role check at call sites.", + "fix_files": [ + "crates/macos/src/tree/builder.rs", + "crates/macos/src/adapter.rs" + ], + "area": "macos-tree", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Return Option<AXElement> instead of AXElement and propagate None to callers to let them emit a proper error. Alternatively, if a window element is mandatory, return Ok(first_candidate) for the fallback and document the relaxed semantics, but add a role check at call sites." + }, + { + "id": "macos-tree-10", + "file": "crates/macos/src/tree/element.rs", + "line": "220-239", + "severity": "P3", + "category": "unsafe hygiene", + "title": "count_children wraps the entire function body in a single unsafe block including safe operations", + "description": "The unsafe block at line 220 encloses CFString::from_static_string, integer comparisons, and loop control flow in addition to the AXUIElementGetAttributeValueCount FFI call. Only AXUIElementGetAttributeValueCount requires unsafe. Wrapping safe operations in unsafe obscures which lines actually require it, makes future audits harder, and could allow a future developer to add an unsafe operation in the block without realising it needs justification.", + "why_it_matters": "Minimising unsafe scope is a Rust best practice and is particularly important for a crate that will be ported to Windows/Linux where these patterns will be reviewed and replicated. Broad unsafe blocks on non-macOS stubs of the same functions would be misleading.", + "suggested_fix": "Move the unsafe block to wrap only 'AXUIElementGetAttributeValueCount(element.0, attr.as_concrete_TypeRef(), &mut count)' and let the CFString creation, loop, and comparison remain in safe code.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed real. count_children (element.rs:220-239) wraps the entire function body in a single unsafe block including CFString::from_static_string, loop control, and integer comparisons. Only AXUIElementGetAttributeValueCount requires unsafe. Unlike macos-tree-09 (a non-macOS stub), this is live macOS production code that will be read and audited. Broad unsafe blocks obscure which lines require justification and set a pattern that will be replicated in new functions. No current UB results, but the hygiene issue is in active production code. P3 is correct.", + "fix_approach": "Extract the unsafe block to wrap only the AXUIElementGetAttributeValueCount call: let err = unsafe { AXUIElementGetAttributeValueCount(element.0, attr.as_concrete_TypeRef(), &mut count) }; Leave CFString creation, loop control, and comparisons in safe code.", + "fix_files": [ + "crates/macos/src/tree/element.rs" + ], + "area": "macos-tree", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Extract the unsafe block to wrap only the AXUIElementGetAttributeValueCount call: let err = unsafe { AXUIElementGetAttributeValueCount(element.0, attr.as_concrete_TypeRef(), &mut count) }; Leave CFString creation, loop control, and comparisons in safe code." + }, + { + "id": "macos-axerr-03", + "file": "crates/macos/src/actions/ax_helpers.rs", + "line": "251-257", + "severity": "P2", + "category": "Swallowed AX failure", + "title": "ax_error_result promotes only kAXErrorAPIDisabled; stale elements and all other AX error codes become Ok(()) and surface as Ok(false) to callers", + "description": "The private function ax_error_result converts kAXErrorAPIDisabled to a hard AdapterError::permission_denied() and returns Ok(()) for every other AX error code. This means kAXErrorInvalidUIElement (stale element), kAXErrorCannotComplete (app busy / retry exhausted), kAXErrorFailure, kAXErrorAttributeUnsupported, and kAXErrorActionUnsupported all silently become Ok(()). Callers that check the bool return (e.g., try_ax_action_retried returns Ok(false)) cannot distinguish \"attribute not supported\" from \"element vanished between snapshot and action time.\" The entire chain then exhausts its steps and falls through to a CGClick or a generic ActionNotSupported error, with no diagnostic indicating that the element ref was stale.", + "why_it_matters": "A stale element (kAXErrorInvalidUIElement) is indistinguishable from \"action not applicable\" to any chain step. The chain wastes all its AX attempts and then reports \"element may not be interactable / try mouse-click\" when the real fix is refreshing the snapshot. This misdirection increases latency and gives agents incorrect remediation guidance.", + "suggested_fix": "Promote kAXErrorInvalidUIElement to a distinct error variant (e.g., AdapterError::element_not_found with a platform detail noting the element was invalidated). Consider also logging kAXErrorCannotComplete at debug level after retries are exhausted so operators can distinguish transient app-busy from structural failure.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in ax_helpers.rs lines 251–257. `ax_error_result` promotes only `kAXErrorAPIDisabled` to a hard error; all other AX error codes (including `kAXErrorInvalidUIElement` for stale elements, `kAXErrorFailure`, `kAXErrorActionUnsupported`) become `Ok(())`. `try_ax_action_retried_or_err` therefore returns `Ok(false)` for stale elements, which is indistinguishable from 'action not applicable' to chain steps. The chain exhausts all AX attempts and falls through with 'element may not be interactable / try mouse-click', giving agents incorrect remediation guidance for a fundamentally different root cause. This is reachable whenever a snapshot ref becomes stale — a common production scenario when the target app updates its UI between snapshot and action.", + "fix_approach": "Import `kAXErrorInvalidUIElement` from `accessibility_sys` and add a distinct arm in `ax_error_result` that returns `Err(AdapterError::element_not_found(...).with_platform_detail(format!('{operation} failed: element ref is stale (kAXErrorInvalidUIElement)')))`. Optionally add a `tracing::debug!` for `kAXErrorCannotComplete` after retry exhaustion to distinguish transient-busy from structural failure. No API surface change required.", + "fix_files": [ + "crates/macos/src/actions/ax_helpers.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Import `kAXErrorInvalidUIElement` from `accessibility_sys` and add a distinct arm in `ax_error_result` that returns `Err(AdapterError::element_not_found(...).with_platform_detail(format!('{operation} failed: element ref is stale (kAXErrorInvalidUIElement)')))`. Optionally add a `tracing::debug!` for `kAXErrorCannotComplete` after retry exhaustion to distinguish transient-busy from structural failure. No API surface change required." + }, + { + "id": "macos-timeout-04", + "file": "crates/macos/src/actions/ax_helpers.rs", + "line": "166-168", + "severity": "P2", + "category": "Timeout escape hatch", + "title": "set_messaging_timeout discards the AXError return of AXUIElementSetMessagingTimeout; the 1-second budget may silently fail to apply", + "description": "set_messaging_timeout calls AXUIElementSetMessagingTimeout and wraps the result with unsafe { ... }, discarding the i32 AXError return value entirely. If the system rejects the timeout (for example, for system-process elements or elements owned by protected apps), subsequent AX calls on that element will block for the system default timeout rather than the intended 1-second cap. Chain execution calls this function on both the pid-level element and the target element before attempting any chain steps, so a silent failure here undermines the per-chain deadline mechanism established in chain.rs.", + "why_it_matters": "If the 1-second messaging timeout cannot be applied, a single unresponsive AX call can hold the chain thread for many seconds, making the overall AGENT_DESKTOP_CHAIN_TIMEOUT_MS deadline ineffective. Under high concurrency this can cause cascading blocking.", + "suggested_fix": "Capture and log the return value: let err = unsafe { AXUIElementSetMessagingTimeout(...) }; if err != kAXErrorSuccess { tracing::warn!(err, \"AXUIElementSetMessagingTimeout failed; chain calls may block beyond budget\"); }. A warning is sufficient since this is not recoverable, but the absence of any diagnostic currently makes timeout overruns very hard to diagnose.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in ax_helpers.rs line 166–168. `set_messaging_timeout` calls `AXUIElementSetMessagingTimeout` and discards the return value with no log. The function is called at chain startup (chain.rs lines 38–40) for both the pid-level element and the target element. `AXUIElementSetMessagingTimeout` is documented to fail for system-process elements and protected apps; on failure the system default timeout applies. This is reachable in production whenever the agent targets elements owned by protected apps (e.g., System Preferences panels, Finder restricted views). The consequence — chains blocking beyond the configured `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` deadline — is a real availability concern under concurrency. Diagnosing timeout overruns without this log is very difficult.", + "fix_approach": "Capture the `i32` return: `let err = unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; if err != kAXErrorSuccess as i32 { tracing::warn!(err, \"AXUIElementSetMessagingTimeout failed; AX calls may block beyond budget\"); }`. A warning is sufficient since there is no recovery action; the log is the essential addition.", + "fix_files": [ + "crates/macos/src/actions/ax_helpers.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Capture the `i32` return: `let err = unsafe { AXUIElementSetMessagingTimeout(el.0, seconds) }; if err != kAXErrorSuccess as i32 { tracing::warn!(err, \"AXUIElementSetMessagingTimeout failed; AX calls may block beyond budget\"); }`. A warning is sufficient since there is no recovery action; the log is the essential addition." + }, + { + "id": "macos-cliprestore-05", + "file": "crates/macos/src/actions/type_text.rs", + "line": "44", + "severity": "P3", + "category": "Swallowed failure / clipboard integrity", + "title": "ClipboardRestore::drop swallows restore errors silently with let _ = and no tracing log", + "description": "The RAII guard ClipboardRestore restores the user's prior clipboard in its Drop impl via let _ = self.previous.restore(). If restore() fails (e.g., pasteboard ownership was taken by another app or the prior snapshot contained types the current pasteboard owner refuses), the user's clipboard is permanently replaced with the text that was pasted, and no log entry records the failure. The intent of the guard (documented in the source comment) is specifically to prevent the pasted text from leaking into the clipboard on all exit paths, so a silent failure directly violates the guard's contract.", + "why_it_matters": "On failure, the user's original clipboard content is lost and the automation-generated text remains on the pasteboard indefinitely. In headless CI environments this is invisible; in user-facing desktop sessions it corrupts the clipboard without any trace in logs.", + "suggested_fix": "Replace let _ = self.previous.restore() with: if let Err(e) = self.previous.restore() { tracing::warn!(error = %e, \"ClipboardRestore: failed to restore prior clipboard; user clipboard may contain pasted text\"); }. Drop cannot propagate errors, so logging is the only option, but the log line is essential for diagnosability.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in type_text.rs line 44. `ClipboardRestore::drop` uses `let _ = self.previous.restore()`. `restore()` returns `Err` if `NSPasteboard writeObjects:` fails, which can happen when another app has taken pasteboard ownership (e.g., an app that monitors the clipboard). The failure is silent — no tracing, no side channel — so the user's clipboard is silently left containing the pasted automation text. The RAII guard's stated contract (documented in the source comment) is violated. Reachability is theoretical since pasteboard ownership loss during a paste is uncommon, but the consequence (lost clipboard content, no log) justifies the fix.", + "fix_approach": "Replace `let _ = self.previous.restore()` with `if let Err(e) = self.previous.restore() { tracing::warn!(error = %e, \"ClipboardRestore: failed to restore prior clipboard; pasted text may remain on pasteboard\"); }`. Drop cannot propagate errors; logging is the only option and is essential for diagnosability.", + "fix_files": [ + "crates/macos/src/actions/type_text.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace `let _ = self.previous.restore()` with `if let Err(e) = self.previous.restore() { tracing::warn!(error = %e, \"ClipboardRestore: failed to restore prior clipboard; pasted text may remain on pasteboard\"); }`. Drop cannot propagate errors; logging is the only option and is essential for diagnosability." + }, + { + "id": "macos-focus-06", + "file": "crates/macos/src/actions/dispatch.rs", + "line": "33", + "severity": "P3", + "category": "Swallowed AX failure / headless-first", + "title": "ensure_app_focused silently swallowed with let _ = at four physical-input call sites before keyboard and mouse synthesis", + "description": "ensure_app_focused is called with let _ = and the error discarded at dispatch.rs:33 (before physical click), type_text.rs:23 (before keyboard/clipboard paste), scroll.rs:98 (before keyboard scroll), and scroll.rs:117 (before mouse-wheel scroll). In all four cases, physical input synthesis proceeds even if focus acquisition failed. The type_text.rs site is largely backstopped by the ax_focus_or_err call at line 25 (which propagates failure), and the scroll.rs:107 keyboard path checks AXFocused set success inline. However, dispatch.rs:33 (click) and scroll.rs:117 (mouse-wheel) have no downstream check; mouse events may be delivered to the previously focused application rather than the intended target.", + "why_it_matters": "Physical mouse and keyboard events synthesized against the wrong application produce undetected automation errors. The agent reports the action as successful while the event was consumed by a different process.", + "suggested_fix": "Promote ensure_app_focused failures to at least tracing::warn before the synthesis call. For the click and mouse-wheel paths where there is no downstream focus verification, consider returning an error if focus acquisition fails: let () = crate::system::app_ops::ensure_app_focused(pid)?;", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at four sites: dispatch.rs line 33 (before CGClick), type_text.rs line 23 (before keyboard/clipboard), scroll.rs line 98 (before keyboard scroll), scroll.rs line 117 (before mouse-wheel scroll). All four use `let _ =`. Crucially, in dispatch.rs `click_via_bounds`, `ensure_app_focused` is called only after passing the explicit policy gate at line 26 (`if !policy.allow_cursor_move || !policy.allow_focus_steal { return Err(...) }`). The scroll.rs mouse-wheel path at line 115 is similarly gated. The type_text.rs path has a downstream `ax_focus_or_err` backstop at line 25. The 'events to wrong app' scenario therefore requires both a policy gate to be passed AND `ensure_app_focused` to fail silently — a narrow window. The issue is a missing `tracing::warn` on the most sensitive paths (click and mouse-wheel), not a full correctness hole. P3 is appropriate.", + "fix_approach": "On the click and mouse-wheel paths (dispatch.rs line 33 and scroll.rs line 117) where there is no downstream focus check, add a warning if focus acquisition fails: `if let Err(e) = crate::system::app_ops::ensure_app_focused(pid) { tracing::warn!(error = %e, \"ensure_app_focused failed before physical input; events may target wrong app\"); }`. For type_text.rs the downstream `ax_focus_or_err` already propagates errors, so no change needed there.", + "fix_files": [ + "crates/macos/src/actions/dispatch.rs", + "crates/macos/src/actions/scroll.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "On the click and mouse-wheel paths (dispatch.rs line 33 and scroll.rs line 117) where there is no downstream focus check, add a warning if focus acquisition fails: `if let Err(e) = crate::system::app_ops::ensure_app_focused(pid) { tracing::warn!(error = %e, \"ensure_app_focused failed before physical input; events may target wrong app\"); }`. For type_text.rs the downstream `ax_focus_or_err` already propagates errors, so no change needed there." + }, + { + "id": "macos-pasteverify-07", + "file": "crates/macos/src/actions/type_text.rs", + "line": "131", + "severity": "P3", + "category": "Chain verification / swallowed failure", + "title": "verify_paste_effect returns Ok(()) when after.is_none(), treating an unreadable post-paste value as success with no verification", + "description": "verify_paste_effect (lines 130-141) returns Ok(()) when before.is_none() || after.is_none() || before != after. The after.is_none() branch is intended for secure text fields whose values cannot be read back. However, this branch fires for any element that returns None from readable_value after the paste, including fields that have become stale or whose AXValue reading fails for any transient reason. In those cases, the paste is reported as successful even though there is no evidence the content changed.", + "why_it_matters": "A paste into a field that ignores the clipboard (e.g., an intercepting web component) will return None for after if the field's AX value is not directly readable, and verify_paste_effect will incorrectly report Ok(()). The agent proceeds as though the text was entered when the field may be empty.", + "suggested_fix": "Tighten the guard by checking whether the field is a known-unreadable type before treating None as a verification pass: if before.is_none() || after.is_none() { return Ok(()); } is already the current behavior. The actual improvement is to only allow the after.is_none() bypass for elements where is_secure_text_field() returned true (already computed in the caller), and surface a warning for other None cases to flag unexpected readability loss.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in type_text.rs lines 130–141. `verify_paste_effect` returns `Ok(())` when `before.is_none() || after.is_none()`, not only for secure text fields. `readable_value()` returns `None` for `is_secure_text_field(el)=true` explicitly, and also whenever `copy_value_typed(el)` returns `None` (stale element, web component without AXValue, etc.). The problematic scenario: `before = Some('old text')` (field was AX-readable pre-paste), `after = None` (field became unreadable — e.g., element ref went stale between the two reads) → function returns `Ok(())` even though paste success is unverifiable. For a web component that always returns `None` from AXValue, both `before` and `after` are `None` so the check is harmless. The concerning case is the stale-after-paste path, which is reachable in production. The fix suggested (gating the `None` bypass on `is_secure_text_field`) requires threading that flag as a parameter.", + "fix_approach": "Pass a boolean `is_secure: bool` parameter to `verify_paste_effect`. Return `Ok(())` unconditionally only when `is_secure` is true. When `is_secure` is false and `after.is_none()`, emit `tracing::warn!(\"verify_paste_effect: post-paste value became unreadable; paste success unverifiable\")` before returning `Ok(())`. The caller `type_via_clipboard_paste` already has access to `is_secure_text_field(el)` and can pass it directly.", + "fix_files": [ + "crates/macos/src/actions/type_text.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Pass a boolean `is_secure: bool` parameter to `verify_paste_effect`. Return `Ok(())` unconditionally only when `is_secure` is true. When `is_secure` is false and `after.is_none()`, emit `tracing::warn!(\"verify_paste_effect: post-paste value became unreadable; paste success unverifiable\")` before returning `Ok(())`. The caller `type_via_clipboard_paste` already has access to `is_secure_text_field(el)` and can pass it directly." + }, + { + "id": "macos-triple-08", + "file": "crates/macos/src/actions/chain_defs.rs", + "line": "193-198", + "severity": "P3", + "category": "HEADLESS-FIRST: no AX path attempted", + "title": "triple_click goes directly to physical CGEvent click with no AX semantic action attempted first", + "description": "triple_click calls crate::actions::dispatch::click_via_bounds(el, MouseButton::Left, 3, policy) immediately, with no prior AX action attempt. By contrast, double_click checks for an AXOpen action first (if caps.has_ax_action(kAXOpenAction)), attempting a headless path before falling back to physical. There is no macOS AX triple-click semantic equivalent, so this is a platform limitation rather than an oversight, but the function makes no AX attempt whatsoever and immediately fails under any headless policy (allow_cursor_move: false or allow_focus_steal: false) without a meaningful error distinguishing \"not supported headlessly\" from a policy violation.", + "why_it_matters": "When ported to Windows/Linux, any triple_click call under a headless interaction policy immediately returns a policy error with no attempted fallback. Documentation and callers should know that triple_click is unconditionally physical. During the Windows/Linux port this function will require platform-specific handling.", + "suggested_fix": "Add a clear doc comment noting this action is inherently physical and has no AX semantic equivalent. For the headless-policy error path, consider returning ErrorCode::ActionNotSupported rather than the policy-denied error so callers can distinguish capability absence from policy restriction.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in chain_defs.rs lines 193–198. `triple_click` calls `click_via_bounds` directly with no AX semantic attempt. The comparison with `double_click` (which checks `AXOpen` at line 187) is accurate and the asymmetry is real. However, there is no macOS AX semantic equivalent for triple-click — this is a platform limitation, not an oversight. `click_via_bounds` already gates on policy at line 26 of dispatch.rs, so a headless policy correctly returns a policy_denied error. The error message ('Physical click fallback is disabled by the current interaction policy') is reasonably clear. The finding's concern — that callers cannot distinguish 'capability absent' from 'policy violation' — is a minor UX issue. The finding is valid as a P3 documentation and error-clarity gap.", + "fix_approach": "Add a doc comment on `triple_click` noting it has no AX semantic equivalent on macOS and is unconditionally physical. Optionally return `ErrorCode::ActionNotSupported` with a message 'TripleClick has no AX semantic; requires headed policy' before the `click_via_bounds` call when `!policy.allow_cursor_move || !policy.allow_focus_steal`, to distinguish capability absence from policy restriction more clearly than the generic policy_denied message.", + "fix_files": [ + "crates/macos/src/actions/chain_defs.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add a doc comment on `triple_click` noting it has no AX semantic equivalent on macOS and is unconditionally physical. Optionally return `ErrorCode::ActionNotSupported` with a message 'TripleClick has no AX semantic; requires headed policy' before the `click_via_bounds` call when `!policy.allow_cursor_move || !policy.allow_focus_steal`, to distinguish capability absence from policy restriction more clearly than the generic policy_denied message." + }, + { + "id": "macos-clipleak-09", + "file": "crates/macos/src/actions/type_text.rs", + "line": "53", + "severity": "P3", + "category": "Clipboard integrity / data exposure", + "title": "type_via_clipboard_paste writes text to the system pasteboard before Cmd+V; clipboard managers can capture it before ClipboardRestore runs", + "description": "type_via_clipboard_paste writes the target text to the system pasteboard at line 53 (crate::input::clipboard::set(text)) and then synthesizes Cmd+V. ClipboardRestore is constructed at line 52 and will restore the prior clipboard on all Rust exit paths. However, between the set() call and the restore(), any active clipboard manager (Alfred, Raycast, Pasta, Clipboard Manager) running in the user's session can snapshot the new clipboard contents. The restore() call at Drop time returns the system pasteboard to the prior state, but cannot un-capture what a clipboard manager already recorded. This means the pasted text (which may contain passwords or sensitive automation payloads) permanently enters the clipboard history.", + "why_it_matters": "For automation involving credentials, form auto-fill, or sensitive data, the clipboard-paste path inadvertently persists the secret into third-party clipboard history. This is unpreventable at the Rust level without disabling clipboard managers, but callers and documentation should be explicit about this risk.", + "suggested_fix": "Document this limitation in the function-level doc comment. For environments where clipboard leakage is unacceptable (e.g., password fields), type_via_ax_value is the correct path. Consider refusing the clipboard-paste fallback for secure text fields (is_secure_text_field() returns true) and returning an explicit error rather than silently using the pasteboard for credentials.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in type_text.rs lines 49–62. `type_via_clipboard_paste` writes text to the system pasteboard at line 53 before Cmd+V. `ClipboardRestore` at line 52 restores the prior clipboard on Rust exit paths but cannot un-capture what a clipboard manager (Alfred, Raycast, Pasta, etc.) recorded during the window. The path is reached for non-ASCII text when `type_via_ax_value` fails. Critically, `AXSecureTextField` maps to role `'textfield'` via `roles.rs` line 6, so `is_text_target()` returns `true` for secure fields. `type_via_ax_value` then attempts `ax_set_value` and, for `is_secure_text_field()=true`, returns `Ok(())` without verifying the write — so secure fields normally succeed via the AX path. However, if `AXValue` is not settable on a particular password field implementation (e.g., some web-backed inputs), `type_via_ax_value` fails, and non-ASCII text would reach `type_via_clipboard_paste`. The risk is real and unrepairable at the Rust level for clipboard managers, but the guard and documentation improvements are achievable.", + "fix_approach": "Add to `type_via_clipboard_paste` a guard: if `is_secure_text_field(el)` return `Err(AdapterError::policy_denied('Clipboard paste refused for secure text fields to prevent credential capture by clipboard managers; use set-value for secure fields'))` before writing to pasteboard. Add a function-level doc comment noting that active clipboard managers will capture the intermediate pasteboard state for non-ASCII text. This prevents the worst case (password in clipboard history) without breaking the non-ASCII non-secure use case.", + "fix_files": [ + "crates/macos/src/actions/type_text.rs" + ], + "area": "macos-actions", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add to `type_via_clipboard_paste` a guard: if `is_secure_text_field(el)` return `Err(AdapterError::policy_denied('Clipboard paste refused for secure text fields to prevent credential capture by clipboard managers; use set-value for secure fields'))` before writing to pasteboard. Add a function-level doc comment noting that active clipboard managers will capture the intermediate pasteboard state for non-ASCII text. This prevents the worst case (password in clipboard history) without breaking the non-ASCII non-secure use case." + }, + { + "id": "macos-input-01", + "file": "crates/macos/src/input/keyboard.rs", + "line": "93-94", + "severity": "P1", + "category": "stuck-key / modifier symmetry", + "title": "synthesize_key_state(down=true): modifier already pressed but not released on subsequent press failure", + "description": "Inside the `down` branch of the closure (lines 92-96), each modifier is pressed via `post_checked(sys_wide, modifier_keycode(m), true, 0, 1)?`. If any press fails, `?` exits the closure immediately. Modifiers that were successfully pressed in earlier iterations remain held with no release path — no call to `release_modifiers` exists on this branch. CFRelease is correctly called (line 106) but that only frees the AX object; modifier state in the HID system is not cleaned up.", + "why_it_matters": "Any subsequent keystroke, including unrelated user input, fires against a system with one or more modifier keys logically held down. Depending on the modifier combination, the target application interprets the next keypress as a chorded command (e.g., stuck Cmd means the next 'c' pastes as Cmd+C, quit, close, etc.) until the user manually re-presses and releases the modifier.", + "suggested_fix": "Track pressed modifiers in a local Vec before the closure (as `synthesize_key_for_element` already does at line 177), and on error call `release_modifiers(sys_wide, &pressed)` before propagating the error.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in keyboard.rs lines 91-96. The down-branch iterates modifiers with `post_checked(...)?`. On failure the closure exits immediately via `?` with no cleanup. Compare synthesize_key (lines 42-53) which calls `release_modifiers` before returning on error. synthesize_key_state has no such call; already-pressed modifiers remain held system-wide until the user physically presses and releases them.", + "fix_approach": "Before the closure declare `let mut pressed: Vec<_> = Vec::new();`. After each successful post_checked push the modifier. On `?`-exit call `release_modifiers(sys_wide, &pressed)` before propagating. Mirrors the pattern in synthesize_key_for_element lines 177-183.", + "fix_files": [ + "crates/macos/src/input/keyboard.rs" + ], + "area": "macos-input", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Before the closure declare `let mut pressed: Vec<_> = Vec::new();`. After each successful post_checked push the modifier. On `?`-exit call `release_modifiers(sys_wide, &pressed)` before propagating. Mirrors the pattern in synthesize_key_for_element lines 177-183." + }, + { + "id": "macos-input-02", + "file": "crates/macos/src/input/keyboard.rs", + "line": "99-100", + "severity": "P1", + "category": "stuck-key / modifier symmetry", + "title": "synthesize_key_state(down=false): early `?` exit on modifier-release failure leaves remaining modifiers pressed", + "description": "The `else` branch (lines 97-102) releases modifiers in reverse order via `post_checked(..., false, 0, 1)?`. The `?` operator causes an early return on the first release failure. All modifiers later in the iteration order are never sent a key-up event. Because modifier state is session-global and persists beyond this function call, those modifiers remain held.", + "why_it_matters": "This is the release half of a held-key gesture (called from `PlatformAdapter::key_event`). A partial release leaves the keyboard in a corrupted state system-wide: the stuck modifiers affect every subsequent keypress in any application until the user physically touches the physical key.", + "suggested_fix": "Use a best-effort release loop that collects but does not short-circuit on errors: iterate all modifiers, accumulate the first error, and propagate after the loop completes. Pattern already used correctly in `synthesize_key_for_element` lines 187-192.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in keyboard.rs lines 97-103. The else-branch iterates modifiers.iter().rev() releasing each via `post_checked(...)?`. The `?` exits the closure on the first failure, leaving remaining modifiers held. This is the mirror of finding 01: the key-down path accumulated modifier state; the key-up path must release all of it even when individual posts fail.", + "fix_approach": "Remove `?` from the modifier-release loop in the else-branch. Collect the first error into `let mut err: Result<(), _> = Ok(());` and set it on first failure without breaking. Return the accumulated error after the loop. Exactly the pattern used in synthesize_key_for_element lines 187-193.", + "fix_files": [ + "crates/macos/src/input/keyboard.rs" + ], + "area": "macos-input", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Remove `?` from the modifier-release loop in the else-branch. Collect the first error into `let mut err: Result<(), _> = Ok(());` and set it on first failure without breaking. Return the accumulated error after the loop. Exactly the pattern used in synthesize_key_for_element lines 187-193." + }, + { + "id": "macos-input-04", + "file": "crates/macos/src/input/keyboard.rs", + "line": "185, 194-196", + "severity": "P1", + "category": "stuck-key", + "title": "synthesize_key_for_element: char key remains stuck when post_char_key key-up fails; system-wide fallback only releases modifiers", + "description": "`post_char_key` (line 185) sends key-down then key-up via `post_checked`. If key-down succeeds and key-up fails, `key_result` is Err and the key is logically held. Lines 187-192 release modifiers (best-effort, correctly). Lines 194-196 then call `release_modifiers_system_wide(&pressed)` as a last-resort — but `release_modifiers_system_wide` sends only modifier key-up events. There is no call to send a key-up for `key_code` itself. The character key remains held system-wide after the function returns.", + "why_it_matters": "The char key being held triggers auto-repeat in the focused application until the user physically presses another key or the OS clears the state. For destructive keys (Delete, Escape, arrow keys used for selection) this can cause data loss or unexpected navigation.", + "suggested_fix": "After `post_char_key` returns Err, check whether the error came from key-up specifically (or unconditionally attempt a key-up via `post_event(el.0, key_code, false)`). Mirror this in the system-wide fallback path.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in keyboard.rs lines 185-197. When post_char_key fails on key-up (key-down succeeded so the char key is logically held), key_result is Err. Lines 187-193 release modifiers correctly. Lines 194-196 call release_modifiers_system_wide(&pressed) as a last resort but that function sends key-up events only for modifier codes. There is no key-up sent for key_code itself. A stuck destructive key (Delete=51, Escape=53, arrow keys) triggers auto-repeat in the focused application, causing data loss or unexpected navigation.", + "fix_approach": "After post_char_key returns Err, attempt a best-effort key-up for key_code via a system-wide AX element. Add a helper release_key_system_wide(key_code: u16) that allocates AXUIElementCreateSystemWide, posts the key-up, and CFReleases.", + "fix_files": [ + "crates/macos/src/input/keyboard.rs" + ], + "area": "macos-input", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "After post_char_key returns Err, attempt a best-effort key-up for key_code via a system-wide AX element. Add a helper release_key_system_wide(key_code: u16) that allocates AXUIElementCreateSystemWide, posts the key-up, and CFReleases." + }, + { + "id": "macos-input-05", + "file": "crates/macos/src/input/clipboard.rs", + "line": "34-43", + "severity": "P1", + "category": "clipboard restore", + "title": "ClipboardSnapshot::restore clears the pasteboard before write; if writeObjects fails, original content is permanently destroyed", + "description": "`restore()` calls `clear_pasteboard(pb)` (line 37) unconditionally before calling `write_objects(pb, self.items)` (line 38). If `write_objects` returns `false`, the function returns `Err(\"writeObjects: failed\")` (line 39). At this point the pasteboard has already been wiped; the original content that the snapshot was meant to preserve is permanently gone. The caller (e.g., `ClipboardRestore::drop` in `type_text.rs:44`) can only observe an error, but the content is unrecoverable. The same pattern exists in `set()`'s failure path (lines 87-92): when `write_string` fails and `previous.restore()` is called, a second `clear_pasteboard` is issued before `write_objects`, and if that also fails, the error from `set()` is returned while the clipboard is blank with no indication to the caller.", + "why_it_matters": "Any transient failure of NSPasteboard during the restore phase — which can occur due to app focus changes, permission dialogs, or pasteboard server hiccups — silently erases clipboard content the user has not saved anywhere else. The error message returned ('writeObjects: failed') does not communicate that data was destroyed.", + "suggested_fix": "Snapshot the raw type/data bytes at capture time (deep copy per item per type-string), then write brand-new `NSPasteboardItem` objects populated from those bytes at restore time. This decouples restore from the now-invalidated item proxies and eliminates the clear-then-fail hazard because the write is done with data fully in hand before touching the pasteboard.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": true, + "live_repro_plan": "1. Copy sentinel string 'SAVED_CONTENT' to clipboard. 2. Focus a TextEdit document text field. 3. Issue a type command with non-ASCII text (e.g. 'café') to trigger type_via_clipboard_paste. 4. Revoke Accessibility permission for the agent mid-flight to force writeObjects to fail. 5. Read clipboard after error is returned. Expected: 'SAVED_CONTENT' is present. Buggy: clipboard is empty because clearContents ran before writeObjects failed.", + "rationale": "Confirmed in clipboard.rs lines 34-43. clear_pasteboard(pb) is called unconditionally at line 37, then write_objects at line 38. If write_objects returns false the function returns Err, but the pasteboard has already been wiped. The original items are unrecoverable. The set() function at lines 82-95 compounds this: on write failure it calls previous.restore(), which issues a second clearContents before write_objects, meaning a failure during restore also leaves the pasteboard blank with no data-loss indication.", + "fix_approach": "Deep-copy each NSPasteboardItem's data at capture time: iterate pasteboardItems, for each item iterate its types, call [item dataForType:] to retrieve raw NSData, and store (type-string, NSData) pairs. At restore time allocate fresh NSPasteboardItem instances, populate from the stored bytes, then call writeObjects. With data fully in hand before touching the pasteboard, clearContents is safe because the write cannot fail due to missing data.", + "fix_files": [ + "crates/macos/src/input/clipboard.rs" + ], + "area": "macos-input", + "empirical_result": "NOT_TESTABLE", + "final_verdict": "KEEP", + "regression_risk": "low", + "regression_notes": "The proposed fix changes capture from \"retain the NSArray of NSPasteboardItems\" to \"deep-copy each item's data as (UTI-string, NSData) pairs and reconstruct fresh NSPasteboardItems at restore time.\" Risks examined: (1) Lazy/promise-based clipboard content (Finder file copies use NSFilePromiseProvider): [item dataForType:] forces synchronous materialization at capture time; if the data provider is still live this works, if not the type is silently dropped. Under the current code these types are also unreliable post-clearContents, so this is not a regression of working behavior. (2) High-volume UTI representations: deep-copying all representations at every type_via_clipboard_paste call increases transient memory use, but the window is brief. (3) Currently-working flow: happy-path test confirmed clipboard IS restored after a successful type command. The fix keeps the same restore call sequence (clearContents then writeObjects with fresh items) so the success path is structurally identical. No currently-working behavior is broken by the fix.", + "safe_fix_approach": "In ClipboardSnapshot::capture(), after calling retain_pasteboard_items, iterate the NSArray of NSPasteboardItem objects. For each item, call [item types] to get the NSArray of UTI type strings, then for each type call [item dataForType:] to obtain the raw NSData; skip (do not store) any type for which dataForType: returns nil (lazy types that cannot be eagerly resolved). Store a Vec of (String, Vec<u8>) pairs — the UTI type name and the copied bytes — in place of the raw Id.\n\nIn restore(), allocate fresh NSPasteboardItem objects (one per original item, or one per captured (type, data) pair), call [item setData:forType:] for each pair, then call clearContents followed by writeObjects with the freshly populated items. Because all data is in Rust-owned memory before clearContents runs, the write cannot fail due to stale item references or expired lazy providers; if writeObjects still fails (OS fault) the in-memory copy is still intact for a retry.\n\nThe set() error-path restore (line 89) automatically benefits from the same fix since it calls ClipboardSnapshot::restore(). No changes needed in type_text.rs or ClipboardRestore::drop.", + "evidence": "Static code analysis confirms the bug structure exactly as described. In crates/macos/src/input/clipboard.rs:\n\nrestore() (lines 34-43):\n line 37: clear_pasteboard(pb); // clears unconditionally\n line 38: if !self.items.is_null() && !write_objects(pb, self.items) {\n line 39: return Err(AdapterError::internal(\"NSPasteboard writeObjects: failed\"));\n // pasteboard is empty at this point; original content is gone\n\nset() (lines 82-95):\n line 87: clear_pasteboard(pb); // first wipe\n line 88: if !write_string(pb, text) {\n line 89: let _ = previous.restore(); // calls clear_pasteboard again (line 37) then write_objects\n // double-wipe if restore also fails\n\nHappy-path test (observed):\n cmd: printf 'SAVED_CONTENT' | pbcopy\n cmd: agent-desktop type @e2 \"café\" --snapshot s32r1740c1pwh3 --headed\n result: {\"command\":\"type\",\"data\":{\"post_state\":{\"role\":\"textfield\",\"value\":\"café\"}},\"ok\":true}\n after: pbpaste => \"SAVED_CONTENT\" (correctly restored via ClipboardRestore RAII drop)\n\nFailure-path test: NOT_TESTABLE. The proposed repro (revoke Accessibility permission mid-flight) does not affect writeObjects: — NSPasteboard operations run through the pasteboard server (pbs) and are independent of TCC Accessibility permissions. Revoking Accessibility would only fail AXUIElementPostKeyboardEvent, not the NSPasteboard write. Forcing writeObjects to return false requires either (a) lazy/promise-based clipboard content (e.g., Finder file copy), which requires Finder interaction prohibited by guardrails, or (b) a pasteboard server fault, which cannot be engineered from outside the process. No guardrail-safe method exists to trigger the writeObjects failure path." + }, + { + "id": "macos-input-06", + "file": "crates/macos/src/input/clipboard.rs", + "line": "25-43", + "severity": "P2", + "category": "clipboard restore", + "title": "Retained NSPasteboardItem objects are invalidated after clearContents; writeObjects silently restores empty items", + "description": "`capture()` retains the NSArray from `[NSPasteboard pasteboardItems]` (lines 25-30). `restore()` later calls `clearContents` then passes those same retained item objects to `writeObjects` (line 38). Per Apple's documentation and Pasteboard architecture, `NSPasteboardItem` objects obtained via `pasteboardItems` are proxy objects whose data is lazily served from the pasteboard server under the pasteboard owner's generation token. After `clearContents` changes the pasteboard owner, those items are invalidated: their internal generation token no longer matches the server's current state. A subsequent `writeObjects` call with these invalidated items succeeds (returns `YES`) but writes items with no type/data pairs — the pasteboard is restored structurally empty. `writeObjects` returning `true` is treated as success, so the caller receives no error signal.", + "why_it_matters": "Every call to `type_via_clipboard_paste` (and any future API using `ClipboardRestore`) silently wipes the user's clipboard even on the happy path. The `ClipboardRestore` guard's `drop` call in `type_text.rs:44` always 'succeeds' but always writes nothing back. The user's clipboard content is gone after every clipboard-paste-based typing action.", + "suggested_fix": "Deep-copy each item's data at capture time: iterate `pasteboardItems`, for each `NSPasteboardItem` iterate its `types`, and for each type call `[item dataForType:]` to retrieve the raw `NSData`, store the (type-string, NSData) pairs in the snapshot. At restore time, allocate fresh `NSPasteboardItem` instances, set the copied data per type, and write those new items. This is the only approach that survives a generation change.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": true, + "live_repro_plan": "1. Copy 'ORIGINAL' to clipboard. 2. Focus a TextEdit document. 3. Issue a type command with non-ASCII text (e.g. 'naïve') to trigger type_via_clipboard_paste. 4. Wait for success. 5. Read clipboard. Expected: clipboard contains 'ORIGINAL'. Buggy: clipboard is empty because retain_pasteboard_items captured proxy NSPasteboardItems that were invalidated by clearContents, and writeObjects with those stale proxies returns YES but writes no data.", + "rationale": "Confirmed in clipboard.rs lines 24-43. retain_pasteboard_items at lines 123-130 retains the NSArray returned by [NSPasteboard pasteboardItems] without deep-copying the per-item type/data bytes. After clearContents changes the pasteboard owner, the NSPasteboardItem objects lose their generation token. writeObjects with those items succeeds (returns YES) but writes structurally empty items. The caller sees no error yet the clipboard is blank on every type_via_clipboard_paste call.", + "fix_approach": "Same root fix as finding 05: deep-copy each item's data per type-string at capture time so restore is independent of the generation token.", + "fix_files": [ + "crates/macos/src/input/clipboard.rs" + ], + "area": "macos-input", + "empirical_result": "CONFIRMED", + "final_verdict": "KEEP", + "regression_risk": "low", + "regression_notes": "The fix (deep-copy each item's type-string data at capture time into owned Rust/NSData structures, then build fresh NSPasteboardItems in restore()) touches only ClipboardSnapshot::capture() and restore() in crates/macos/src/input/clipboard.rs lines 24-43. The AX value path (type_via_ax_value) is completely independent and unaffected — it never calls ClipboardSnapshot. The clipboard-get and clipboard-clear public functions do not use ClipboardSnapshot. The only reachable callers of ClipboardSnapshot are: (a) type_via_clipboard_paste in crates/macos/src/actions/type_text.rs and (b) clipboard::set() internally for error recovery. Both callers are currently broken (process crash); a correct deep-copy implementation can only improve behavior. No currently-working code path uses the restore() branch on the success path.", + "safe_fix_approach": "In capture(), replace retain_pasteboard_items() with an eager deep-copy: iterate [NSPasteboard pasteboardItems], for each NSPasteboardItem read every type string from [item types], call [item dataForType:] for each, store the (type, NSData) pairs in a Vec owned by ClipboardSnapshot. In restore(), call clear_pasteboard(pb), then for each original item create a fresh NSPasteboardItem via [NSPasteboardItem new], write each (type, data) pair via [item setData:forType:], and pass the array of new items to writeObjects:. This is fully independent of generation tokens because all data is materialized at capture time. Do NOT pass the retained proxy NSArray to writeObjects after clearContents under any circumstances.", + "evidence": "Test setup: installed release binary at target/release/agent-desktop (panic=abort, LTO, strip=true). Triggered the clipboard path by targeting a non-textfield element with SetValue action (role=incrementor @e13 in the e2e fixture), using --headed and non-ASCII text \"naïve\". This causes is_text_target() to return false, type_via_ax_value() to fail, and with --headed policy.allow_focus_steal=true, execution falls through to type_via_clipboard_paste().\n\nRun 1 (definitive): printf 'ORIGINAL' | pbcopy && agent-desktop type @e13 --snapshot s2aueoel652hfv \"naïve\" --headed -v 2>&1; EXIT=$?; pbpaste\n\nObserved log (stdout): clipboard: set 6 chars → keyboard: synthesize_key Cmd+v → thread 'main' panicked at library/panic_abort/src/lib.rs:28:5: internal error: entered unreachable code\nEXIT:134 (SIGABRT from panic=abort)\nPOST-TYPE CLIPBOARD: (empty — pbpaste returned nothing)\n\nThe clipboard was \"ORIGINAL\" before the test. After set() completed (confirmed by synthesize_key running), the clipboard had \"naïve\". The panic occurs after Cmd+V, consistent with placement inside ClipboardRestore::drop() → restore() → after clear_pasteboard(pb) runs (which is why clipboard is empty, not \"naïve\") and before write_objects(pb, self.items) completes (stale proxy items throw an ObjC exception or trigger bool ABI UB that aborts the process). Reproduced identically on slider (@e7) in a prior run. AX-path runs (TextEdit textfield, fixture multiline) show no clipboard: set in -v logs and preserve the clipboard correctly — confirming the bug is isolated to the type_via_clipboard_paste code path.\n\nKey discrepancy from finding description: the finding says writeObjects \"silently succeeds and writes nothing.\" Empirically, it crashes the process (exit:134). The clipboard is destroyed regardless — the root cause (stale proxy items after clearContents) is confirmed." + }, + { + "id": "macos-input-07", + "file": "crates/macos/src/input/keyboard.rs", + "line": "43-52", + "severity": "P2", + "category": "modifier symmetry", + "title": "synthesize_key: on partial modifier-press failure, release_modifiers fires on the full combo slice instead of only the pressed subset", + "description": "When modifier-down fails mid-loop (lines 46-52), `release_modifiers(sys_wide, &combo.modifiers)` is called with the entire `combo.modifiers` slice, not just the modifiers that were actually pressed before the failure. The function iterates in reverse and sends key-up for every modifier in the slice, including those whose key-down was never sent. In contrast, `synthesize_key_for_element` at line 177-183 correctly maintains a `pressed` Vec and only releases that subset.", + "why_it_matters": "Spurious key-up events for modifiers that were never pressed can interfere with modifier state that the user or another input source is legitimately managing. More importantly, it diverges from `synthesize_key_for_element`'s correct pattern and makes the two code paths behave differently for the same failure scenario.", + "suggested_fix": "Add a `let mut pressed: Vec<&Modifier> = Vec::new();` before the modifier loop, push each modifier after successful press, and pass `&pressed` to `release_modifiers` on error — matching the pattern already used in `synthesize_key_for_element`.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed in keyboard.rs lines 42-53. On modifier-press failure, release_modifiers(sys_wide, &combo.modifiers) is called with the full slice including modifiers whose key-down was never sent. A spurious key-up for a modifier the user is holding (e.g. user holds Cmd while agent fires Cmd+Shift+T and Shift fails) cancels the user's Cmd state. The bug is real but reachable only when AXUIElementPostKeyboardEvent returns an error for a modifier key-down, which is rare in normal operation.", + "fix_approach": "Replace the fail-fast loop with the pressed-Vec pattern from synthesize_key_for_element: maintain `let mut pressed: Vec<_> = Vec::new();`, push each modifier after a successful post, pass `&pressed` to release_modifiers on failure.", + "fix_files": [ + "crates/macos/src/input/keyboard.rs" + ], + "area": "macos-input", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace the fail-fast loop with the pressed-Vec pattern from synthesize_key_for_element: maintain `let mut pressed: Vec<_> = Vec::new();`, push each modifier after a successful post, pass `&pressed` to release_modifiers on failure." + }, + { + "id": "macos-input-08", + "file": "crates/macos/src/input/mouse.rs", + "line": "203-210", + "severity": "P2", + "category": "CGEvent correctness / UB", + "title": "set_click_count passes address of Rust wrapper struct instead of the underlying CGEventRef", + "description": "`set_click_count(event: &CGEvent, ...)` casts the argument as `event as *const CGEvent as *const c_void` (line 206). `CGEvent` is a `foreign_type!` newtype (verified in core-graphics-0.25.0/src/event.rs) wrapping `*mut sys::CGEvent`. The cast produces a void pointer equal to the stack address of the CGEvent Rust wrapper object — not the `CGEventRef` (the underlying CF opaque pointer) stored inside it. The C function `CGEventSetIntegerValueField(CGEventRef event, CGEventField field, int64_t value)` expects the CF object pointer, not a pointer to the Rust struct that holds it. The value received by C is therefore the stack frame address of the wrapper variable in the caller, which is unrelated to any CF allocation. Behavior is undefined: the function operates on the wrong memory and click-state field 1 (`kCGMouseEventClickState`) is never written to the real event.", + "why_it_matters": "Multi-click synthesis (double-click, triple-click) requires `kCGMouseEventClickState` to be set on each posted event. Because the field is never set on the real `CGEventRef`, every click in a `count > 1` loop posts as a single click. Applications that distinguish double-click (text word-select, file open) from single-click (cursor placement) will receive the wrong event type. Depending on the macOS version and what `CGEventSetIntegerValueField` does when called with a stack address, this could also corrupt unrelated stack memory.", + "suggested_fix": "Replace `event as *const CGEvent as *const c_void` with `event.as_ptr() as *const c_void`. The `as_ptr()` method from the `ForeignType` trait returns the underlying `*mut sys::CGEvent` (i.e., the real CGEventRef).", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": true, + "live_repro_plan": "1. Open TextEdit and type a paragraph. 2. Issue a double-click (MouseEventKind::Click { count: 2 }) on any word. Expected: the word is selected (AXSelectedTextRange spans the word). Buggy: cursor is merely placed in the word with no selection, because the click-count field is never set on the real CGEventRef so both pairs arrive with default click-state which applications treat as single-click.", + "rationale": "Confirmed by reading foreign-types-macros-0.2.3/src/build.rs line 63: foreign_type! generates `#[repr(transparent)] struct CGEvent(NonNull<sys::CGEvent>)`. Therefore `event as *const CGEvent` (mouse.rs line 206) is the stack address of the NonNull wrapper, NOT the CGEventRef value. The CGEventRef is the value stored inside the wrapper, returned by as_ptr() (build.rs lines 112-114: `NonNull::as_ptr(self.0)`). The C function CGEventSetIntegerValueField receives a stack address and interprets it as a CF object, so the click-state field is never written to the real event. Every multi-click event is effectively single-click.", + "fix_approach": "Replace `event as *const CGEvent as *const c_void` with `event.as_ptr() as *const c_void` in set_click_count. Use the named constant EventField::MOUSE_EVENT_CLICK_STATE (value 1, defined in the same crate's event.rs) instead of the magic literal 1.", + "fix_files": [ + "crates/macos/src/input/mouse.rs" + ], + "area": "macos-input", + "empirical_result": "CONFIRMED", + "final_verdict": "KEEP", + "regression_risk": "low", + "regression_notes": "The fix touches only `set_click_count`, whose only caller is `synthesize_click` inside the `MouseEventKind::Click { count }` branch. For count=1 (single click), the fix changes the click-state field from unset/0 to explicitly 1 — applications treat click-state 1 and 0 identically as single-click, so no behavioral regression. For count≥2, the fix makes the function work as designed for the first time. No other codepath is affected. Checked: no other callers of `set_click_count` or `CGEventSetIntegerValueField` exist in the crate.", + "safe_fix_approach": "In crates/macos/src/input/mouse.rs, replace `event as *const CGEvent as *const c_void` with `event.as_ptr() as *const c_void` at line 206. Optionally replace the magic literal `1` with `core_graphics::event::EventField::MOUSE_EVENT_CLICK_STATE` (value 1, defined in the same event.rs) for clarity. `as_ptr()` is defined on the `ForeignType` trait that `CGEvent` implements, so no additional import is needed beyond what is already in scope. The `*mut sys::CGEvent` returned by `as_ptr()` converts safely to `*const c_void`.", + "evidence": "Static analysis:\n- ~/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/foreign-types-macros-0.2.3/src/build.rs lines 62-64 generate `#[repr(transparent)] struct CGEvent(NonNull<CType>)`. Lines 112-114 generate `fn as_ptr(&self) -> *mut CType { NonNull::as_ptr(self.0) }`. Therefore `event as *const CGEvent as *const c_void` is the stack address of the NonNull slot, not the CF object pointer. There is no codegen path where this is accidentally equivalent to `event.as_ptr() as *const c_void`.\n- crates/macos/src/input/mouse.rs lines 203-210: the buggy cast is confirmed present in shipped code.\n\nEmpirical tests (agent-desktop release binary against TextEdit, launched fresh):\n1. cmd: `mouse-click --xy 200,183 --count 1 --headed` → screenshot shows cursor placed between letters in \"quick\", AX query `AXSelectedText=[]` — single click correctly places cursor with zero selection.\n2. cmd: `mouse-click --xy 200,183 --count 2 --headed` (identical coords, same 40ms inter-click gap, well under macOS 500ms threshold) → screenshot pixel-identical to single-click result (no blue highlight), AX query `AXSelectedText=[]` — word NOT selected despite double-click operation.\n3. Reference: `press cmd+a` → screenshot shows clear blue highlight over all text, AX query `AXSelectedText=[The quick brown fox]` — proves both visual and AX observables correctly detect real selections. This eliminates \"can't observe word selections\" as an alternative explanation.\n4. Screenshots saved at /tmp/textedit_after_double.png (no highlight) vs /tmp/textedit_cmd_a.png (full blue highlight).\n\nMechanism: CGEventSetIntegerValueField receives the stack address of the NonNull wrapper instead of the CGEventRef, silently no-ops (CF likely rejects the invalid object), leaving kCGMouseEventClickState at its default of 0 on both the click-down and click-up events for every iteration. TextEdit receives all events with click-state 0 and interprets them as cursor placements, never triggering word-selection on double-click." + }, + { + "id": "macos-input-09", + "file": "crates/macos/src/input/mouse.rs", + "line": "302", + "severity": "P3", + "category": "CGEvent correctness", + "title": "synthesize_scroll_at uses kCGScrollEventUnitPixel (0), making typical dy/dx values sub-perceptual", + "description": "`CGEventCreateScrollWheelEvent(std::ptr::null(), 0, 2, dy, dx)` passes units=0 which is `kCGScrollEventUnitPixel`. The sole caller (`scroll.rs:151-154`) computes `dy = amount as i32 * 5`. With pixel units and amount=1, the scroll event carries 5 pixels of scroll — typically less than a single line height on any display. The `kCGScrollEventUnitLine` constant is 1; line units interpret the same value as 5 scroll lines, which is perceptible and matches user expectation for a discrete scroll action.", + "why_it_matters": "Scroll actions driven by the agent produce imperceptibly small movements. Lists, code editors, and web pages will not visibly respond unless the caller uses very large amount values (hundreds), but there is no documentation at the call site indicating this requirement. The semantic mismatch is invisible in logs because the function succeeds without error.", + "suggested_fix": "Change the units argument from `0` to `1` (`kCGScrollEventUnitLine`) or define named constants `const PIXEL: u32 = 0; const LINE: u32 = 1;` and use `LINE` here. If pixel units are intentionally wanted for some callers, expose the unit as a parameter.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": true, + "live_repro_plan": "1. Open Safari at a long page. 2. Issue a scroll Down with amount=1 via the agent (falls through to synthesize_scroll_at at scroll.rs lines 119-125). 3. Observe page movement. Expected (kCGScrollEventUnitLine=1): visible scroll of ~5 lines. Buggy (kCGScrollEventUnitPixel=0, confirmed in scroll.rs lines 150-155 producing amount*5 pixel deltas): 5 physical pixels of movement, imperceptible on any Retina display. Confirm with amount=10 to make 50px vs 50 lines unambiguous.", + "rationale": "Confirmed in mouse.rs line 302 (units=0 = kCGScrollEventUnitPixel) and scroll.rs scroll_wheel_delta (lines 150-155) which produces -(amount*5) for Down and +(amount*5) for Up. With pixel units and default amount=1 the event carries 5 pixels, which is sub-perceptible. The scroll action path (scroll.rs lines 115-128) passes these pixel-unit deltas as the last-resort fallback when all AX-based scroll mechanisms fail.", + "fix_approach": "Change the units argument from 0 to 1 (kCGScrollEventUnitLine) in mouse.rs line 302. Define named constants `const PIXEL_UNIT: u32 = 0; const LINE_UNIT: u32 = 1;` for clarity, or use the CGScrollEventUnit type alias already exported from the core-graphics crate.", + "fix_files": [ + "crates/macos/src/input/mouse.rs" + ], + "area": "macos-input", + "empirical_result": "NOT_TESTABLE", + "final_verdict": "KEEP", + "regression_risk": "low", + "regression_notes": "The fix (`0` → `1`) touches only the last-resort CGEvent fallback in synthesize_scroll_at. All AX-based scroll paths (AXIncrement/AXDecrement, AXScrollDownByPage, value-shift, sub-elements, keyboard keycode) are completely unchanged. The E2E scroll test in tests/e2e/run.sh exercises the fixture's AX-backed NSScrollView and observes offset change via the scroll-offset status label — it never reaches the CGEvent path and is unaffected. The unit tests for scroll_wheel_delta (scroll.rs:329-330) assert numeric values `(0, 10)` and `(0, -10)` for the returned deltas; those values are unchanged by the fix. No caller can depend on the current 5-physical-pixel-per-unit behavior because it is currently imperceptible and functionally broken.", + "safe_fix_approach": "Change the literal `0` to `1` at crates/macos/src/input/mouse.rs:302: `CGEventCreateScrollWheelEvent(std::ptr::null(), 1, 2, dy, dx)`. Optionally define `const PIXEL_UNITS: u32 = 0; const LINE_UNITS: u32 = 1;` above the call for clarity. No other changes are strictly required for correctness; however, consider whether the `* 5` multiplier in scroll_wheel_delta is still appropriate with line units (amount=3 default → 15 lines, which is fine; amount=10 used in E2E → 50 lines, a large but unlikely path since the fixture's AX scroll path fires first). The multiplier adjustment is optional and separate from the unit fix.", + "evidence": "Static analysis of crates/macos/src/input/mouse.rs:302 confirms: `CGEventCreateScrollWheelEvent(std::ptr::null(), 0, 2, dy, dx)` — the literal `0` is kCGScrollEventUnitPixel. crates/macos/src/actions/scroll.rs:150-154 confirms `scroll_wheel_delta` returns `amount as i32 * 5` pixels: with amount=1, the event carries 5 physical pixels. The defective CGEvent path lives at scroll.rs:115-128, behind the guard `policy.allow_focus_steal && policy.allow_cursor_move` (--headed flag), and is only reached after all AX-based scroll methods fail. Guardrail-allowed targets (TextEdit, Finder, Calculator, E2E SwiftUI fixture NSScrollView) all have working AXIncrement/AXDecrement support, so step 1 of the fallback chain succeeds and synthesize_scroll_at is never called. Observing the defect requires a target where AX scroll fails (e.g., Safari web content, WKWebView), which is not permitted under the current guardrails. Live repro was not possible without a forbidden target." + }, + { + "id": "macos-system-01", + "file": "crates/macos/src/notifications/nc_session.rs", + "line": "147-154", + "severity": "P1", + "category": "swallowed-error", + "title": "open_nc discards osascript result; caller receives a misleading timeout error", + "description": "open_nc spawns `/usr/bin/osascript` to click the ControlCenter Clock menu-bar item and stores the result with `let _ = crate::system::process::run_with_timeout(...)`. If the osascript fails — because ControlCenter has been renamed (macOS 15 restructured this item), Automation permission is denied, or the UI hierarchy changed — the error is completely discarded. The function then returns `Ok(())` unconditionally after a fixed 500 ms sleep. wait_for_nc_ready then polls for 2 s and surfaces AdapterError::timeout('Notification Center did not open within 2 seconds') with no causal context about why it failed.", + "why_it_matters": "Every notify operation (list, dismiss, action) routes through NcSession::open. A silent failure here produces an opaque timeout error with no diagnostic path for the operator; the real cause — a changed UI label, a denied Automation permission, or a crashed ControlCenter — is permanently lost.", + "suggested_fix": "Propagate the osascript result with `?`. Change the call to `crate::system::process::run_with_timeout(&mut command, \"osascript open-nc\", ...)` without `let _ =`, or at minimum bind to a variable and emit `tracing::warn` with the stderr on non-success status before proceeding.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at nc_session.rs lines 148-152: `let _ = crate::system::process::run_with_timeout(...)` followed by unconditional `Ok(())` at line 154. When osascript fails (Automation TCC denied, ControlCenter UI label changed), NC never opens and wait_for_nc_ready polls for 2 s before surfacing a timeout error. The root cause is permanently discarded with no tracing::warn. P1 is inflated: a real error IS returned (the timeout); the loss is causal context, not silently swallowing the failure. P2 is correct — every notification operation is blocked but the caller still gets an error, just an opaque one.", + "fix_approach": "Replace `let _ = run_with_timeout(...)` with `if let Err(e) = crate::system::process::run_with_timeout(...) { tracing::warn!(\"open_nc osascript failed: {e}\"); }`. For stronger hygiene, propagate with `?` and extend wait_for_nc_ready's timeout error to carry the osascript stderr as a platform detail.", + "fix_files": [ + "crates/macos/src/notifications/nc_session.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace `let _ = run_with_timeout(...)` with `if let Err(e) = crate::system::process::run_with_timeout(...) { tracing::warn!(\"open_nc osascript failed: {e}\"); }`. For stronger hygiene, propagate with `?` and extend wait_for_nc_ready's timeout error to carry the osascript stderr as a platform detail." + }, + { + "id": "macos-system-02", + "file": "crates/macos/src/notifications/list.rs", + "line": "11-15", + "severity": "P1", + "category": "swallowed-error", + "title": "list_notifications discards a successful Vec when session.close() fails", + "description": "list_notifications calls `session.close()?` on line 13 AFTER binding the list result on line 12 with `let result = list_from_nc(filter)`. If list_from_nc succeeds (returns Ok(vec)) but close_nc() inside session.close() fails (e.g., AXUIElementCreateSystemWide() returns null or AXUIElementPostKeyboardEvent returns an error), the `?` propagates the close error and discards the valid list result. The equivalent operations in actions.rs correctly use the close_session helper which prioritises the action result over the close error, but list.rs does not use that pattern.", + "why_it_matters": "A caller expecting a list of notifications gets an opaque keyboard-synthesis error instead of successfully enumerated notifications. The window for close() to fail is narrow but real, and the divergence from the pattern in actions.rs is a latent correctness hazard when close_nc is changed.", + "suggested_fix": "Mirror the actions.rs close_session pattern: call session.close() unconditionally, capture both errors, and prefer the action error: if result.is_ok() and close_result.is_err() return the close error; if result.is_err() return the action error regardless of close_result.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at list.rs lines 11-14: `let result = list_from_nc(filter); session.close()?; result`. If close_nc fails (AXUIElementPostKeyboardEvent returns a non-success error), the `?` propagates the keyboard error and the successfully-collected Vec is discarded. actions.rs correctly uses close_session which prefers the action error over the close error. The divergence is real and the fix is minimal. However P1 is inflated: close_nc synthesises a single Escape keystroke and can only fail if AX is fundamentally broken — yet list_from_nc, which makes many more AX calls, already succeeded. The window for this to matter is extremely narrow, making it P2 not P1.", + "fix_approach": "Extract and apply the same close_session helper already present in actions.rs: `let close_result = session.close(); match (result, close_result) { (Ok(v), Ok(())) => Ok(v), (Ok(_), Err(e)) => Err(e), (Err(e), _) => Err(e) }`. No new logic needed — just reuse the existing pattern.", + "fix_files": [ + "crates/macos/src/notifications/list.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Extract and apply the same close_session helper already present in actions.rs: `let close_result = session.close(); match (result, close_result) { (Ok(v), Ok(())) => Ok(v), (Ok(_), Err(e)) => Err(e), (Err(e), _) => Err(e) }`. No new logic needed — just reuse the existing pattern." + }, + { + "id": "macos-system-04", + "file": "crates/macos/src/system/permissions.rs", + "line": "63", + "severity": "P1", + "category": "permission-detection", + "title": "Automation permission hardcoded NotRequired while several code paths require it", + "description": "permissions::report() and request_report() both set `automation: PermissionState::NotRequired`. However, multiple operations unconditionally call `/usr/bin/osascript` to control 'System Events': frontmost_app (nc_session.rs:52), open_nc (nc_session.rs:142), reactivate_app (nc_session.rs:77), and close_app_impl for the graceful-quit path (app_ops.rs:238). On macOS 10.14+ these calls require the Automation TCC permission (System Preferences > Privacy > Automation > System Events). If the user has not granted it, osascript fails with error -1743 or 'Not authorized to send Apple events'. Because the permission report says NotRequired, users will not be directed to fix the root cause.", + "why_it_matters": "A user who runs 'check-permissions' and sees everything green will still get cryptic 'Notification Center did not open within 2 seconds' or 'Failed to request graceful quit' errors because the actual blocking permission is invisible in the report.", + "suggested_fix": "Add an `automation` entry to the permission detection layer. On macOS 10.14+ the appropriate probe is to run a trivial osascript (`osascript -e 'tell application \"System Events\" to return name'`) and check exit status, or use the AEDeterminePermissionToAutomateTarget() private API. Set PermissionState::Granted or Denied accordingly.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at permissions.rs lines 63 and 74: both report() and request_report() hard-code `automation: PermissionState::NotRequired`. Code inspection confirms that frontmost_app (nc_session.rs:56), open_nc (nc_session.rs:148), reactivate_app (nc_session.rs:80), and the close_app_impl osascript path (app_ops.rs:238) all invoke /usr/bin/osascript targeting 'System Events'. On macOS 10.14+ every osascript that addresses System Events requires the Automation TCC entitlement. A user who runs check-permissions and sees a clean report can still hit 'Notification Center did not open within 2 seconds' with zero actionable guidance because the actual blocking permission is invisible in the report. P1 stands.", + "fix_approach": "Add an automation probe in imp: run `osascript -e 'tell application \"System Events\" to return name'` with a short timeout and check exit status. If non-zero, set PermissionState::Denied with a suggestion pointing to System Settings > Privacy & Security > Automation. Populate the automation field in both report() and request_report().", + "fix_files": [ + "crates/macos/src/system/permissions.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add an automation probe in imp: run `osascript -e 'tell application \"System Events\" to return name'` with a short timeout and check exit status. If non-zero, set PermissionState::Denied with a suggestion pointing to System Settings > Privacy & Security > Automation. Populate the automation field in both report() and request_report()." + }, + { + "id": "macos-system-05", + "file": "crates/macos/src/notifications/nc_session.rs", + "line": "163-173", + "severity": "P1", + "category": "physical-input-policy", + "title": "close_nc synthesizes a physical system-wide Escape key event with no focus guarantee", + "description": "close_nc calls keyboard::synthesize_key with the Escape key. synthesize_key calls AXUIElementCreateSystemWide() and posts the key event to that system-wide element via AXUIElementPostKeyboardEvent. This posts the Escape key to whichever application holds keyboard focus at the moment the HID event is processed — not necessarily the Notification Center. The NC close path in NcSession::close calls close_nc() BEFORE reactivate_app(), but there is a window between close_nc initiating and the event arriving where focus can shift. Additionally, close_nc contains a hard-coded 300 ms sleep with no readiness poll to verify NC actually closed.", + "why_it_matters": "On a loaded system or when NC dismiss is slow, the synthesized Escape can land in the user's foreground application (cancelling a dialog, closing a modal, discarding an edit). The 300 ms fixed sleep is also non-deterministic: NC may not have closed, leaving it open.", + "suggested_fix": "Focus the NotificationCenter window directly via AX (AXRaise on the NC window element) before synthesizing Escape, or use an AX action on the NC window directly instead of a system-wide keyboard event. Replace the 300 ms sleep with a poll on is_nc_open() with a bounded deadline.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at nc_session.rs lines 163-173: close_nc calls keyboard::synthesize_key(Escape) which uses AXUIElementCreateSystemWide() and AXUIElementPostKeyboardEvent — a system-wide event to whatever holds focus. The finding's concern is real: there is no AX focus guarantee. However, at the point close_nc is invoked from NcSession::close, reactivate_app has NOT yet run (line 29 is after line 26), so NC remains the frontmost application. In practice the Escape lands in NC. The race exists but requires NC to lose focus between the user operation completing and close_nc executing on the same blocking thread — extremely unlikely. P1 is inflated; P2 is appropriate. The 300 ms sleep is a genuine non-determinism issue.", + "fix_approach": "Before posting Escape, use AXRaise on the NC window element to guarantee NC holds keyboard focus. Replace the 300 ms sleep with a bounded poll on is_nc_open() returning false, with a 500 ms deadline.", + "fix_files": [ + "crates/macos/src/notifications/nc_session.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Before posting Escape, use AXRaise on the NC window element to guarantee NC holds keyboard focus. Replace the 300 ms sleep with a bounded poll on is_nc_open() returning false, with a 500 ms deadline." + }, + { + "id": "macos-system-06", + "file": "crates/macos/src/notifications/nc_session.rs", + "line": "31", + "severity": "P2", + "category": "memory-leak", + "title": "std::mem::forget(self) in NcSession::close leaks the previous_app String allocation", + "description": "NcSession::close takes self by value, runs close_nc() and reactivate_app(), then calls std::mem::forget(self) to prevent the Drop impl from running a second close. Because forget skips the destructor, the previous_app: Option<String> field's heap buffer is never freed. This allocates and leaks a String on every NC session that was not already open. The standard approach to prevent double-invocation of cleanup in Drop is a boolean flag (e.g., `closed: bool`) set inside close() and checked at the top of drop(); that approach runs the destructor normally and frees all fields.", + "why_it_matters": "Every notification list, dismiss, or action call that opens a fresh NC session leaks a heap String. On long-running daemons or rapid automated sequences this accumulates without bound. Rust's ownership model makes this a real correctness issue even for small allocations.", + "suggested_fix": "Add a `closed: bool` field to NcSession, defaulting to false. At the start of drop() check `if self.closed { return; }`. In close(), set `self.closed = true` after running cleanup, then return (do not call forget). This lets the normal destructor free previous_app.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at nc_session.rs line 31: `std::mem::forget(self)` is called in NcSession::close after cleanup completes. NcSession holds `previous_app: Option<String>`. forget prevents the Drop impl from running, which means the heap allocation backing the String is never freed. This occurs on every NC session that was not already open — i.e., on every list/dismiss/action call that has to open NC. The memory loss per call equals the length of the frontmost app name. On a long-running daemon performing automated notification management this accumulates without bound. The fix is minimal and standard.", + "fix_approach": "Add a `closed: bool` field to NcSession (default false). In Drop::drop(), return early if `self.closed`. In close(), after running cleanup, set `self.closed = true` and return normally. Remove the mem::forget call entirely. The destructor then runs normally and frees previous_app.", + "fix_files": [ + "crates/macos/src/notifications/nc_session.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add a `closed: bool` field to NcSession (default false). In Drop::drop(), return early if `self.closed`. In close(), after running cleanup, set `self.closed = true` and return normally. Remove the mem::forget call entirely. The destructor then runs normally and frees previous_app." + }, + { + "id": "macos-system-07", + "file": "crates/macos/src/notifications/nc_session.rs", + "line": "80-84", + "severity": "P2", + "category": "swallowed-error", + "title": "reactivate_app silently discards subprocess failure with no trace log", + "description": "reactivate_app fires `/usr/bin/osascript -e 'tell application <name> to activate'` and stores the entire Result with `let _ = crate::system::process::run_with_timeout(...)`. There is no tracing::warn, tracing::debug, or any other signal emitted on failure. If the osascript fails (Automation permission revoked, ControlCenter focus transition, app name not matching a running process), the previous app is silently not reactivated and the user is left staring at the Notification Center after a dismiss operation.", + "why_it_matters": "Reactivation failure is a visible user-experience defect (focus stays on NC instead of the app the user was in). Without a trace event the only signal is the visible UI state, with no programmatic path to diagnose the cause.", + "suggested_fix": "Bind the result and log on failure: `if let Err(e) = crate::system::process::run_with_timeout(...) { tracing::warn!(\"reactivate_app osascript failed: {e}\"); }`. Alternatively make reactivate_app return Result<(), AdapterError> and let callers decide whether to propagate.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at nc_session.rs lines 80-84: `let _ = crate::system::process::run_with_timeout(...)` with no tracing::warn, tracing::debug, or any other signal. If osascript fails (Automation denied, app name not matching a running process, timeout), the previous app is silently not reactivated and the user is left looking at Notification Center. The UX effect is a visible focus defect with no diagnostic path. This is reachable whenever Automation TCC is denied or the previous app exited between session open and close.", + "fix_approach": "Bind the result: `if let Err(e) = crate::system::process::run_with_timeout(...) { tracing::warn!(\"reactivate_app osascript failed for app {:?}: {e}\", name); }`. Alternatively convert reactivate_app to return Result<(), AdapterError> and let NcSession::close log or surface the error.", + "fix_files": [ + "crates/macos/src/notifications/nc_session.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Bind the result: `if let Err(e) = crate::system::process::run_with_timeout(...) { tracing::warn!(\"reactivate_app osascript failed for app {:?}: {e}\", name); }`. Alternatively convert reactivate_app to return Result<(), AdapterError> and let NcSession::close log or surface the error." + }, + { + "id": "macos-system-08", + "file": "crates/macos/src/system/key_dispatch.rs", + "line": "164", + "severity": "P2", + "category": "deprecated-api", + "title": "AXUIElementPostKeyboardEvent is deprecated since macOS 10.11 and used throughout keyboard synthesis", + "description": "ax_post_keyboard_event (key_dispatch.rs:164) and the entire keyboard::synthesize_key family (keyboard.rs:45, 130, 184, 252) call AXUIElementPostKeyboardEvent, which Apple deprecated in macOS 10.11 (El Capitan, 2015). The API still works in current macOS releases but Apple may remove or restrict it without notice. The recommended replacement for synthesizing keyboard events is CGEventCreateKeyboardEvent + CGEventPost, which is already used in the mouse synthesis layer (mouse.rs). The deprecated API also has subtle differences in how it processes modifier state relative to CGEventPost.", + "why_it_matters": "On a future macOS release, all keyboard synthesis paths break simultaneously. The Windows/Linux port cannot use AX APIs at all, so any port work will have to replace this layer regardless, making the migration cost larger the longer it is deferred.", + "suggested_fix": "Replace AXUIElementPostKeyboardEvent with CGEventCreateKeyboardEvent + CGEventPost (kCGHIDEventTap) throughout keyboard.rs and key_dispatch.rs. The synthesize_key family in keyboard.rs becomes a thin wrapper around CGEvent, consistent with mouse.rs.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: AXUIElementPostKeyboardEvent is called throughout keyboard.rs (post_event, release_modifiers, synthesize_key, synthesize_key_state, synthesize_text, synthesize_keycode, synthesize_key_for_element) and in key_dispatch.rs ax_post_keyboard_event. The API was deprecated in macOS 10.11 (2015) and still functions in current macOS. P2 is inflated because there is no current defect: Apple has left the API in place for 10+ years without removal. This is real technical debt worthy of tracking but carries no immediate risk. P3 is the correct bucket.", + "fix_approach": "Replace AXUIElementPostKeyboardEvent with CGEventCreateKeyboardEvent + CGEventPost(kCGHIDEventTap) throughout keyboard.rs and key_dispatch.rs, consistent with the CGEvent-based approach already used in mouse.rs. This is a non-trivial migration requiring careful modifier-state handling but eliminates the deprecated dependency.", + "fix_files": [ + "crates/macos/src/input/keyboard.rs", + "crates/macos/src/system/key_dispatch.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace AXUIElementPostKeyboardEvent with CGEventCreateKeyboardEvent + CGEventPost(kCGHIDEventTap) throughout keyboard.rs and key_dispatch.rs, consistent with the CGEvent-based approach already used in mouse.rs. This is a non-trivial migration requiring careful modifier-state handling but eliminates the deprecated dependency." + }, + { + "id": "macos-system-09", + "file": "crates/macos/src/notifications/actions.rs", + "line": "99-104", + "severity": "P2", + "category": "physical-input-policy", + "title": "dismiss_entry sends a physical CGEvent mouse-move without an explicit permission pre-check", + "description": "When AX dismiss and close-button strategies fail, dismiss_entry falls through to hover_over(element) (line 99), which calls crate::input::mouse::synthesize_mouse with MouseEventKind::Move. synthesize_mouse calls CGEvent::new_mouse_event and posts it via CGEventTapLocation::HID. On macOS 10.15+ posting CGEvents to the HID tap requires accessibility permission. While accessibility is implicitly exercised by the earlier AX calls (finding the notification element, reading kAXChildrenAttribute), there is no explicit guard; on a system where accessibility is later revoked mid-session, the CGEvent post fails with an opaque internal error rather than a PermDenied code. The 300 ms thread sleep following the event is also a fixed delay that does not confirm hover registration.", + "why_it_matters": "A permission denial mid-session returns a confusing error unrelated to permissions. The mouse event also moves the physical cursor system-wide, which is more invasive than an AX action: a user mid-drag will have their cursor warped to the NC notification center coordinates.", + "suggested_fix": "Add a crate::system::permissions::report().accessibility.is_granted() guard before calling hover_over, returning AdapterError::PermDenied if not granted. Replace the 300 ms sleep with a short poll on whether the close button appears in the refreshed AXChildren array.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at actions.rs lines 99/204-218: hover_over calls synthesize_mouse(MouseEventKind::Move) which posts CGEventType::MouseMoved via CGEventTapLocation::HID, physically moving the system cursor. The permission concern is partially overstated: by the time dismiss_entry falls through to hover_over, multiple earlier AX operations (list_ax_actions, try_action_from_list, copy_ax_array, try_dismiss_button) have already implicitly verified accessibility is granted — a mid-session revocation is possible but extremely rare. The cursor-warping UX concern IS real (a user mid-drag would have their cursor teleported), but this is a last-resort path that only fires when all AX dismiss strategies have already failed. P2 is inflated; P3 is appropriate given the narrow trigger conditions.", + "fix_approach": "Add a `crate::system::permissions::report().accessibility.is_granted()` guard before hover_over to surface a clear PermDenied error on mid-session revocation. Replace the 300 ms sleep with a short AXChildren-refresh poll to confirm the close button appeared. Consider emitting a tracing::debug when falling through to the hover path since it signals AX dismiss is broken for this notification.", + "fix_files": [ + "crates/macos/src/notifications/actions.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add a `crate::system::permissions::report().accessibility.is_granted()` guard before hover_over to surface a clear PermDenied error on mid-session revocation. Replace the 300 ms sleep with a short AXChildren-refresh poll to confirm the close button appeared. Consider emitting a tracing::debug when falling through to the hover path since it signals AX dismiss is broken for this notification." + }, + { + "id": "macos-system-10", + "file": "crates/macos/src/system/key_dispatch.rs", + "line": "153", + "severity": "P3", + "category": "undocumented-behavior", + "title": "combo_to_ax_modifiers silently maps Modifier::Cmd to zero bits with no comment", + "description": "In combo_to_ax_modifiers, the match arm `Modifier::Cmd => {}` produces no bits in the output. This is correct: kAXMenuItemCmdModifiers encodes modifiers additional to Command (Shift = bit 0, Option = bit 1, Control = bit 2) because Command is the implicit base for all menu shortcuts — a menu item with only Command has modifiers == 0. However the empty arm looks like an accidental omission at a glance, particularly to anyone porting the modifier logic to Windows (where there is no such implicit base).", + "why_it_matters": "A reviewer or port author who sees `Modifier::Cmd => {}` may add `mods |= 1 << 3` as a 'fix', breaking menu-shortcut matching for all Cmd-only shortcuts (e.g., Cmd+C, Cmd+S).", + "suggested_fix": "Add an inline comment: `// Command is implicit in kAXMenuItemCmdModifiers; no bit needed.`", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at key_dispatch.rs lines 146-157: combo_to_ax_modifiers has `Modifier::Cmd => {}` producing no bits. This is CORRECT per kAXMenuItemCmdModifiers semantics — Command is the implicit base for all menu shortcuts and is not encoded in that bitmask. The finding correctly identifies that the empty arm looks like an accidental omission and could mislead a reviewer or porter into adding `mods |= 1 << 3`, which would silently break all Cmd-only shortcuts (Cmd+C, Cmd+S, etc.). The only fix needed is a single comment line.", + "fix_approach": "Add an inline comment on the empty arm: `Modifier::Cmd => {} // Command is the implicit base in kAXMenuItemCmdModifiers; no bit set here.`", + "fix_files": [ + "crates/macos/src/system/key_dispatch.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add an inline comment on the empty arm: `Modifier::Cmd => {} // Command is the implicit base in kAXMenuItemCmdModifiers; no bit set here.`" + }, + { + "id": "macos-system-11", + "file": "crates/macos/src/system/window_ops.rs", + "line": "128-136", + "severity": "P3", + "category": "swallowed-error", + "title": "AXUIElementSetAttributeValue(AXMain) fallback in raise_window discards return value silently", + "description": "raise_window first tries AXRaise via AXUIElementPerformAction. If that returns an error, it falls back to setting AXMain to true via AXUIElementSetAttributeValue. The return value of the AXUIElementSetAttributeValue call is an i32 (AXError) that is evaluated as a statement and immediately dropped — not stored with `let _ =`, not checked, not logged. If both AXRaise and AXMain fail, the function proceeds to poll wait_until_main, which will silently time out, and the window may not be raised.", + "why_it_matters": "Failures to raise a window affect subsequent click and keyboard operations that rely on the target window being at the front; errors are invisible to tracing, making diagnosis of stuck-focus bugs difficult.", + "suggested_fix": "Capture the return value and emit a tracing::debug on non-success: `let err = unsafe { AXUIElementSetAttributeValue(...) }; if err != kAXErrorSuccess { tracing::debug!(\"AXMain fallback failed err={err}\"); }`", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at window_ops.rs lines 127-136: the fallback `unsafe { AXUIElementSetAttributeValue(window.0, main_attr.as_concrete_TypeRef(), CFBoolean::true_value().as_CFTypeRef()) };` discards its i32 return value as a statement expression. If both AXRaise and the AXMain fallback fail, wait_until_main silently polls and times out, and the window is silently left un-raised. Subsequent click and keyboard operations that depend on the window being frontmost will fail for opaque reasons. This is a P3 diagnostic gap in an error-recovery fallback path, not a primary execution path.", + "fix_approach": "Capture the return value and log it: `let ax_err = unsafe { AXUIElementSetAttributeValue(...) }; if ax_err != kAXErrorSuccess { tracing::debug!(\"raise_window: AXMain fallback returned err={ax_err}\"); }`", + "fix_files": [ + "crates/macos/src/system/window_ops.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Capture the return value and log it: `let ax_err = unsafe { AXUIElementSetAttributeValue(...) }; if ax_err != kAXErrorSuccess { tracing::debug!(\"raise_window: AXMain fallback returned err={ax_err}\"); }`" + }, + { + "id": "macos-system-12", + "file": "crates/macos/src/system/workspace_apps.rs", + "line": "60-67", + "severity": "P3", + "category": "resource-management", + "title": "dlopen handle for AppKit.framework is intentionally leaked with no RAII wrapper", + "description": "appkit_loaded calls dlopen('/System/Library/Frameworks/AppKit.framework/AppKit', RTLD_LAZY) and stores only the boolean result in a OnceLock, discarding the returned handle pointer. The handle is never passed to dlclose. While AppKit is a system framework that will stay resident for the process lifetime and the OnceLock prevents double-loading, the lack of a RAII handle means there is no structured ownership of the dynamic linker reference. The OS's reference count for the library is incremented once and never decremented.", + "why_it_matters": "For the planned Windows/Linux port, this pattern does not translate; both platforms require matching their equivalents of dlclose/FreeLibrary. Flagging it now captures the pattern before it is duplicated across platform backends.", + "suggested_fix": "Wrap the raw handle in a newtype with a Drop impl that calls dlclose, and store it in the OnceLock<DlopenHandle> alongside (or instead of) the bool. Alternatively, document why the handle is intentionally discarded with a `// AppKit is a process-lifetime framework; handle deliberately not closed.` comment.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at workspace_apps.rs lines 59-68: appkit_loaded stores only `OnceLock<bool>`, discarding the raw handle returned by dlopen. On macOS, AppKit is always resident for the process lifetime so the missing dlclose has no functional consequence. The issue is code hygiene and portability: the pattern cannot translate to Windows (FreeLibrary) or Linux (dlclose). The fix is either a RAII wrapper or a comment explaining the intentional design decision. No active defect on macOS today.", + "fix_approach": "Add a comment adjacent to the OnceLock: `// AppKit is a process-lifetime system framework on macOS; the dlopen handle is intentionally not stored. Any future non-macOS port must use a RAII handle with the platform equivalent of dlclose/FreeLibrary.` Alternatively, store the handle in a newtype with a Drop impl calling dlclose.", + "fix_files": [ + "crates/macos/src/system/workspace_apps.rs" + ], + "area": "macos-system", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add a comment adjacent to the OnceLock: `// AppKit is a process-lifetime system framework on macOS; the dlopen handle is intentionally not stored. Any future non-macOS port must use a RAII handle with the platform equivalent of dlclose/FreeLibrary.` Alternatively, store the handle in a newtype with a Drop impl calling dlclose." + }, + { + "id": "macos-adapter-04", + "file": "crates/macos/src/actions/chain_steps.rs", + "line": "96", + "severity": "P2", + "category": "Result swallowed", + "title": "Value-relay rollback write error silently discarded", + "description": "When the chain's value-write verification fails, the code attempts to restore the original value with `let _ = ax_helpers::set_ax_string_or_err(&owner, \"AXValue\", o)`. The error from the rollback write is explicitly discarded with `let _ =` and the function returns `Ok(false)` either way.", + "why_it_matters": "If the rollback write itself fails, the element is left in a modified state (the new value that was being written, not the original). The caller receives Ok(false), indicating the chain step failed cleanly, when in fact the element's accessibility value is now in an unknown state. Data corruption from failed rollback is invisible to the caller and produces no diagnostic.", + "suggested_fix": "Log a warning (tracing::warn!) when the rollback set_ax_string_or_err returns Err, or escalate the error by propagating it so the caller can distinguish 'step skipped cleanly' from 'rollback failed, element state unknown'.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at chain_steps.rs:95-98. When try_value_relay's AXConfirm step fails and orig exists, the rollback write error is discarded with let _ = set_ax_string_or_err(...) and the function returns Ok(false). If the rollback write itself fails, the element is left with the new value (the one being written) rather than the original. The caller receives Ok(false) — indicating a clean skip — when in fact the element's AXValue is in an unknown state. This is a real data-corruption path that is reachable whenever try_value_relay is invoked on a dialog element and AXConfirm fails mid-write.", + "fix_approach": "At minimum, add tracing::warn!(\"value_relay: rollback write failed: {e:?}\") on the Err arm so the corruption is visible in logs. For stronger correctness, return Err(e) from the rollback branch so the chain can distinguish a clean skip from an element left in an unknown state.", + "fix_files": [ + "crates/macos/src/actions/chain_steps.rs" + ], + "area": "macos-adapter", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "At minimum, add tracing::warn!(\"value_relay: rollback write failed: {e:?}\") on the Err arm so the corruption is visible in logs. For stronger correctness, return Err(e) from the rollback branch so the chain can distinguish a clean skip from an element left in an unknown state." + }, + { + "id": "macos-adapter-05", + "file": "crates/macos/src/actions/type_text.rs", + "line": "44", + "severity": "P2", + "category": "Result swallowed in Drop — no diagnostics", + "title": "ClipboardRestore::drop silently discards clipboard restore failure", + "description": "ClipboardRestore's Drop impl uses `let _ = self.previous.restore()` with no tracing, logging, or any other diagnostic on failure. If restore() returns Err, the error is discarded and the clipboard is left in the modified (pasted) state.", + "why_it_matters": "A failed clipboard restore means the user's clipboard content is permanently overwritten by the text that was pasted, with no indication that restoration failed. The inconsistency is especially notable because NcSession::drop (nc_session.rs:39) in the same crate calls `tracing::warn!` when its cleanup action fails. Users relying on clipboard contents can lose data silently.", + "suggested_fix": "Add `tracing::warn!(err = ?e, \"clipboard restore failed after type_text\")` in the Err arm of matching self.previous.restore(), mirroring the pattern in NcSession::drop.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at type_text.rs:44. ClipboardRestore::drop silently discards restore failures with let _ = self.previous.restore(). A failed restore means the user's clipboard is permanently left with the pasted text rather than the original contents, with no log entry, no user-visible error, and no diagnostic. The inconsistency with NcSession::drop (nc_session.rs:39) which calls tracing::warn! on failure is noted in the finding and confirmed in the code. This is a real user-facing data loss path triggered whenever type_via_clipboard_paste is called (non-ASCII text on a non-headless element) and the subsequent restore fails.", + "fix_approach": "Replace let _ = self.previous.restore() with a match that calls tracing::warn!(err = ?e, \"clipboard restore failed after type_text\") on the Err arm, mirroring the pattern in NcSession::drop.", + "fix_files": [ + "crates/macos/src/actions/type_text.rs" + ], + "area": "macos-adapter", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace let _ = self.previous.restore() with a match that calls tracing::warn!(err = ?e, \"clipboard restore failed after type_text\") on the Err arm, mirroring the pattern in NcSession::drop." + }, + { + "id": "macos-adapter-07", + "file": "crates/macos/src/actions/scroll.rs", + "line": "27", + "severity": "P3", + "category": "Result swallowed — unacknowledged unsafe return", + "title": "AXScrollToVisible return value dropped without even let _ = acknowledgment", + "description": "`unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) };` — the i32 AX error code is dropped with a bare semicolon on the unsafe block. There is no `let _ =`, no binding, no logging.", + "why_it_matters": "An unacknowledged return value inside an unsafe block looks like a mistake rather than intentional. AXScrollToVisible failure (e.g., element not scrollable, permissions) is silently ignored, and the subsequent action that depends on the element being visible may fail in a harder-to-diagnose way. A missing `let _ =` also means future audits cannot distinguish 'explicitly ignored' from 'forgotten'.", + "suggested_fix": "Add `let _ = unsafe { ... };` and, if desired, a `tracing::debug!` on non-success error codes to make the best-effort intent explicit.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at scroll.rs:27: the i32 AX error code from AXUIElementPerformAction(el.0, scroll_visible) is dropped with a bare semicolon on the unsafe block. The AXScrollToVisible step is best-effort (scroll continues via other paths), but the unacknowledged unsafe return value is indistinguishable from a forgotten binding during future audits. The fix is a one-character change.", + "fix_approach": "Change the line to `let _ = unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) };` to make the intentional discard explicit. Optionally add a tracing::debug! on non-zero error codes.", + "fix_files": [ + "crates/macos/src/actions/scroll.rs" + ], + "area": "macos-adapter", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Change the line to `let _ = unsafe { AXUIElementPerformAction(el.0, scroll_visible.as_concrete_TypeRef()) };` to make the intentional discard explicit. Optionally add a tracing::debug! on non-zero error codes." + }, + { + "id": "macos-adapter-08", + "file": "crates/macos/src/system/window_ops.rs", + "line": "129-135", + "severity": "P3", + "category": "Result swallowed — unacknowledged unsafe return", + "title": "AXUIElementSetAttributeValue fallback result dropped without acknowledgment in raise_window", + "description": "In the fallback path of raise_window, when AXRaise fails, the code attempts AXUIElementSetAttributeValue(AXMain=true). The i32 return value of this unsafe call is discarded via a bare semicolon on the unsafe block — no `let _ =`, no binding, no logging. raise_window itself returns ().", + "why_it_matters": "Both the primary (AXRaise) and fallback (AXMain) paths can fail silently. The window may never be raised, yet the caller receives no indication. While the function is best-effort by design, an unacknowledged return value inside an unsafe block is indistinguishable from a forgotten binding during future audits.", + "suggested_fix": "Use `let _ = unsafe { AXUIElementSetAttributeValue(...) };` to make the discard explicit and follow the same acknowledgment pattern used elsewhere in the codebase.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at window_ops.rs:129-135: inside the raise_window fallback, AXUIElementSetAttributeValue is called inside an unsafe block whose return value is dropped with a bare semicolon. The function is documented as best-effort, so silent failure is acceptable behaviour, but the unacknowledged return in an unsafe block is indistinguishable from a forgotten binding.", + "fix_approach": "Change the call to `let _ = unsafe { AXUIElementSetAttributeValue(...) };` to make the intentional discard explicit, consistent with the codebase pattern.", + "fix_files": [ + "crates/macos/src/system/window_ops.rs" + ], + "area": "macos-adapter", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Change the call to `let _ = unsafe { AXUIElementSetAttributeValue(...) };` to make the intentional discard explicit, consistent with the codebase pattern." + }, + { + "id": "macos-adapter-09", + "file": "crates/macos/src/notifications/nc_session.rs", + "line": "76-85", + "severity": "P3", + "category": "Result swallowed", + "title": "reactivate_app discards osascript result with no logging", + "description": "reactivate_app uses `let _ = crate::system::process::run_with_timeout(&mut command, \"osascript reactivate-app\", ...)` to discard the osascript exit code and any run_with_timeout error. No tracing or logging on failure.", + "why_it_matters": "If the script fails (osascript unavailable, wrong PID, permission error), reactivate_app silently returns Ok(()). Notification center operations that depend on the app being reactivated proceed as if reactivation succeeded, making failures harder to diagnose from logs.", + "suggested_fix": "Add `if let Err(e) = run_with_timeout(...) { tracing::warn!(err = ?e, \"reactivate_app osascript failed\"); }` to provide a diagnostic trace without changing the best-effort behavior.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed at nc_session.rs:80-84. reactivate_app discards the run_with_timeout result with let _ = and no tracing. The function returns () so best-effort design is intentional, but a failed osascript (wrong app name, permissions issue, osascript not available) is completely invisible in logs. Any subsequent notification centre operation that depends on app reactivation will fail with a harder-to-diagnose error. A single tracing::warn! on failure would not change behaviour but would aid diagnostics.", + "fix_approach": "Add `if let Err(e) = run_with_timeout(...) { tracing::warn!(err = ?e, \"reactivate_app osascript failed for app={name}\"); }` to give a diagnostic trace without changing the best-effort return type.", + "fix_files": [ + "crates/macos/src/notifications/nc_session.rs" + ], + "area": "macos-adapter", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add `if let Err(e) = run_with_timeout(...) { tracing::warn!(err = ?e, \"reactivate_app osascript failed for app={name}\"); }` to give a diagnostic trace without changing the best-effort return type." + }, + { + "id": "core-trait-01", + "file": "crates/core/src/roles.rs", + "line": "10-13", + "severity": "P1", + "category": "macOS assumptions baked into core", + "title": "INTERACTIVE_ROLES contains three macOS AX-exclusive role names", + "description": "INTERACTIVE_ROLES includes \"colorwell\" (AXColorWell / NSColorWell), \"dockitem\" (AXDockItem / macOS Dock), and \"incrementor\" (AXIncrementor / NSStepper). These are macOS Accessibility API role strings with no equivalent UIA ControlType on Windows or AT-SPI role on Linux.", + "why_it_matters": "ref_alloc.rs:48-49 gates ref allocation on INTERACTIVE_ROLES.contains(&node.role) || advertises_primary_action. A Windows or Linux adapter whose adapter maps a native control to a canonical role not in this list, and whose node has no available_actions, will get zero refs — elements silently disappear from snapshots. Additionally, is_mutable_value_role (roles.rs:58) and is_expandable_role (roles.rs:52) reference \"incrementor\" and \"treeitem\", further propagating macOS role vocabulary into core decision logic.", + "suggested_fix": "Remove \"colorwell\", \"dockitem\", and \"incrementor\" from the INTERACTIVE_ROLES constant. Move them to the macOS adapter's own role-canonicalization layer. Define a shared canonical vocabulary (e.g., \"stepper\" for incrementor, \"dockbutton\" for dockitem) if the concept needs cross-platform expression.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: roles.rs lines 10/12/13 list 'colorwell', 'dockitem', 'incrementor'. The macOS adapter's tree/roles.rs maps AXColorWell→colorwell, AXDockItem→dockitem, AXIncrementor→incrementor, confirming these roles appear in production trees. The INTERACTIVE_ROLES header comment says 'Each entry must be produced by at least one platform adapter's native-to-canonical role mapping', which technically justifies their presence today. However, these are vocabulary terms from Apple's Accessibility API embedded in core logic (is_mutable_value_role uses 'incrementor'). A future Windows adapter producing 'stepper' or 'colorswatch' for the same controls gets zero refs if the element has no available_actions, because ref_alloc.rs:49 gates on INTERACTIVE_ROLES.contains OR advertises_primary_action. Severity downgraded from P1 to P2: the Windows/Linux adapters are currently stubs with no role mapping at all, so no production impact exists today.", + "fix_approach": "Move the three macOS-specific role strings out of core INTERACTIVE_ROLES and into the macOS adapter's own role-policy layer. Define a platform-neutral replacement only if the concept needs cross-platform expression (e.g., 'stepper' for incrementor). Update is_mutable_value_role in roles.rs accordingly.", + "fix_files": [ + "crates/core/src/roles.rs", + "crates/macos/src/tree/roles.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Move the three macOS-specific role strings out of core INTERACTIVE_ROLES and into the macOS adapter's own role-policy layer. Define a platform-neutral replacement only if the concept needs cross-platform expression (e.g., 'stepper' for incrementor). Update is_mutable_value_role in roles.rs accordingly." + }, + { + "id": "core-trait-02", + "file": "crates/core/src/adapter.rs", + "line": "21-30", + "severity": "P1", + "category": "macOS assumptions baked into core", + "title": "SnapshotSurface variants Sheet, Menubar, and Popover are macOS-only surface concepts", + "description": "SnapshotSurface (adapter.rs:21-30) declares Sheet (line 27), Menubar (line 26), and Popover (line 28). Sheet is an NSWindow sheet — a dialog that slides from a parent window and has no Windows/Linux equivalent. Menubar corresponds to the macOS global application menu bar, not window-embedded menus. Popover is NSPopover, which differs from Windows Popup/Tooltip in lifecycle and ownership semantics.", + "why_it_matters": "SnapshotSurface is serialized in RefEntry::source_surface (refs.rs:43) and used to build snapshot scopes (adapter.rs:44). A Windows adapter will receive SnapshotSurface values that have no native mapping, making snapshot surface selection either a no-op or a forced mapping onto incorrect variants. The enum is also not #[non_exhaustive], so adding Windows/Linux surface variants later is a breaking change.", + "suggested_fix": "Rename Sheet, Menubar, and Popover to platform-neutral names (e.g., ModalOverlay, GlobalMenuBar, FloatingPanel) with doc comments listing each platform's concrete mapping. Add #[non_exhaustive] to allow additive variants for Windows/Linux surfaces without breakage.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: adapter.rs:19-30 declares Sheet, Menubar, Popover as explicit variants. Critically, the macOS adapter's get_tree match (adapter.rs:50-63) exhaustively handles all seven SnapshotSurface values including Sheet, Menubar, Popover with real implementations. These variants ARE needed by the only production adapter. The cross-platform harm is naming: 'Sheet' is a macOS NSWindow concept, 'Menubar' is macOS's global app menu, 'Popover' is NSPopover. A Windows or Linux adapter must match these in its get_tree implementation or add new variants, both requiring platform-specific knowledge of macOS terminology. Severity downgraded from P1 to P2 because: (a) Windows/Linux are stubs and inherit the core default (not_supported), so no wrong routing happens today; (b) the enum IS used correctly on macOS.", + "fix_approach": "Rename Sheet→ModalPanel, Menubar→GlobalMenuBar, Popover→FloatingPanel with doc-comments listing each platform's concrete mapping. Add #[non_exhaustive] (see core-trait-11). Update get_tree match arms in the macOS adapter to use the renamed variants.", + "fix_files": [ + "crates/core/src/adapter.rs", + "crates/macos/src/adapter.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Rename Sheet→ModalPanel, Menubar→GlobalMenuBar, Popover→FloatingPanel with doc-comments listing each platform's concrete mapping. Add #[non_exhaustive] (see core-trait-11). Update get_tree match arms in the macOS adapter to use the renamed variants." + }, + { + "id": "core-trait-03", + "file": "crates/core/src/action.rs", + "line": "143-148", + "severity": "P1", + "category": "macOS assumptions baked into core", + "title": "Modifier enum contains only Cmd with no Win/Super/Meta variant", + "description": "The Modifier enum (action.rs:143) has Cmd, Ctrl, Alt, Shift. Cmd is the macOS Command key (⌘, VK_LWIN analogue but distinct). Windows keyboard automation requires a Win/Super modifier (the Windows key, VK_LWIN / VK_RWIN). Linux X11 / Wayland use Super (Mod4). There is no platform-neutral way to express a Windows key shortcut using current variants.", + "why_it_matters": "Key combos containing the Windows key (e.g., Win+D to show desktop, Win+L to lock) are common automation targets on Windows. A Windows adapter receiving KeyCombo { key: \"D\", modifiers: [Modifier::Win] } cannot be expressed with the current enum. Cmd cannot be repurposed as Win because macOS adapters already claim Cmd as ⌘. The enum is also not #[non_exhaustive], so adding Win later is a breaking change to every exhaustive Modifier match.", + "suggested_fix": "Add a Win (or Super/Meta) variant and mark the enum #[non_exhaustive]. Document each variant's platform mapping in the doc comment (Cmd = macOS ⌘; Win = Windows VK_LWIN; Super = Linux Mod4). Platform adapters translate canonical Modifier to native virtual key codes.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: action.rs:142-148 has only Cmd, Ctrl, Alt, Shift with no Win/Super. A Windows stub adapter (crates/windows/src/adapter.rs) exists as an empty PlatformAdapter impl, explicitly signaling cross-platform intent. Windows key shortcuts (Win+D, Win+L) are prime automation targets and cannot be expressed. Cmd cannot be reused as Win since the macOS adapter's keyboard.rs already interprets Cmd as kCGEventFlagMaskCommand. Downgraded from P1 to P2 because the Windows adapter is a pure stub; no Windows keyboard automation paths are exercisable today.", + "fix_approach": "Add Win (or Super) variant to Modifier and add #[non_exhaustive] (see core-trait-13). Add doc-comment mapping: Cmd=macOS ⌘, Win=Windows VK_LWIN, Super=Linux Mod4. Each platform adapter translates Modifier to native key codes.", + "fix_files": [ + "crates/core/src/action.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add Win (or Super) variant to Modifier and add #[non_exhaustive] (see core-trait-13). Add doc-comment mapping: Cmd=macOS ⌘, Win=Windows VK_LWIN, Super=Linux Mod4. Each platform adapter translates Modifier to native key codes." + }, + { + "id": "core-trait-04", + "file": "crates/core/src/node.rs", + "line": "83-88", + "severity": "P2", + "category": "macOS assumptions baked into core", + "title": "AppInfo::bundle_id is an Apple-exclusive concept serialized as null on non-macOS", + "description": "AppInfo (node.rs:83-88) includes pub bundle_id: Option<String> with no #[serde(skip_serializing_if = \"Option::is_none\")] annotation. CFBundleIdentifier is an Apple construct that does not exist on Windows (Win32/WPF apps have no bundle ID; UWP uses Package Family Names) or Linux (no standard equivalent).", + "why_it_matters": "Every AppInfo value produced by a Windows or Linux adapter will serialize \"bundle_id\": null into JSON output. This violates the codebase's own omit-null rule (every other Option in AppInfo and AccessibilityNode uses skip_serializing_if), pollutes the stable serialization contract with a macOS-only field, and forces downstream consumers to handle a null field that is always absent on non-macOS platforms. If the field is ever used as a lookup key, Windows/Linux callers will silently receive None and fall through to undefined behavior.", + "suggested_fix": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to bundle_id. Consider moving it to a macOS-specific extension struct returned only by the macOS adapter, keeping AppInfo cross-platform clean.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: node.rs:87 `pub bundle_id: Option<String>` has no serde attributes whatsoever. All other Option fields on AppInfo and AccessibilityNode carry #[serde(skip_serializing_if = \"Option::is_none\")]. The macOS app_inventory sources (process_apps.rs:43, window_inventory.rs:24) explicitly set bundle_id: None for process-sourced and window-inventory-sourced apps. Running list_apps on macOS today produces `\"bundle_id\": null` in JSON output for those entries. This is a current production serde rule violation that bloats serialized output and creates an inconsistent schema. Severity kept at P2.", + "fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to AppInfo::bundle_id. Both attributes needed: skip_serializing_if omits null on write; default allows round-trip deserialization of stored JSON that has the field absent.", + "fix_files": [ + "crates/core/src/node.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to AppInfo::bundle_id. Both attributes needed: skip_serializing_if omits null on write; default allows round-trip deserialization of stored JSON that has the field absent." + }, + { + "id": "core-trait-05", + "file": "crates/core/src/adapter.rs", + "line": "202-204", + "severity": "P3", + "category": "macOS assumptions baked into core", + "title": "release_handle doc comment references CFRelease/CFRetain (CoreFoundation) in the core trait", + "description": "The release_handle trait method (adapter.rs:199-207) documents \"macOS implementations must CFRelease here to balance the CFRetain that happened during resolve.\" CFRelease and CFRetain are CoreFoundation APIs exclusive to Apple platforms.", + "why_it_matters": "CoreFoundation lifecycle details belong in the macOS adapter's docs, not the core platform-neutral trait. The comment currently implies that Windows/Linux adapters are secondary (\"can leave this as the default no-op\"), framing the trait's design from a macOS perspective. Future adapter authors reading core docs should not encounter Apple-specific memory management concepts.", + "suggested_fix": "Replace the CFRelease/CFRetain mention with a platform-neutral ownership contract: \"Implementations that transfer ownership of a platform handle during resolve must release it here. The macOS adapter uses this to balance CFRetain; other adapters may leave the default no-op if their handle lifecycle does not require explicit release.\"", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "no", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: adapter.rs:199-207 doc comment says 'macOS implementations must CFRelease here to balance the CFRetain that happened during resolve. Windows/Linux consumers can leave this as the default no-op.' The macOS adapter's release_handle (macos/adapter.rs:99-108) does call CFRelease correctly. The doc is accurate for macOS but embeds Apple memory-management vocabulary in the core trait. No runtime impact; pure documentation pollution that would mislead future Windows/Linux adapter authors.", + "fix_approach": "Rewrite the doc comment to use platform-neutral ownership language: 'Implementations that take ownership of a platform handle during resolve must release it here. The default no-op is correct for adapters whose handle lifecycle requires no explicit release.' Move CFRelease/CFRetain mention to the macOS adapter's own release_handle doc.", + "fix_files": [ + "crates/core/src/adapter.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Rewrite the doc comment to use platform-neutral ownership language: 'Implementations that take ownership of a platform handle during resolve must release it here. The default no-op is correct for adapters whose handle lifecycle requires no explicit release.' Move CFRelease/CFRetain mention to the macOS adapter's own release_handle doc." + }, + { + "id": "core-trait-06", + "file": "crates/core/src/action.rs", + "line": "103-109", + "severity": "P3", + "category": "macOS assumptions baked into core", + "title": "DragParams::drop_delay_ms field doc documents macOS-only drop-target dwell behavior", + "description": "The doc comment on DragParams::drop_delay_ms (action.rs:105-109) reads: \"macOS drop targets often need the drag to dwell over them before they register as the drop destination; too short and the gesture lands as a drag with no drop.\" This is macOS-specific drag-and-drop semantics.", + "why_it_matters": "The field is on a core struct shared across all platforms. Future Windows/Linux adapter authors reading this doc will receive macOS-centric guidance. Windows drag-and-drop via IDropTarget does not require the same dwell timing; the doc risks guiding incorrect adapter behavior.", + "suggested_fix": "Generalize the comment: \"Time to hold over the destination before releasing. Some platforms require a minimum dwell before the drop target registers; None uses the adapter default.\"", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "no", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: action.rs:104-109 doc on drop_delay_ms reads 'macOS drop targets often need the drag to dwell over them before they register as the drop destination; too short and the gesture lands as a drag with no drop.' The DragParams struct is in core and shared across all adapters. The comment is factually macOS-specific; Windows IDropTarget dwell semantics differ. No runtime impact.", + "fix_approach": "Replace with platform-neutral text: 'Time to hold over the destination before releasing. Some platforms require a minimum dwell before the drop target registers; None uses the adapter default.'", + "fix_files": [ + "crates/core/src/action.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Replace with platform-neutral text: 'Time to hold over the destination before releasing. Some platforms require a minimum dwell before the drop target registers; None uses the adapter default.'" + }, + { + "id": "core-trait-11", + "file": "crates/core/src/adapter.rs", + "line": "19-30", + "severity": "P1", + "category": "enums needing additive variants but not #[non_exhaustive]", + "title": "SnapshotSurface enum is not #[non_exhaustive]", + "description": "SnapshotSurface (adapter.rs:19-30) has 7 variants and no #[non_exhaustive]. Windows surfaces that would need new variants include Tooltip, SystemTray, TaskbarThumbnail, and ContextMenu (which differs from macOS Menu in ownership and lifetime). Linux Wayland has XDG surface types with no macOS analogue.", + "why_it_matters": "SnapshotSurface is matched exhaustively in is_window (adapter.rs:33-35) and is stored in RefEntry::source_surface (refs.rs:43), which is serialized. Adding a variant breaks all existing match arms and changes the serialized snapshot format, requiring a coordinated migration.", + "suggested_fix": "Add #[non_exhaustive] to SnapshotSurface. Update SnapshotSurface::is_window to use matches! (already done) which naturally handles unknown variants. Audit other match sites.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: adapter.rs:19-30 SnapshotSurface has no #[non_exhaustive]. The is_window method (lines 32-36) already uses matches!(surface, Self::Window), so it correctly handles unknown variants and will not break when new variants are added. However, RefEntry::source_surface (refs.rs:43) is serialized to persistent JSON via the RefMap, and the macOS adapter's get_tree match (macos/adapter.rs:50-64) exhaustively handles all seven variants. Adding a Windows or Linux surface variant changes the serialized format and breaks that exhaustive match. Retained at P2 because the serialization contract and get_tree match are genuine break points, even though is_window itself is already safe.", + "fix_approach": "Add #[non_exhaustive] to SnapshotSurface. The is_window method already uses matches! (safe). Update the macOS adapter's get_tree match to add a `_ => return Err(AdapterError::not_supported(...))` arm for unknown surfaces.", + "fix_files": [ + "crates/core/src/adapter.rs", + "crates/macos/src/adapter.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[non_exhaustive] to SnapshotSurface. The is_window method already uses matches! (safe). Update the macOS adapter's get_tree match to add a `_ => return Err(AdapterError::not_supported(...))` arm for unknown surfaces." + }, + { + "id": "core-trait-12", + "file": "crates/core/src/action.rs", + "line": "128-134", + "severity": "P2", + "category": "enums needing additive variants but not #[non_exhaustive]", + "title": "WindowOp enum is not #[non_exhaustive]", + "description": "WindowOp (action.rs:128-134) has 5 variants and no #[non_exhaustive]. Windows-specific operations not yet representable include SetAlwaysOnTop (HWND_TOPMOST), SnapLeft/SnapRight (Windows Snap Layouts), and SetFullscreen. Linux compositors (KWin, Mutter) have tiling and virtual-desktop operations.", + "why_it_matters": "Adding any new WindowOp for Windows or Linux breaks all exhaustive matches in the platform adapter and command layer without #[non_exhaustive].", + "suggested_fix": "Add #[non_exhaustive] to WindowOp. Update the window_op trait method's default impl to return not_supported for unrecognized variants.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: action.rs:128-134 WindowOp enum has Resize, Move, Minimize, Maximize, Restore with no #[non_exhaustive]. The macOS adapter's window_ops::execute matches WindowOp exhaustively. Adding a Windows-specific operation (SetAlwaysOnTop, SnapLeft, SetFullscreen) would break the macOS adapter match. Caught at compile time but requires coordinated updates.", + "fix_approach": "Add #[non_exhaustive] to WindowOp. Update window_op trait default impl to add `_ => Err(AdapterError::not_supported(\"window_op\"))` for unrecognized variants. Update the macOS adapter's exhaustive match similarly.", + "fix_files": [ + "crates/core/src/action.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[non_exhaustive] to WindowOp. Update window_op trait default impl to add `_ => Err(AdapterError::not_supported(\"window_op\"))` for unrecognized variants. Update the macOS adapter's exhaustive match similarly." + }, + { + "id": "core-trait-13", + "file": "crates/core/src/action.rs", + "line": "142-148", + "severity": "P2", + "category": "enums needing additive variants but not #[non_exhaustive]", + "title": "Modifier enum is not #[non_exhaustive]", + "description": "Modifier (action.rs:142-148) has 4 variants (Cmd, Ctrl, Alt, Shift) with no #[non_exhaustive]. The missing Win/Super variant (core-trait-03) must be added; without #[non_exhaustive] every existing exhaustive match site on Modifier breaks at that addition.", + "why_it_matters": "Platform adapters that translate Modifier to virtual key codes perform exhaustive matches. Adding Win without #[non_exhaustive] requires simultaneous updates to every adapter and test that matches Modifier.", + "suggested_fix": "Add #[non_exhaustive] to Modifier and add the Win/Super variant as documented in core-trait-03.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: action.rs:142-148 Modifier enum has no #[non_exhaustive]. The macOS keyboard adapter maps Modifier variants to CGEventFlags in an exhaustive match. Adding Win/Super (as identified in core-trait-03) forces simultaneous update of every exhaustive match on Modifier across all adapters. The missing #[non_exhaustive] attribute is the mechanical prerequisite for safely adding the Win variant.", + "fix_approach": "Add #[non_exhaustive] to Modifier alongside adding the Win/Super variant (see core-trait-03). Update all exhaustive match sites in the macOS keyboard adapter to add a catch-all arm.", + "fix_files": [ + "crates/core/src/action.rs", + "crates/macos/src/input/keyboard.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[non_exhaustive] to Modifier alongside adding the Win/Super variant (see core-trait-03). Update all exhaustive match sites in the macOS keyboard adapter to add a catch-all arm." + }, + { + "id": "core-trait-14", + "file": "crates/core/src/permission_state.rs", + "line": "3-8", + "severity": "P2", + "category": "enums needing additive variants but not #[non_exhaustive]", + "title": "PermissionState enum is not #[non_exhaustive]", + "description": "PermissionState (permission_state.rs:3-8) has 4 variants and no #[non_exhaustive]. Windows UIA has no accessibility permission gate (apps get AX access automatically), so NotRequired is always used, but future Windows security features (UI Access manifest, UAC integrity levels) or Linux Flatpak portal permission flows may need new states such as PendingUserApproval or RequiresElevation.", + "why_it_matters": "PermissionReport is serialized and matched exhaustively in permission_report.rs. Adding a variant breaks callers without prior notice.", + "suggested_fix": "Add #[non_exhaustive] to PermissionState. Audit accessibility_suggestion and screen_recording_suggestion match arms in permission_report.rs to ensure they compile with an added variant.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: permission_state.rs:1-8 PermissionState has Granted, Denied{suggestion}, NotRequired, Unknown with no #[non_exhaustive]. permission_report.rs:27-43 has two exhaustive match arms (accessibility_suggestion, screen_recording_suggestion) that explicitly list all four variants. Adding a new state (e.g., PendingUserApproval for Linux Flatpak portals) would break compilation at those two match sites. Real concern given Windows/Linux stub adapters exist.", + "fix_approach": "Add #[non_exhaustive] to PermissionState. Update the two match arms in permission_report.rs (accessibility_suggestion, screen_recording_suggestion) to add a wildcard `_ => None` arm.", + "fix_files": [ + "crates/core/src/permission_state.rs", + "crates/core/src/permission_report.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[non_exhaustive] to PermissionState. Update the two match arms in permission_report.rs (accessibility_suggestion, screen_recording_suggestion) to add a wildcard `_ => None` arm." + }, + { + "id": "core-trait-15", + "file": "crates/core/src/refs.rs", + "line": "26", + "severity": "P2", + "category": "serialization-rule violations", + "title": "RefEntry::name lacks skip_serializing_if and default, serializing null", + "description": "pub name: Option<String> at refs.rs:26 has no serde attributes. Every other Option field on RefEntry immediately below (value at line 28, description at line 30) carries #[serde(default, skip_serializing_if = \"Option::is_none\")]. When name is None, the field serializes as \"name\": null. The missing #[serde(default)] also means any stored JSON that omits the name key will fail deserialization after a round-trip fix is applied.", + "why_it_matters": "RefMap is persisted to disk and reloaded across invocations (refs.rs:125 load). Null fields in persisted JSON bloat the file, violate the codebase's established omit-null convention, and make the schema unstable across adapter platforms where name is always absent.", + "suggested_fix": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to name. Both attributes are required: skip_serializing_if omits null on write; default allows reading stored JSON that predates the fix (where the field was present as null or absent).", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: refs.rs:26 `pub name: Option<String>` has no serde attributes. The immediately following value field (line 27-28) and description field (line 29-30) both carry #[serde(default, skip_serializing_if = \"Option::is_none\")]. When name=None, RefEntry serializes `\"name\": null` into the persisted RefMap file. This is a current production issue on macOS: refs for elements without meaningful names (skeleton anchors, role-only entries) emit null into the 1 MB capped RefMap file, wasting space and violating the codebase's established omit-null convention.", + "fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::name. The default attribute is required for round-trip deserialization of existing stored RefMaps that contain `\"name\": null`.", + "fix_files": [ + "crates/core/src/refs.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::name. The default attribute is required for round-trip deserialization of existing stored RefMaps that contain `\"name\": null`." + }, + { + "id": "core-trait-16", + "file": "crates/core/src/refs.rs", + "line": "35", + "severity": "P2", + "category": "serialization-rule violations", + "title": "RefEntry::bounds_hash lacks skip_serializing_if and default, serializing null", + "description": "pub bounds_hash: Option<u64> at refs.rs:35 has no serde attributes. When bounds are absent (no include_bounds), bounds_hash is None and serializes as \"bounds_hash\": null. The adjacent bounds field (line 34) correctly has #[serde(default, skip_serializing_if = \"Option::is_none\")] but bounds_hash does not.", + "why_it_matters": "bounds_hash is a derived cache key used for ref identity (ref_identity.rs). It is None for most refs in headless/no-bounds snapshots. Emitting null on every such ref inflates RefMap size (which has a 1 MB cap, refs.rs:15) and produces inconsistent JSON output depending on snapshot options.", + "suggested_fix": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to bounds_hash. Both attributes needed for the same round-trip reason as core-trait-15.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: refs.rs:35 `pub bounds_hash: Option<u64>` has no serde attributes. The adjacent bounds field (line 33-34) carries #[serde(default, skip_serializing_if = \"Option::is_none\")]. bounds_hash is None for all refs taken without include_bounds (the default), which is the vast majority of production snapshots. Every such ref emits `\"bounds_hash\": null`, inflating every persisted RefMap and violating the omit-null convention.", + "fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::bounds_hash. Include default for round-trip safety with existing stored RefMaps.", + "fix_files": [ + "crates/core/src/refs.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::bounds_hash. Include default for round-trip safety with existing stored RefMaps." + }, + { + "id": "core-trait-17", + "file": "crates/core/src/refs.rs", + "line": "36", + "severity": "P2", + "category": "serialization-rule violations", + "title": "RefEntry::available_actions lacks skip_serializing_if and default, serializing []", + "description": "pub available_actions: Vec<String> at refs.rs:36 has no serde attributes. When empty it serializes as \"available_actions\": []. The adjacent states field (line 32) correctly carries #[serde(default, skip_serializing_if = \"Vec::is_empty\")], establishing the codebase convention.", + "why_it_matters": "Skeleton anchor entries in ref_alloc.rs:175 explicitly set available_actions = vec![], so every skeleton ref emits an empty array. This bloats stored RefMaps (1 MB cap) and violates the omit-empty contract that other Vec fields in RefEntry follow.", + "suggested_fix": "Add #[serde(default, skip_serializing_if = \"Vec::is_empty\")] to available_actions. Add default so stored JSON without the key round-trips correctly.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: refs.rs:36 `pub available_actions: Vec<String>` has no serde attributes. The adjacent states field (line 31-32) carries #[serde(default, skip_serializing_if = \"Vec::is_empty\")]. ref_alloc.rs:175 (skeleton entries) sets available_actions to vec![], so every skeleton anchor ref emits `\"available_actions\": []` into the persisted RefMap. Non-skeleton refs also emit [] when the element advertises no capabilities. This is a current production bloat against the 1 MB RefMap cap.", + "fix_approach": "Add #[serde(default, skip_serializing_if = \"Vec::is_empty\")] to RefEntry::available_actions. Include default for round-trip deserialization.", + "fix_files": [ + "crates/core/src/refs.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[serde(default, skip_serializing_if = \"Vec::is_empty\")] to RefEntry::available_actions. Include default for round-trip deserialization." + }, + { + "id": "core-trait-18", + "file": "crates/core/src/refs.rs", + "line": "37", + "severity": "P2", + "category": "serialization-rule violations", + "title": "RefEntry::source_app lacks skip_serializing_if and default, serializing null", + "description": "pub source_app: Option<String> at refs.rs:37 has no serde attributes. When source_app is None the field serializes as \"source_app\": null. The immediately following fields source_window_id (line 39) and source_window_title (line 41) both carry #[serde(default, skip_serializing_if = \"Option::is_none\")].", + "why_it_matters": "source_app will be None in contexts where the adapter cannot determine an owning application (common on Linux where the window manager and application may not expose a name). Every such ref emits null, violating the omit-null rule and inflating persisted RefMaps.", + "suggested_fix": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to source_app. Add default for round-trip safety.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: refs.rs:37 `pub source_app: Option<String>` has no serde attributes. The fields immediately following (source_window_id line 38-39, source_window_title line 40-41) both carry #[serde(default, skip_serializing_if = \"Option::is_none\")]. On macOS, refs from the window-inventory source that have no owning application name emit `\"source_app\": null`, violating the omit-null convention. Current production issue.", + "fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::source_app. Include default for round-trip deserialization.", + "fix_files": [ + "crates/core/src/refs.rs" + ], + "area": "core-trait", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add #[serde(default, skip_serializing_if = \"Option::is_none\")] to RefEntry::source_app. Include default for round-trip deserialization." + }, + { + "id": "core-commands-press-02", + "file": "crates/core/src/commands/key_down.rs", + "line": "9-10 (key_down.rs); 9-10 (key_up.rs)", + "severity": "P1", + "category": "policy/permission-preflight", + "title": "BLOCKED_COMBOS check is not applied in key_down / key_up, enabling trivial bypass on macOS", + "description": "Both `key_down::execute` and `key_up::execute` call `press::parse_combo` to validate the combo but never consult `BLOCKED_COMBOS`. An agent can therefore synthesize `key_down(cmd+q)` followed by `key_up(cmd+q)` to quit the target application, even though `press(cmd+q)` is explicitly rejected at line 24 of press.rs. The bypass requires no elevated privileges and works with the public API as-is.", + "why_it_matters": "The safety policy in press.rs is defeated by a trivially available alternative path. The protection is only as strong as the most permissive entrypoint, and key_down/key_up are more permissive than press for all blocked combos.", + "suggested_fix": "Extract a shared `fn check_blocked_combo(normalized: &str) -> Result<(), AppError>` from press.rs and call it at the top of both `key_down::execute` and `key_up::execute` before dispatching to the adapter. Alternatively, move the check into `parse_combo` itself so any caller automatically inherits the policy.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": true, + "live_repro_plan": "On macOS with a foreground app (e.g. TextEdit) open: run `agent-desktop key-down cmd+q` then `agent-desktop key-up cmd+q`. Buggy observation: both commands return success JSON and the app receives Cmd+Q, triggering a quit prompt. Correct/fixed observation: both commands return {\"code\":\"INVALID_ARGS\",\"message\":\"Key combo 'cmd+q' is blocked for safety\"} identical to what `agent-desktop press cmd+q` already returns. Confirmed by reading key_down.rs (line 9: calls parse_combo only, no BLOCKED_COMBOS check) and macos/src/adapter.rs (line 247: key_event is a real implementation via synthesize_key_state). The dispatch layer exposes KeyDown/KeyUp as full CLI commands (dispatch/mod.rs lines 163-167).", + "rationale": "Confirmed real bypass. key_down::execute and key_up::execute each call parse_combo (validates syntax) but never consult BLOCKED_COMBOS. The macOS adapter has a real key_event implementation. An agent can trivially synthesize any blocked combo with two calls. The bypass requires no privileges beyond what press requires, and it works against all five listed blocked combos.", + "fix_approach": "Extract a shared check_blocked_combo(normalized: &str) -> Result<(), AppError> from press.rs and call it at the start of both key_down::execute and key_up::execute after lowercasing/stripping spaces, before delegating to adapter.key_event. Alternatively, move the check into parse_combo so any future caller inherits it automatically.", + "fix_files": [ + "crates/core/src/commands/key_down.rs", + "crates/core/src/commands/key_up.rs", + "crates/core/src/commands/press.rs" + ], + "area": "core-commands", + "empirical_result": "CONFIRMED", + "final_verdict": "KEEP", + "regression_risk": "low", + "regression_notes": "The fix adds a rejection path only for the five combos in BLOCKED_COMBOS (cmd+q, cmd+shift+q, cmd+alt+esc, ctrl+cmd+q, cmd+shift+delete). All other key-down/key-up combos (e.g., key-down shift, key-down cmd+c, key-down return) are unaffected. No existing test files for key_down or key_up were found in the repo (only key_down.rs and key_up.rs implementation files), so no test suite regressions. The check in press.rs that already works is unchanged.", + "safe_fix_approach": "Expose the blocked-combo logic as a shared function in press.rs — e.g., `pub fn check_blocked_combo(raw: &str) -> Result<(), AppError>` that normalizes (lowercase + strip spaces) and checks against BLOCKED_COMBOS, returning AppError::invalid_input if matched. Then call this function at the top of both key_down::execute and key_up::execute, before delegating to parse_combo/adapter.key_event. This is a minimal, zero-ambiguity change: it touches only the two unsafe entry points, reuses the identical normalization logic already proven correct in press.rs, and cannot break any currently-working combo (non-blocked combos pass through unchanged).", + "evidence": "1. Static analysis: key_down.rs line 9 calls parse_combo() only — no BLOCKED_COMBOS check. key_up.rs line 9 identical. press.rs lines 23-29 explicitly blocks cmd+q via BLOCKED_COMBOS before calling parse_combo.\n\n2. Binary control test (directly observed):\n Command: target/release/agent-desktop press cmd+q\n Output: {\"command\":\"press\",\"error\":{\"code\":\"INVALID_ARGS\",\"message\":\"Key combo 'cmd+q' is blocked for safety\"},\"ok\":false,\"version\":\"2.0\"}\n Confirms press.rs block is live in the binary.\n\n3. key-down command confirmed present and functional in binary v0.4.3 (compiled Jun 27 2026, after all source file modification dates: key_down.rs Jun 3, press.rs Jun 20). `agent-desktop key-down --help` returns the command's usage spec with no mention of safety filtering.\n\n4. keyboard.rs synthesize_key_state (line 78-108): pure AXUIElementPostKeyboardEvent call — no blocked combo check at the adapter layer. The safety gate exists ONLY in press::execute.\n\n5. Dispatch layer mod.rs lines 163-167 routes KeyDown/KeyUp directly to key_down::execute / key_up::execute with no wrapping filter.\n\n6. Actual key-down cmd+q firing was not performed — would send real AX keystrokes to the frontmost app (terminal), which is a forbidden action under guardrails — but the code path is mechanically complete and unambiguous: parse_combo succeeds for cmd+q, adapter.key_event fires synthesize_key_state, no rejection possible." + }, + { + "id": "core-commands-skills-01", + "file": "crates/core/src/commands/skills.rs", + "line": "13-14, 63-65", + "severity": "P2", + "category": "platform-specific-logic", + "title": "macos.md is unconditionally embedded and served regardless of the running OS", + "description": "Line 13 uses `include_str!` to compile `skills/agent-desktop/references/macos.md` into the binary at build time, and line 63 registers it as a named ref in the `agent-desktop` skill's ref list. There is no `#[cfg(target_os = \"macos\")]` guard, no Windows or Linux equivalent doc, and no runtime check — every binary on every platform advertises and serves a reference that describes macOS-specific TCC permissions, CGWindowServer quirks, and macOS accessibility APIs.", + "why_it_matters": "An agent or user running on Windows or Linux will be given macOS-specific guidance (e.g., 'Open System Preferences > Security & Privacy' steps) when requesting skill documentation, potentially leading to incorrect troubleshooting steps. The skill list response includes `references/macos.md` as a valid ref on all platforms, so tooling that discovers and indexes available refs will also be polluted.", + "suggested_fix": "Either gate the `SKILL_DESKTOP_REF_MACOS` constant and its `SkillRef` entry behind `#[cfg(target_os = \"macos\")]`, or add Windows (`windows.md`) and Linux (`linux.md`) equivalents and include only the appropriate one at compile time using cfg attributes. The `refs` slice can be built with conditional compilation: `&[..., #[cfg(target_os=\"macos\")] SkillRef { rel_path: \"references/macos.md\", body: SKILL_DESKTOP_REF_MACOS }]`.", + "verdict": "KEEP", + "validated_severity": "P2", + "reachable": "yes", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: SKILL_DESKTOP_REF_MACOS is include_str! with no cfg guard (skills.rs lines 13-14) and the SkillRef entry is unconditionally in the refs slice (lines 62-65). The skills command runs before the adapter (main.rs lines 61-73 call skills:: directly without build_adapter), so it executes on all platforms. On Windows or Linux, skills list advertises references/macos.md and skills get agent-desktop --reference macos serves macOS-specific TCC/CGWindowServer guidance. Doc pollution is real and reachable, not speculative.", + "fix_approach": "Gate the SKILL_DESKTOP_REF_MACOS constant and its SkillRef entry behind #[cfg(target_os = \"macos\")]. Optionally add parallel windows.md / linux.md constants behind their respective cfg attributes. The refs slice can be built with cfg-gated elements or a conditional push at runtime.", + "fix_files": [ + "crates/core/src/commands/skills.rs" + ], + "area": "core-commands", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Gate the SKILL_DESKTOP_REF_MACOS constant and its SkillRef entry behind #[cfg(target_os = \"macos\")]. Optionally add parallel windows.md / linux.md constants behind their respective cfg attributes. The refs slice can be built with cfg-gated elements or a conditional push at runtime." + }, + { + "id": "core-commands-press-03", + "file": "crates/core/src/commands/press.rs", + "line": "54-58", + "severity": "P3", + "category": "platform-specific-logic", + "title": "parse_combo recognises no Win/Super/Meta key modifier, so Windows system shortcuts are unrepresentable and unblockable", + "description": "The modifier match in `parse_combo` (lines 54-58) accepts `cmd|command`, `ctrl|control`, `alt|option`, and `shift`. The Windows key (⊞) and its Linux equivalents (`super`, `meta`, `win`) are absent. This means callers cannot express combos like `win+l` (lock workstation), `win+d` (show desktop), or `win+e` (file explorer), and these combos cannot be added to `BLOCKED_COMBOS` even if the Windows blocklist concern from finding core-commands-press-01 is addressed. Any attempt to pass a combo containing `win` returns `AppError::invalid_input(\"Unknown modifier: 'win'\")` rather than dispatching or blocking the key.", + "why_it_matters": "On Windows, `win+l` is equivalent in impact to `cmd+ctrl+q` (screen lock) on macOS. Without a `win` modifier token, neither the safety blocklist nor legitimate automation of Windows-key shortcuts can be implemented in the current parser, requiring future port work to touch `parse_combo` before any Windows-key policy can be expressed.", + "suggested_fix": "Add `Modifier::Win` (or `Modifier::Super`) to the `Modifier` enum and extend the match in `parse_combo` with `\"win\" | \"windows\" | \"super\" | \"meta\" => Modifier::Win`. Then add the relevant Windows-key combos to a `#[cfg(target_os = \"windows\")]` BLOCKED_COMBOS extension (see core-commands-press-01). The adapter layer on macOS can map `Modifier::Win` to `kVK_Command` or no-op, while the Windows adapter maps it to `VK_LWIN`.", + "verdict": "KEEP", + "validated_severity": "P3", + "reachable": "theoretical", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "Confirmed: parse_combo's modifier match (press.rs lines 54-58) accepts cmd/command, ctrl/control, alt/option, shift and nothing else. Passing 'win', 'super', or 'meta' returns AppError::invalid_input('Unknown modifier: win'). This is a genuine parser gap. It is directly linked to press-01: even if a Windows BLOCKED_COMBOS list were added, win+l could not be expressed or blocked without this fix. The P3 severity is appropriate — no current production user is affected since Windows key dispatch is unimplemented — but the fix is a prerequisite for any future Windows-key policy.", + "fix_approach": "Add Modifier::Win (or Modifier::Super) to the Modifier enum in action.rs and extend parse_combo's match arm with \"win\" | \"windows\" | \"super\" | \"meta\" => Modifier::Win. The macOS adapter can map Modifier::Win to a no-op; a future Windows adapter maps it to VK_LWIN.", + "fix_files": [ + "crates/core/src/commands/press.rs", + "crates/core/src/action.rs" + ], + "area": "core-commands", + "empirical_result": "STATIC", + "final_verdict": "KEEP", + "regression_risk": "unknown", + "safe_fix_approach": "Add Modifier::Win (or Modifier::Super) to the Modifier enum in action.rs and extend parse_combo's match arm with \"win\" | \"windows\" | \"super\" | \"meta\" => Modifier::Win. The macOS adapter can map Modifier::Win to a no-op; a future Windows adapter maps it to VK_LWIN." + } + ], + "refuted": [ + { + "id": "macos-system-03", + "file": "crates/macos/src/system/window_ops.rs", + "line": "36-39", + "severity": "P1", + "category": "window-op-correctness", + "title": "Maximize uses raw display bounds (0,0 origin), placing the window under the macOS menu bar", + "description": "maximize_to_main_display calls `CGDisplay::main().bounds()` which returns the full physical display rectangle with origin (0.0, 0.0) and height equal to the total screen height including the 25-pt menu bar. The function then calls set_position(el, 0.0, 0.0) and set_size(el, display_width, display_height). Because the macOS window compositor places the menu bar at y=0, the window's title bar slides under the menu bar — the window occupies the region from the very top pixel to the bottom, hidden behind the menu bar. The Dock area is also not excluded. The correct APIs are NSScreen.visibleFrame (excludes both menu bar and dock) or the AXZoom attribute toggle (respects the application-defined zoom rect). AXZoom is the attribute the green button uses.", + "why_it_matters": "Users who invoke Maximize on any window get a window they cannot interact with via the title bar because it is hidden beneath the menu bar. This is visually and functionally broken on all macOS configurations that have the menu bar enabled.", + "suggested_fix": "Replace CGDisplay::main().bounds() with NSScreen's visibleFrame to exclude the menu bar and dock. Alternatively, set AXZoom (kAXZoomedAttribute) to CFBoolean::true_value() on the window element, which delegates the correct sizing to each application's own zoom-rect logic, matching what the green zoom button does.", + "verdict": "KEEP", + "validated_severity": "P1", + "reachable": "yes", + "testable": true, + "live_repro_plan": "1. Launch TextEdit (or any resizable macOS window). 2. Send a maximize-window command targeting TextEdit via agent-desktop. 3. Take a full-display screenshot. Expected: window occupies the visible area below the menu bar (origin y ≥ ~25 pt in AX coordinates). Buggy: maximize_to_main_display calls CGDisplay::main().bounds() which returns {origin=(0,0), size=(W,H)} including the menu bar region; set_position is then called with y=0, placing the window title bar directly behind the 25-pt menu bar ribbon. The window is functionally inaccessible via its title bar.", + "rationale": "Confirmed at window_ops.rs lines 36-39: maximize_to_main_display calls CGDisplay::main().bounds() and passes bounds.origin (0.0, 0.0) to set_position and the full display height to set_size. AX kAXPositionAttribute uses top-left origin coordinates; y=0 is the top pixel of the display — behind the menu bar. NSScreen.visibleFrame excludes both the menu bar and Dock and would give the correct origin. This is a universal defect: every Maximize invocation on every macOS machine with a menu bar produces a broken window.", + "fix_approach": "Replace CGDisplay::main().bounds() with NSScreen mainScreen visibleFrame via ObjC message send (consistent with how workspace_apps.rs already calls into AppKit), or set kAXZoomedAttribute (AXZoom) to CFBoolean::true_value() on the window element, which delegates sizing to each application's own zoom-rect logic identical to the green zoom button.", + "fix_files": [ + "crates/macos/src/system/window_ops.rs" + ], + "area": "macos-system", + "empirical_result": "REFUTED", + "final_verdict": "REJECT", + "regression_risk": "low", + "regression_notes": "Verified working flow: `maximize --app TextEdit` produces a correctly placed window (y=30, below menu bar) on macOS 15 via AX clamping. The Dock is not excluded — bottom of window reaches y=967 (screen bottom) regardless of Dock height — but this is not the defect under review and any fix must not change behavior for callers who rely on the window filling the full non-menu-bar area. Proposed fix using NSScreen.visibleFrame carries a coordinate conversion risk: NSScreen uses Cocoa coordinates (bottom-left origin, y increasing upward) while AX kAXPositionAttribute uses top-left/y-down; a naive substitution would invert the y-axis and misplace the window. The AXZoom approach (set kAXZoomedAttribute=CFBoolean::true_value()) avoids coordinate math entirely and is the lower-risk path, though it delegates sizing to each application's own zoom rect, which may differ from a full-screen fill on some apps.", + "safe_fix_approach": "If the Dock exclusion gap is considered worth addressing separately (low-severity, separate from this finding), replace the two-step set_position/set_size with setting kAXZoomedAttribute to CFBoolean::true_value() on the window element — the same action as the green zoom button, which correctly respects both menu bar and Dock. This requires no coordinate conversion and cannot introduce an AX coordinate mismatch. Avoid NSScreen.visibleFrame unless a Cocoa-to-AX coordinate transform (flip y: ax_y = screen_height - cocoa_y - cocoa_height) is also applied. The current implementation's menu-bar overlap defect does not exist in practice due to OS-level AX clamping, so this fix is optional cleanup rather than a P1 correctness issue.", + "evidence": "Swift confirmed CGDisplay::main().bounds() returns origin=(0.0,0.0) size=(1496.0,967.0) — full display height including menu bar. The code calls set_position(el, 0.0, 0.0) then set_size(el, 1496.0, 967.0). After running `agent-desktop maximize --app TextEdit` against an untitled TextEdit document (pid 49812, launched fresh this session), an independent AppleScript query (`tell application \"System Events\" to tell process \"TextEdit\" to get position/size of front window`) returned position=(0, 30) and size=(1496, 937). The delta is exactly 30pt (menu bar height): macOS AX silently clamped y=0→30 and height=967→937. The window title bar is at y=30, fully below the menu bar and accessible. The P1 claim (\"title bar slides under menu bar, window functionally inaccessible\") does not manifest in practice. Baseline before maximize: position=(150, 69), size=(603, 509). TextEdit was closed gracefully after repro (close-app, no save dialog — document was empty)." + } + ], + "rejectedStatic": [ + { + "id": "macos-tree-09", + "file": "crates/macos/src/tree/builder.rs", + "line": "349-350", + "severity": "P3", + "category": "builder traversal", + "title": "Non-macOS build_subtree stub names the set _visited, contradicting the ancestor-path semantics of the macOS implementation", + "description": "The macOS implementation (line 77) names the parameter 'ancestors' and implements ancestor-path semantics: insert before recursion, remove after. The non-macOS stub (line 349) names it '_visited', which suggests global-visited (never-removed) semantics. The docstring and calling convention are consistent with ancestor-path (the set is shared across root calls from callers), but the name mismatch will mislead any developer porting the non-macOS stub to implement the function, who may implement a global-visited set instead and suppress legitimate revisits via different paths.", + "why_it_matters": "On a future Windows/Linux port, implementing the parameter as a global-visited set would prevent valid second-path traversals of shared subtree nodes, producing an incorrect tree that silently omits nodes reachable via more than one parent.", + "suggested_fix": "Rename the non-macOS stub parameter from _visited to _ancestors to match the macOS parameter name and communicate the ancestor-path (insert/remove) contract.", + "verdict": "REJECT", + "validated_severity": "P3", + "reachable": "no", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "The non-macOS build_subtree stub (builder.rs:344-354) unconditionally returns None and is never executed in production. The _visited vs _ancestors naming inconsistency is speculative: it only matters if a future Windows/Linux porter implements the function using the stub parameter name as a semantic cue. No such port exists today. Per YAGNI discipline, this is a speculative abstraction with no current need. REJECT.", + "fix_approach": "N/A — rejected as speculative. If a non-macOS port is planned, rename _visited to _ancestors in the stub as part of that work.", + "fix_files": [], + "area": "macos-tree" + }, + { + "id": "macos-adapter-10", + "file": "crates/macos/src/adapter.rs", + "line": "330-340", + "severity": "P3", + "category": "Audit: clean areas", + "title": "No non-test unwrap/expect/panic/unreachable/index-panic found", + "description": "Exhaustive grep across all crates/macos/src/** for unwrap(), expect(), panic!, unreachable!, todo!, and index-access patterns: all occurrences of panic-inducing calls are inside #[cfg(test)] mod blocks (chain_verify.rs:105-149, dispatch.rs:308, force_close.rs:159/224, process.rs tests). Index accesses in list.rs:177-185 are guarded by explicit len checks. element.rs:99 indexing is bounded by a 0..=7 match arm. Main-thread safety: no NSApplication or other main-thread-only AppKit API called off-main; the autorelease pool gap (macos-adapter-03) is the closest related risk. builder.rs:310 let _ = children is a cfg(not(target_os=macos)) dead stub. resolve_roots.rs:245 .ok() is a parse().ok() string-to-Option conversion — intentional.", + "why_it_matters": "Confirms the four audit categories are covered. The absence of non-test panics and index panics is a positive finding; the audit is exhaustive for these categories.", + "suggested_fix": "No action needed for these categories.", + "verdict": "REJECT", + "validated_severity": "P3", + "reachable": "no", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "This is a positive audit certification, not a defect. The finding documents the absence of non-test unwrap/expect/panic!/unreachable!/todo! and unguarded index accesses in crates/macos/src/**. Confirmed by grep: all panic-inducing calls are inside #[cfg(test)] blocks. The list.rs and element.rs index accesses are guarded. This is useful audit evidence but requires no code change.", + "fix_approach": "No action needed.", + "fix_files": [], + "area": "macos-adapter" + }, + { + "id": "core-commands-close-app-01", + "file": "crates/core/src/commands/close_app.rs", + "line": "16", + "severity": "P2", + "category": "platform-specific-logic", + "title": "Protected-process suggestion text hardcodes macOS-specific daemon names", + "description": "The `with_suggestion` call on line 16 names `loginwindow`, `WindowServer`, `Dock`, `Finder`, and `launchd` as the protected processes. All five are macOS-exclusive. On Windows the analogous critical processes include `winlogon.exe`, `csrss.exe`, `wininit.exe`, `dwm.exe`, and `explorer.exe`; on Linux they include `systemd`, `Xorg`/`wayland`, and display-manager processes.", + "why_it_matters": "When the `is_protected_process` guard fires on Windows or Linux, the error message it returns names daemons that do not exist on those platforms. This confuses the operator about why the command was rejected and which processes are actually protected on their OS.", + "suggested_fix": "Either make the suggestion text platform-neutral ('Target a regular application; protected system processes cannot be closed.') or introduce a `protected_process_hint() -> &str` method on `PlatformAdapter` that each adapter populates with OS-appropriate examples, mirroring the existing `is_protected_process` design.", + "verdict": "REJECT", + "validated_severity": "P3", + "reachable": "no", + "testable": false, + "live_repro_plan": "STATIC", + "rationale": "The macOS-named suggestion text (loginwindow, WindowServer, etc.) is only emitted when adapter.is_protected_process() returns true. The PlatformAdapter trait default implementation (adapter.rs line 251) returns false unconditionally. Neither WindowsAdapter nor LinuxAdapter overrides is_protected_process — they are empty stubs. Only MacOSAdapter overrides it (macos/src/adapter.rs line 134). Therefore the branch in close_app.rs that emits the suggestion is dead code on Windows and Linux. The finding's claim that Windows/Linux users would see this message is factually wrong. The text is only reachable on macOS where the named processes are accurate.", + "fix_approach": "No fix required. If a future Windows/Linux adapter adds is_protected_process support, the suggestion text should be made platform-neutral at that time.", + "fix_files": [], + "area": "core-commands" + } + ], + "total_reviewed": 77 +} diff --git a/docs/plans/2026-06-29-001-feat-wait-for-selector-flag-plan.md b/docs/plans/2026-06-29-001-feat-wait-for-selector-flag-plan.md new file mode 100644 index 0000000..9864e1d --- /dev/null +++ b/docs/plans/2026-06-29-001-feat-wait-for-selector-flag-plan.md @@ -0,0 +1,262 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +execution: code +type: feat +product_contract_source: ce-plan-bootstrap +origin: https://github.com/lahfir/agent-desktop/issues/84 +created: 2026-06-29 +--- + +# feat: add `--wait-for` selector polling flags + +## Summary + +Add global `--wait-for` / `--wait-for-gone` (with `-w` short for `--wait-for`) and `--wait-timeout` flags that poll the accessibility tree until an element matching a compact selector appears (or disappears), then return the resulting snapshot. This collapses the agent-side "snapshot → check → sleep → retry" loop into a single invocation, and lets action commands confirm a post-action UI change in one call (`click @e5 -w ":Saved!"`). + +The feature is **pure CLI orchestration over existing engines** — it reuses `find`'s matcher and `wait`'s poll loop. No new platform/AX code, no CSS parser, no new adapter methods. + +--- + +## Problem Frame + +Today an agent that needs to wait for a UI state change (modal appears, spinner disappears, button becomes available) must hand-roll a polling loop: `snapshot`, inspect JSON, `sleep`, retry. That burns tokens, adds latency, and is re-implemented in every agent workflow. + +The **engine** to avoid this already shipped in 0.4.4 as the `wait` command (`wait --text/--window/--element … --timeout`), which polls with 200ms→1s backoff and returns a `kind: "wait_timeout"` error envelope on expiry. What's missing is **ergonomics**: a single-string selector and a flag that fuses the wait onto the command that produces the snapshot, so `click; wait; snapshot` becomes `click … -w "…"`. Issue #84 asks for exactly this, and explicitly frames it as small CLI-only work. + +The issue's `button:contains('Submit')` spelling is **illustrative, not a contract** — its own "Key design points" say "Reuses existing selector/filter infrastructure," which is `find`'s `--role/--name/--text/--value` + `node_matches`. We honor the latter. + +--- + +## Requirements + +- **R1.** A global `--wait-for <SELECTOR>` flag (short `-w`) blocks until an element matching `<SELECTOR>` is present in the polled tree, then returns the snapshot. +- **R2.** A global `--wait-for-gone <SELECTOR>` flag blocks until **no** element matches `<SELECTOR>`, then returns the snapshot (covers "spinner disappears"). +- **R3.** A global `--wait-timeout <MS>` flag bounds the poll (default 30000ms, matching `wait`). On expiry the command exits 1 with a `kind: "wait_timeout"`, `predicate: "selector"` error envelope, consistent with `wait`. +- **R4.** `<SELECTOR>` is a compact `role:text` string parsed into a `FindQuery` and matched by the **same** `node_matches` used by `find`. Forms: `"button:Submit"` (role+text), `"button"` (role only), `":Saved!"` (text only). +- **R5.** `snapshot --wait-for …` polls the snapshot's own scope (`--app`/`--window-id`/`--surface`/depth) and returns the matched snapshot. +- **R6.** Ref-resolving action commands (`click`, `double-click`, `triple-click`, `right-click`, `clear`, `focus`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, `scroll-to`, `type`, `set-value`, `select`, `scroll`) accept `--wait-for`/`--wait-for-gone`: perform the action, then poll **the app the acted-on ref belongs to** (`entry.source_app`), returning the post-action snapshot with the original action result preserved. +- **R7.** `--wait-for` and `--wait-for-gone` are mutually exclusive (clap `conflicts_with`, exit 2); `--root` and `--wait-for`/`--wait-for-gone` are mutually exclusive on `snapshot` (ref-rooted drill-down has no poll semantics). +- **R8.** Passing `--wait-for`/`--wait-for-gone` to a command that does **not** support it (anything other than `snapshot` and the R6 action commands) is an `INVALID_ARGS` error, not a silent no-op. The supporting set is a single `&[&str]` checked at dispatch time. +- **R9.** Batch mode does not honor an outer `--wait-for`; batch items run independently and never inherit the CLI-level selector. +- **R10.** No new platform-adapter methods; `agent-desktop-core` remains free of platform crates (CI `cargo tree` gate stays green). + +**Success criteria:** an agent replaces a multi-call poll loop with one `snapshot -w "<selector>"`; a `click @e5 -w ":Saved!"` returns only after "Saved!" is visible; a timeout returns exit 1 with a discriminable envelope. + +--- + +## Key Technical Decisions + +**KTD1 — Compact `role:text` selector, not a DSL.** Parse on the first `:` into role (left) and text (right); each side is **trimmed and an empty side becomes `None`** (unconstrained on that axis) — never `Some("")`. This matters: `search_text::contains` runs `haystack.as_bytes().windows(needle.len())`, which **panics on a zero-length needle**, so `"button:"`, `":"`, and `" : "` must collapse the empty side to `None` rather than threading `Some("")` into `node_matches`. The result is a `FindQuery` fed to `node_matches`. This keeps the single-string ergonomics the issue shows while adding ~5 lines of parsing instead of a CSS grammar that would become a permanent public contract. Role is normalized via `roles::normalize_role_query`; text via `search_text::normalize` (the exact path `find` uses), so selector semantics equal `find` semantics by construction. `node_matches`'s `text` axis already searches name/value/description (`search_text::node_contains`), so `window:Settings` matches a node whose role is `window` and whose text contains "Settings"; matching by *window title* specifically is out of scope (use `wait --window` for that). + +**KTD2 — One matcher, promoted out of `find`.** `FindQuery` and `node_matches` are currently private in `commands/find.rs`. Promote them (plus the `role:text` parser) into a new `commands/query.rs`; `find` imports them. This is the anti-redundancy move: the wait path does not get a parallel matcher — both call sites share one. Behavior of `find` is unchanged (pure move + re-export). + +**KTD3 — One poll helper, reusing the `wait` loop shape.** New `commands/wait_selector.rs` mirrors `wait_for_text`: `snapshot::build` (cheap, no persist) in a 200ms→1s backoff loop, run `node_matches` over the tree, persist the refmap and return the snapshot envelope only on match. Presence vs. absence is one boolean (`gone`). **Retryable poll set is wider than `wait_for_text`'s** — `Timeout`, `ElementNotFound`, **`AppNotFound`, and `WindowNotFound`** are all swallowed and surfaced as `last_error` on timeout. This is the correctness fix for the common post-action case where the action closes/navigates the target (`click @submit -w ":Saved!"` dismisses the dialog): without `AppNotFound`/`WindowNotFound` in the retryable set, the poll would propagate a raw `WINDOW_NOT_FOUND` instead of the promised `wait_timeout` envelope. The empty/match-everything selector guard lives **here** (before the loop) so both call sites (snapshot self-apply and post-action) reject it identically. + +**KTD4 — Global flags carried in `CommandContext`, applied at two scoped call sites, both keyed to the *correct* app.** The flags are defined once as clap globals (like `--headed`) and threaded into `CommandContext` (new `WaitSelector` config, set via a `with_wait_selector(...)` builder mirroring `with_headed`). Two thin application points, both delegating to the single `wait_selector` helper: + - `snapshot::execute` self-applies when a selector is set, scoped to its own `--app`/opts (correct ordering: wait *is* the snapshot; no wasted eager build). + - The **ref-action path in `helpers.rs`** applies it after the action, scoped to **`entry.source_app`** — the app the resolved ref belongs to — embedding the action result under `data.after_action`. + + **Why not a binary post-dispatch hook (rejected).** A hook in `run_with_adapter` only sees the action's JSON result, not the ref's pid/app. In the default **headless** mode AXPress does not foreground the target app, so the frontmost app is usually the *terminal*, not the acted-on app — a focused-app poll would read the wrong tree and time out on the default code path. The ref-action path already holds the `RefEntry` (`source_app`), so scoping there is both correct and the natural place that covers the ref-resolving commands R6 enumerates. This is layering, not duplication — the loop, matcher, and persistence live solely in the helper. + + **Wiring reaches all 16 R6 commands via the two `helpers.rs` entrypoints** (verified against the codebase): 11 commands (`click`, `double-click`, `triple-click`, `clear`, `focus`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, `scroll-to`) go through `execute_ref_action_with_context`; 5 (`type`, `set-value`, `select`, `scroll`, **`right-click`**) go through `execute_ref_action_result_with_context` and then serialize themselves. `right-click` is the only one that builds a custom JSON body (its menu probe), so its wait must run **after** the probe (action → menu probe → wait). `execute_by_ref` (the FFI bridge) also routes through `execute_ref_action_with_context` but is unaffected: the FFI `CommandContext` never carries a `WaitSelector`, so `apply_post_action_wait` early-returns. + +**KTD5 — Timeout returns the `wait` error envelope **plus** the last snapshot's id.** The issue says "return the last snapshot on timeout," but the established `wait` convention is a `TIMEOUT` error (`kind: "wait_timeout"`) carrying a compact `last_error`, and `finish()` maps `Err` → exit 1. We keep the error-envelope convention (consistency, no bespoke success-on-timeout path) **but honor the issue's debuggability intent**: on timeout, persist the last successfully-built tree and include its `snapshot_id` in `details` alongside `last_error`, so the agent can inspect the final tree state without racing a fresh `snapshot`. For post-action timeouts the original action result is also embedded under `details.after_action` so the caller learns the action ran. + +**KTD6 — CLI-only; FFI parity deferred.** The flags live in the binary/CLI + core helper. The FFI cdylib does not gain a `--wait-for` surface in this PR (the issue scopes this as "purely a CLI orchestration feature"). The `wait_selector` core helper is reusable by a future `ad_wait_for` entrypoint; noted under Deferred. + +--- + +## High-Level Technical Design + +``` + --wait-for / --wait-for-gone / --wait-timeout (clap globals) + | + CommandContext (WaitSelector) + | + +-----------------------+------------------------+ + | | + snapshot::execute helpers.rs ref-action path + (selector set?) (after action, selector set?) + | | + | scope = --app/opts | action runs first -> ActionResult + | | scope = entry.source_app (correct in headless) + +----------------------+ +---------------------+ + | | + commands/wait_selector::execute + | + loop: snapshot::build -> node_matches (present|absent) + match -> persist refmap, return snapshot envelope + timeout -> wait_timeout::selector(...) (exit 1) + | + commands/query.rs + FindQuery + node_matches + parse_selector + (shared with find.rs) +``` + +--- + +## Implementation Units + +### U1. Extract shared selector matcher into `query.rs` + +**Goal:** Single matcher shared by `find` and the wait path; add the `role:text` parser. +**Requirements:** R4, KTD1, KTD2. +**Dependencies:** none. +**Files:** +- `crates/core/src/commands/query.rs` (new) — move `FindQuery`, `node_matches`, `count_matches`'s predicate use; add `pub fn parse_selector(&str) -> FindQuery`. +- `crates/core/src/commands/find.rs` — delete the moved items, import from `query`. +- `crates/core/src/commands/mod.rs` — register `query`. +- `crates/core/src/commands/query_tests.rs` (new). +**Approach:** Pure refactor + addition. `FindQuery::from_args` stays in `find` (it depends on `FindArgs`); the reusable pieces are `FindQuery` the struct, `node_matches`, and the new `parse_selector`. `parse_selector` splits once on `:`; **trims each side; an empty-after-trim side becomes `None`, never `Some("")`** (a `Some("")` text would reach `search_text::contains` and panic on `windows(0)` — see KTD1); role through `roles::normalize_role_query`, text through `search_text::normalize`. Also add `pub fn tree_has_match(&AccessibilityNode, &FindQuery) -> bool` here (short-circuiting presence check) so `wait_selector` shares the exact predicate. `find`'s public behavior must not change. +**Patterns to follow:** existing `FindQuery`/`node_matches` in `find.rs`; `roles::normalize_role_query` and `search_text::normalize/contains/node_contains`. +**Test scenarios:** +- `parse_selector("button:Submit")` → role normalized to `button`, text normalized to `submit`. +- `parse_selector("button")` → role set, text `None`. +- `parse_selector(":Saved!")` → role `None`, text set. +- `parse_selector("")`, `parse_selector(":")`, `parse_selector(" : ")` → **both `None`** (match-everything; rejected by `wait_selector`, see U2). +- `parse_selector("button:")` → role `button`, text **`None`** (not `Some("")`). +- `parse_selector(" button : Submit ")` → trimmed to role `button`, text `submit`. +- `parse_selector("textfield:a:b")` → splits on first colon only; text is `a:b`. +- A `FindQuery { text: None }` never causes `search_text::contains` to be called with an empty needle (guards the `windows(0)` panic). +- `node_matches`/`tree_has_match` over a fixture tree returns identical results before/after the move (regression). +**Verification:** `find` tests pass unchanged; new parser tests pass; `cargo tree -p agent-desktop-core` shows no platform crates. + +### U2. `wait_selector` poll helper + +**Goal:** The reusable poll-until-present/absent engine returning a snapshot envelope. +**Requirements:** R1, R2, R3, R5, KTD3, KTD5. +**Dependencies:** U1. +**Files:** +- `crates/core/src/commands/wait_selector.rs` (new). +- `crates/core/src/commands/wait_timeout.rs` — add `pub(crate) fn selector(selector: &str, gone: bool, timeout_ms, last_error, last_snapshot_id: Option<String>) -> Result<Value, AppError>` (`predicate: "selector"`, `kind: "wait_timeout"`, includes `snapshot_id` in details when a tree was built). +- `crates/core/src/commands/mod.rs` — register `wait_selector`. +- `crates/core/src/commands/wait_selector_tests.rs` (new). +**Approach:** `execute(WaitSelectorInput { query, gone, app, opts, timeout_ms }, adapter, context)`. **First, reject a match-everything query** (both axes `None`) with `INVALID_ARGS` — this single guard covers both call sites (snapshot self-apply and post-action). Loop mirrors `wait_for_text`: `snapshot::build(adapter, &opts, app, None)`; `present = query::tree_has_match(&tree, &query)`; `matched = if gone { !present } else { present }`. On match: `RefStore::for_session(...).save_new_snapshot(&result.refmap)`, return a JSON object with **the same field shape `snapshot::execute` emits** (`app`, `window`, `ref_count`, `snapshot_id`, `tree`) plus `elapsed_ms` and `matched_selector`. `snapshot::format_result` is private — either promote it to `pub(crate)` and reuse it, or replicate the shape (the field set is the contract). Swallow the retryable set (`Timeout`, `ElementNotFound`, `AppNotFound`, `WindowNotFound`) into `last_error`, **and on each successful build remember the built tree's persisted id as `last_snapshot_id`**. On deadline call `wait_timeout::selector(... , last_error, last_snapshot_id)`. Reuse the 200ms→1s doubling backoff verbatim. +**Patterns to follow:** `wait::wait_for_text` (loop, backoff, retryable-error handling, persist-on-match, `last_error`); `snapshot::format_result` (envelope shape — promote to `pub(crate)` if reused). +**Test scenarios** (MockAdapter; deterministic trees): +- Match-everything query (`FindQuery` both `None`) → `Err` `INVALID_ARGS` before any poll. +- Element present on first poll → returns snapshot envelope with `matched_selector`, `elapsed_ms`, a persisted `snapshot_id`. +- Element absent then present on Nth poll → returns once present (mock whose tree changes across calls). +- `gone=true`, element present then absent → returns when absent. +- `gone=true`, element never present → returns immediately (already gone). +- Never matches within timeout → `Err` `code == TIMEOUT`, details `kind=="wait_timeout"`, `predicate=="selector"`, `last_error` populated, **`snapshot_id` of the last built tree present in details**. +- Each retryable error (`ElementNotFound`, `AppNotFound`, `WindowNotFound`) mid-poll is swallowed, not surfaced, until timeout — covers the "action closed the window" case. +- Persisted snapshot is loadable by the returned `snapshot_id` (refs usable by a follow-up action). +**Verification:** all unit tests pass; timeout envelope is JSON-discriminable from a `chain_deadline` timeout. + +### U3. Global flags + `CommandContext` wiring + snapshot self-apply + +**Goal:** Define the flags once, carry them, reject them on unsupported commands, and make `snapshot` honor them in its own scope. +**Requirements:** R1, R2, R3, R5, R7, R8, R9, KTD4. +**Dependencies:** U2. +**Files:** +- `src/cli/mod.rs` — add three `global = true` args: `--wait-for`/`-w` (`Option<String>`, `conflicts_with = "wait_for_gone"`), `--wait-for-gone` (`Option<String>`), `--wait-timeout` (`u64`, default 30000). The `--wait-for`/`--wait-for-gone` pair is mutually exclusive via clap `conflicts_with` (exit 2). +- `crates/core/src/context.rs` — add `WaitSelector { query_raw: String, gone: bool, timeout_ms: u64 }` carried in `CommandContext`; `with_wait_selector(Option<WaitSelector>)` builder mirroring `with_headed`; accessor. **Update `for_batch_item` to set `wait_selector: None`** — the struct literal is exhaustive, so this is a required compile fix and the correct semantics (R9: batch items never inherit the outer selector). +- `src/main.rs` — build `WaitSelector` from the parsed globals and attach via `with_wait_selector`. **Before dispatch, when a selector is set, reject it with `INVALID_ARGS` unless the command is `snapshot` or one of the R6 action commands** (R8) — a single `const WAIT_SUPPORTED: &[&str]` checked against `cmd.name()`. This closes the `find -w …` silent-no-op footgun. +- `src/cli_args/mod.rs` (`SnapshotArgs`) / `src/cli/mod.rs` — make `--root` and `--wait-for`/`--wait-for-gone` mutually exclusive (R7); since the wait flags are global, enforce this in `snapshot::execute` (or `main`) as an `INVALID_ARGS` check rather than clap (a global cannot `conflicts_with` a local arg cleanly). +- `crates/core/src/commands/snapshot.rs` — when `context` carries a wait selector **and `root_ref` is `None`**, delegate to `wait_selector::execute` with snapshot's `opts` + `app`; `parse_selector` the raw string. If `root_ref` **and** a selector are both set → `INVALID_ARGS`. +- `crates/core/src/commands/snapshot_tests.rs` — selector-path tests. +**Approach:** Flags are global so they attach to every subcommand with one definition (no per-command duplication; avoids a clap conflict between a global `-w` and a local one). `snapshot::execute` checks the selector branch **after** the existing `root_ref` early-return guard so the two never silently co-apply; the explicit `INVALID_ARGS` makes the combination loud. `--wait-timeout` default 30000 matches `wait`. +**Patterns to follow:** existing globals (`--headed`, `--session`) in `src/cli/mod.rs`; `CommandContext::with_headed` and `for_batch_item` in `crates/core/src/context.rs`; `cmd.name()` dispatch in `src/main.rs`. +**Test scenarios:** +- `snapshot -w "button:Submit"` on a tree containing it → returns matched snapshot (MockAdapter). +- `snapshot --wait-for-gone "progressindicator"` when absent → returns immediately. +- `--wait-for` + `--wait-for-gone` together → clap parse error, exit 2 (`conflicts_with`). +- `find -w "button:Submit"` → `INVALID_ARGS` (unsupported command, R8), not a silent find result. +- `snapshot --root @e5 -w ":button"` → `INVALID_ARGS` (R7), exercised in `snapshot_tests`. +- `snapshot --app Foo -w "…"` polls scope `Foo` (assert app passed through). +- Batch item does not inherit an outer `-w` (assert `for_batch_item` context has `wait_selector: None`). +- snapshot with no wait flags → identical output to today (regression). +- clap: `-w` resolves to `--wait-for`; `--wait-timeout 5000` parses. +**Verification:** `cargo test -p agent-desktop` (binary contract tests) + core snapshot tests pass; `--help` shows the three flags as global. + +### U4. Post-action wait in the ref-action path (`helpers.rs`) + +**Goal:** Ref-resolving action commands perform their action, then wait — scoped to the acted-on ref's app — returning the post-action snapshot with the action result preserved. +**Requirements:** R6, KTD4, KTD5. +**Dependencies:** U2, U3. +**Files:** +- `crates/core/src/commands/helpers.rs` — add `pub(crate) fn apply_post_action_wait(result: Value, app: Option<&str>, context: &CommandContext) -> Result<Value, AppError>`: returns `result` unchanged when context carries no selector; otherwise runs `wait_selector::execute` scoped to `app` (which owns the match-everything guard, U2) and returns the snapshot envelope with `result` embedded under `after_action`. On timeout it surfaces the `wait_timeout` error with the action result in `details.after_action`. +- `crates/core/src/commands/helpers.rs` — in `execute_ref_action_with_context`, stop discarding `_entry`: keep it and pass `entry.source_app.as_deref()` to `apply_post_action_wait` before returning. **This covers the 11 simple ref actions** (`click`, `double-click`, `triple-click`, `clear`, `focus`, `toggle`, `check`, `uncheck`, `expand`, `collapse`, `scroll-to`) in one edit. +- `crates/core/src/commands/{type_text,set_value,select,scroll}.rs` — each receives `(entry, result)` from `execute_ref_action_result_with_context` and ends with `Ok(serde_json::to_value(result)?)` (they do **not** build custom JSON). Wrap that return: `helpers::apply_post_action_wait(serde_json::to_value(result)?, entry.source_app.as_deref(), context)` (4 sites). +- `crates/core/src/commands/right_click.rs` — the **one** command that *does* build a custom JSON body (its menu probe). Call `apply_post_action_wait` **after** the menu-probe JSON is assembled, so ordering is action → menu probe → wait (5th bespoke site). +- `crates/core/src/commands/helpers_ref_action_tests.rs` (**modify — file already exists**) — add post-action merge, scope, timeout, and right-click-ordering tests. +**Approach:** One shared wrapper, **six call sites** (1 helper edit covering 11 commands + 5 bespoke: type/set-value/select/scroll/right-click) — every site already holds the `RefEntry`, so scoping by `entry.source_app` is a field read, not a pid→name lookup. Total coverage = the 16 R6 commands. `execute_by_ref` (FFI bridge) also routes through `execute_ref_action_with_context` but is behaviorally unaffected: the FFI `CommandContext` never carries a `WaitSelector`, so `apply_post_action_wait` early-returns. Snapshot is untouched here (it self-applied in U3), so there is no double-wait. Scoping by `entry.source_app` is the correctness fix over a focused-app hook: in headless mode the acted-on app is generally not frontmost. +**Patterns to follow:** `execute_ref_action_with_context`/`execute_ref_action_result_with_context` in `helpers.rs`; the custom-response assembly in `right_click.rs`; envelope shape in `commands/snapshot::format_result`. +**Test scenarios** (MockAdapter): +- `click @e5 -w ":Saved!"` where the mock tree gains "Saved!" after the action → envelope contains the snapshot **and** `after_action` with the click result. +- Post-action wait is scoped to `entry.source_app`, not the focused app (assert the app handed to `wait_selector` equals the ref's source app even when a different app is "focused" in the mock). +- Post-action selector never matches → `Err` `code==TIMEOUT`, `details.kind=="wait_timeout"`, `details.after_action` carries the action result, `details.snapshot_id` present. +- Action that closes the source app/window mid-wait → times out with the `wait_timeout` envelope (AppNotFound/WindowNotFound swallowed, U2), not a raw `WINDOW_NOT_FOUND`. +- An action command with no wait flag → result returned byte-for-byte unchanged (regression). +- `right-click @e -w ":menuitem"` → wait runs after the menu probe; `after_action` carries the right-click body including its `menu` field. +- A simple ref action (`toggle`), a bespoke serialize-only action (`type`), and `right-click` all merge correctly (covers all three wiring paths). +**Verification:** `cargo test --lib -p agent-desktop-core` ref-action tests pass; `cargo tree -p agent-desktop-core` clean; real smoke deferred to E2E (U5). + +### U5. Docs, help text, and E2E coverage + +**Goal:** Document the flag and prove it against a real app. +**Requirements:** R1–R6 (user-facing surface). +**Dependencies:** U3, U4. +**Files:** +- `skills/agent-desktop/references/commands-interaction.md` and the observation reference — document `--wait-for`/`--wait-for-gone`/`--wait-timeout`, the `role:text` selector, that post-action waits poll the **acted-on ref's app** (`entry.source_app`), the timeout envelope (with `snapshot_id` of the last tree), the unsupported-command `INVALID_ARGS` (R8), the `--root` exclusion (R7), and that batch does not honor an outer `-w` (R9). Note the canonical pattern for unsupported commands like `find`: `snapshot -w "<sel>"` then `find`. +- `src/cli/help_after.txt` — add a `--wait-for` usage example if examples live there. +- `tests/e2e/run.sh` + fixture interactions — one appearance wait and one disappearance wait against the SwiftUI fixture, verified by observation (a control that appears/disappears on action). +**Approach:** Documentation-only + E2E. Keep help text terse; the skills reference carries the detail. E2E asserts by independent observation per the harness contract, not the command's own `ok`. +**Patterns to follow:** existing `--force` documentation added in 0.4.4; `tests/e2e/README.md` conventions. +**Test scenarios:** `Test expectation: none` for docs. E2E: (a) trigger a control that reveals a "Saved!" label, assert `-w ":Saved!"` returns only after it appears; (b) trigger a spinner that clears, assert `--wait-for-gone` returns after it's gone; (c) timeout case returns exit 1 with the `wait_timeout` envelope. +**Verification:** `bash tests/e2e/run.sh` green (headless + `--headed`); skills reference renders. + +--- + +## Scope Boundaries + +**In scope:** the three global flags, the `role:text` selector, the shared matcher extraction, the poll helper (incl. window-close-tolerant retryable set and timeout `snapshot_id`), snapshot self-apply, post-action wait for all 16 R6 commands, the unsupported-command `INVALID_ARGS` guard (R8), the `--root` exclusion (R7), batch isolation (R9), docs + E2E. + +### Deferred to Follow-Up Work +- **FFI `ad_wait_for` entrypoint** — the core `wait_selector` helper is reusable; a cdylib surface is a separate, additive PR (KTD6). +- **Richer selector axes** (`name:`, `value:`, `state:` predicates, multiple constraints in one string) — only `role:text` ships now; the matcher already supports name/value, so a future grammar can expand without re-architecting. +- **`--wait-for` on non-ref commands** (e.g. `launch Foo -w "button:Login"`, `find`) — rejected with `INVALID_ARGS` this PR (R8); the workaround is two calls (`launch Foo; snapshot --app Foo -w "button:Login"`). Extending support to `find`/`launch` is additive follow-up. +- **`--wait-for` in batch mode** — batch items run independently and ignore an outer `-w` (R9). Per-item wait selectors in the batch JSON schema are deferred. +- **Polling-interval flag** — backoff is fixed (200ms→1s) to match `wait`; expose only if measured need arises. +- **`--wait-for-gone` stabilization guard** — see Risks; an optional "require one present observation before accepting absent" guard is deferred unless flakiness is observed. + +### Out of scope (not this product) +- CSS / `:contains()` selector grammar (KTD1). +- A success-on-timeout return path that diverges from the `wait` envelope (KTD5). + +--- + +## Risks & Dependencies + +- **R: post-action wait reads the wrong app's tree (headless default).** Resolved by design: scope is `entry.source_app` (the resolved ref's app), not the focused app — a binary post-dispatch hook was rejected for exactly this reason (KTD4/U4). Test asserts the ref's app is used even when a different app is focused. +- **R: post-action wait errors instead of timing out when the action closes the window.** Resolved: `AppNotFound`/`WindowNotFound` are in the retryable poll set (KTD3/U2), so a window-closing `click @submit -w ":Saved!"` times out with the `wait_timeout` envelope (action result preserved) rather than a raw `WINDOW_NOT_FOUND`. +- **R: empty-needle panic in the matcher.** Resolved: `parse_selector` collapses an empty axis to `None` (KTD1/U1) so `search_text::contains` is never called with a zero-length needle (`windows(0)` panic); U1 tests cover `"button:"`, `":"`, `" : "`. +- **R: `--wait-for-gone` false-positive on a transient empty/partial tree** (first poll fires before the action's AX notification propagates, reads no match, returns "gone"). Confidence moderate. Mitigation this PR: documented limitation + the standard 200ms first-interval gives the tree time to populate; a "require one present observation before accepting absent" guard is deferred (Scope Boundaries) pending observed flakiness. +- **R: `find` regression from the matcher move.** Mitigation: U1 is a pure move with `node_matches`/`tree_has_match` parity tests; `find`'s existing suite must pass unchanged before U2 starts. +- **R: 30s default `--wait-timeout` is long for a blocking post-action call** in an agent pipeline. Mitigation: matches `wait` for consistency; agents set `--wait-timeout` explicitly for tight loops. Documented in U5. +- **R: global `-w` short-flag collision.** Verified clear — only `-v` (global) and `-i` (snapshot) exist today. +- **Dependency:** none external; reuses `snapshot::build`, `RefStore`, `node_matches`/`tree_has_match`, `wait_timeout`, and `RefEntry.source_app`. + +--- + +## Definition of Done + +- All five units landed; `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`, `cargo test --lib --workspace`, `cargo test -p agent-desktop`, and `cargo tree -p agent-desktop-core` (no platform crates) green. +- `snapshot -w "<role:text>"`, `--wait-for-gone`, and post-action `click @e -w "…"` work against the E2E fixture in headless and `--headed` mode. +- Timeout exits 1 with a `kind:"wait_timeout"`, `predicate:"selector"` envelope carrying the last tree's `snapshot_id` and `last_error`; post-action timeout preserves the action result in `details.after_action`. +- All 16 R6 commands honor `--wait-for` (verified: 11 via `execute_ref_action_with_context`, 5 bespoke incl. `right-click`); `find`/`launch`/other unsupported commands return `INVALID_ARGS` (R8); batch ignores an outer `-w` (R9). +- `--wait-for` + `--wait-for-gone` together → clap parse error (exit 2); `--root` + `--wait-for` → `INVALID_ARGS` (R7); a match-everything selector → `INVALID_ARGS`. +- Skills reference and `--help` document the flags; issue #84 referenced in the commit/PR. + +--- + +## Sources & Research + +- Issue #84 (origin) — proposed shape and "why small" framing. +- `crates/core/src/commands/wait.rs` (`wait_for_text`) — poll loop, backoff, persist-on-match, `last_error` pattern reused by `wait_selector`. +- `crates/core/src/commands/find.rs` (`FindQuery`, `node_matches`) — the matcher promoted to `query.rs`. +- `crates/core/src/commands/snapshot.rs` (`build`, `format_result`) — cheap build + envelope shape. +- `crates/core/src/commands/wait_timeout.rs` — `kind:"wait_timeout"` envelope convention (KTD5). +- `src/cli/mod.rs`, `src/main.rs` — global-flag mechanism (`--headed`) and the single post-dispatch seam (`run_with_adapter`/`finish`). diff --git a/docs/plans/2026-06-30-001-refactor-session-first-trace-architecture-plan.md b/docs/plans/2026-06-30-001-refactor-session-first-trace-architecture-plan.md new file mode 100644 index 0000000..4f051de --- /dev/null +++ b/docs/plans/2026-06-30-001-refactor-session-first-trace-architecture-plan.md @@ -0,0 +1,309 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +execution: code +type: refactor +product_contract_source: ce-plan-bootstrap +created: 2026-06-30 +--- + +# refactor: session-first trace architecture + +## Summary + +Make the **session** the first-class container that owns its trace, instead of threading a `--trace <path>` flag through every command. A session created by `session start` records tracing automatically — each process appends to its own segment file under the session directory — and flag/env/pointer activation lets an agent set the session once and never repeat a flag. This closes the footgun where an agent that forgets `--trace` on one command silently loses that command's trace, lights up structured tracing for FFI consumers that opt in, and makes concurrent multi-agent sessions safe by convention enforced with precedence order and a start-time guardrail. The trace **viewer** is a deliberate follow-up plan; this plan builds only the architecture it will sit on. + +Trace-on is gated by the session **manifest** (`session.json`, `trace: on`), not by the mere presence of a `--session` id — so existing `--session` callers and FFI embedders who never run `session start` see today's behavior unchanged and never get surprise files on disk. + +--- + +## Problem Frame + +Sessions and snapshots are already a coherent, lock-protected, hierarchical model on disk (`~/.agent-desktop/sessions/{id}/` with `refstore.lock`, `latest_snapshot_id`, `snapshots/{snapshot_id}/refmap.json`). Tracing is the one piece that does not belong to the session: it is selected per-invocation by `--trace <path>` (`src/cli/mod.rs`, built once in `crates/core/src/context.rs::CommandContext::new`). Three consequences follow, all evidence-backed: + +1. **Silent trace holes.** A command run without `--trace` emits nothing and returns `Ok(())` — no warning. A multi-command run only produces a continuous trace because the agent re-passes the identical path every time. One dropped flag = an invisible gap. +2. **FFI has no structured trace.** `crates/ffi/src/adapter.rs` builds its `CommandContext` with `trace_path: None`, so the entire structured event catalog is CLI-only; Python/Swift/Go/Node consumers get only unstructured log-callback lines. +3. **Shared traces corrupt.** `crates/core/src/trace.rs::write_event` does two separate syscalls (JSON body, then newline) with no buffering and no cross-process lock. Two agents appending to one trace file interleave into invalid JSONL. This bug exists today and becomes the common case the moment a session owns a shared trace. + +Two more gaps block a session-first model: there is **no session lifecycle** (sessions are created implicitly and never cleaned — directories leak, and `discover_snapshot_base` slows as they accumulate), and the **512-snapshot prune** would silently delete snapshots a long session's trace *references* (the trace log itself is unaffected — see KTD5 for the precise boundary). + +Non-goals for this plan: the trace viewer, per-step screenshot/tree artifacts, swapping the lock implementation, and deleting the legacy migration shim (see Scope Boundaries). + +--- + +## Requirements + +- **R1.** When a session **whose manifest has `trace: on`** is active and no explicit `--trace` is given, tracing records automatically to the session directory. No per-command flag is required. A bare `--session <id>` with no manifest selects the snapshot namespace only (today's behavior) and does **not** trace. +- **R2.** Each OS process writes its own trace **segment** (`<session>/trace/<pid>-<proc_start_ts>.jsonl`), computed once per process; concurrent processes write to distinct files and never interleave. Each event line carries a per-process monotonic `seq` so a future reader can tie-break equal `ts_ms`. (Merging segments into a timeline is a reader/viewer concern, deferred.) +- **R3.** Explicit `--trace <path>` still works and overrides the session sink, writing a single file (back-compatible for CI/one-off use). Every event is written with one `write_all` of a fully-buffered line (atomic append). +- **R4.** The active session is resolved once, in precedence order: explicit `--session` > `AGENT_DESKTOP_SESSION` env var > `current_session` pointer file > none. The pointer is written **only** by `session start`. +- **R5.** FFI consumers get the same structured, session-bound trace as the CLI when they use a `trace: on` session — with no per-call flag and no ABI change (the session setter already exists). Setting a session id purely for snapshot scoping does **not** turn on disk writes. +- **R6.** A session lifecycle exists: `session start [--name] [--no-trace]`, `session end [id]`, `session list`, `session gc`. `start` creates the session dir + `trace/`, writes a `session.json` manifest (`trace: on` unless `--no-trace`) and the `current_session` pointer, and prints the id; `end` seals the manifest and clears the pointer; `list` reports the manifest fields (id/name/created/ended); `gc` removes ended and provably-stale sessions. +- **R7.** The trace **log** is never removed by snapshot pruning (segments live outside `snapshots/`). A referenced `snapshot_id` may still be pruned in a long session — full tree replay for pruned state is a viewer concern (KTD5). +- **R8.** Shared-session contract: a session is a shared container; every agent acts on the `snapshot_id` returned by its own snapshot call. Implicit "latest" is a single-agent convenience, not a multi-agent guarantee. +- **R8a.** Independent-session contract: concurrent independent sessions isolate per-process via `AGENT_DESKTOP_SESSION`, never via the global `current_session` pointer (KTD3a). `session start` guards the pointer against silently clobbering a still-live session (KTD3b). +- **R9.** Trace redaction, file-permission hardening (`0600`/`O_NOFOLLOW`, `0700` dirs), and the per-file oversize guard are preserved for the session sink; the manifest `--name` is validated/scrubbed before persistence. +- **R10.** `session gc` will not reap a session with a **live writer** — liveness is checked (active `refstore.lock` pid or recent `trace/` mtime), not creation-age alone. Directory removal uses the same symlink-safe pattern as snapshot prune. +- **R11.** Activating a session relocates the **snapshot/ref namespace** as well as the trace (a session owns both) — this is intentional and documented; explicit `--snapshot <id>` still resolves cross-session, so only implicit "latest" is affected across a `session start` boundary. +- **R12.** No new platform-adapter methods; `agent-desktop-core` stays free of platform crates (CI `cargo tree` gate stays green). Standing constraint, verified by the DoD gate. + +**Success criteria:** an agent runs `session start` (or sets `AGENT_DESKTOP_SESSION`), issues a sequence of commands with no `--trace`, and finds complete, uncorrupted, per-line `ts_ms`+`seq`-ordered segments under the session — even with two agents sharing it — then `session gc` reclaims it without disturbing a live session. + +--- + +## Key Technical Decisions + +**KTD1 — The session owns the trace; `--trace` becomes an override.** Tracing is bracketed by the session lifecycle, mirroring Playwright (`context.tracing.start/stop`), CDP (`Tracing.start/end`), and Appium (session-scoped artifacts). Activation follows the kubectl/ssh-agent convention (set once via env/pointer, per-command override still available) — the correct fit for a one-process-per-command CLI that keeps state on the filesystem rather than in a daemon. + +**KTD1a — Trace-on is gated by the manifest, not by the session id.** Auto-trace requires `session.json` with `trace: on` (written by `session start`; `--no-trace` sets it off). This is the keystone decision: it keeps a bare `--session <id>` a pure namespace selector (today's behavior, no surprise files), wires `--no-trace` through one flag read, and prevents FFI embedders who set a session id for snapshot scoping from getting unexpected data-at-rest. `--trace <path>` still forces tracing regardless of manifest. + +**KTD1b — A session owns snapshots *and* trace (activation relocates both).** `CommandContext.session_id()` already routes `RefStore::for_session()` — where refmaps/`latest_snapshot_id` live — so widening activation (env/pointer) moves the snapshot namespace too, not just the trace. This is intentional and consistent with "a session is the container." Consequence, documented and tested: across a `session start` boundary, implicit "latest" points at the new session; a snapshot taken before is reached only by explicit `--snapshot <id>` (which still resolves cross-session via `discover_snapshot_base`). Without this decision named, the coupling would surface as a mysterious `STALE_REF`. + +**KTD2 — Per-process trace segments, not one shared file.** Each OS process writes `<session>/trace/<pid>-<proc_start_ts>.jsonl`. Lock-free and multi-agent-safe: the two-syscall interleaving bug cannot occur because no two processes share a file. The `(pid, proc_start_ts)` pair is memoized **once per OS process** in a `OnceLock` — so a long-lived FFI host that constructs many `CommandContext`s still writes to one segment (not one per call), and pid reuse across time yields distinct filenames. Each line also carries a per-process monotonic `seq`. The single-file `--trace` override gets the build-line-then-one-`write_all` atomicity fix (R3). Readers must tolerate a truncated final line (crash/OOM/NFS mid-write) by skip-and-warn — a foundation guarantee the viewer relies on. + +**KTD3 — Opt-in per run (a run = a session), stated honestly.** The chosen answer to "the agent forgets `--trace`" is to make the session the trace boundary and start it once per run (`session start`), matching the user's "make each run a session" framing and the Playwright/CI norm. This **reduces** the footgun from once-per-command to once-per-run; it does **not eliminate** it — forgetting `session start` still yields no trace. The rejected alternative, ambient default-on (trace always, zero setup), would fully eliminate it but change default behavior for every existing/CI caller and always write files; it is offered as a deferred, electable option. To make the residual *observable* rather than silent, `status` reports whether a session is active and tracing (no per-command stderr noise). + +**KTD3a — The `current_session` pointer is a single-active-session convenience; concurrent independent sessions use the env var.** The pointer (`~/.agent-desktop/current_session`) is one global file written only by `session start`. It is **not** per-process. Concurrent independent sessions each set `AGENT_DESKTOP_SESSION` (per-process isolation, precedence above the pointer); the pointer is only for a single active session. + +**KTD3b — `session start` guards the pointer against silent clobbering.** Because two agents each calling `session start` would clobber the pointer and cross-contaminate, `start` refuses (with `--force`) when the existing pointer references a **still-live** session (reusing `RefStoreLock`'s pid-liveness). This turns the highest-risk silent-contamination case into a loud failure, so "safe by construction" is honestly "safe by precedence + a start-time guardrail," not a bare convention. + +**KTD4 — Shared container, explicit `snapshot_id` for multi-agent; no per-agent latest.** The `latest_snapshot_id` pointer is a single-agent convenience. The multi-agent contract is: read the shared pool, act on the id your own snapshot returned. No per-agent latest machinery. + +**KTD5 — Trace *log* is decoupled from the working-snapshot cache; referenced-state fidelity is a viewer concern.** Segments live at `<session>/trace/`, never under `snapshots/`, so the 512-cap prune cannot delete the trace. The trace's own per-event data (identity, actionability report, activation-chain steps, post-state) is self-contained. It does **not** guarantee that a `snapshot_id` referenced by an old event still resolves to its full tree — that state can be pruned. Copying per-step refmaps into the trace for full time-travel is deferred to the viewer with a recorded decision (**copy**, Playwright-style). + +**KTD6 — A small `session` core module owns the on-disk contract.** Manifest (`session.json`), the `current_session` pointer, activation resolution, liveness/gc, and list live in one core module; the CLI `session` subcommands are thin wrappers. Reuses `write_private_file` hardening. `gc` uses the snapshot-prune symlink-safe removal pattern. + +--- + +## High-Level Technical Design + +Trace sink selection in `CommandContext::new`: + +``` +explicit --trace <path>? ──yes─▶ single file at <path> (buffered, one write_all) + │ no + ▼ +active session has manifest trace:on? ──yes─▶ segment <session>/trace/<pid>-<procTs>.jsonl + │ no (dir pre-created by `session start`; + ▼ sink opens lazily on first event) +no trace (writer: None) (bare --session, or no session → today's behavior) +``` + +Active-session resolution (once, at process start; drives BOTH snapshot namespace and trace): + +``` +--session <id> ─▶ AGENT_DESKTOP_SESSION ─▶ current_session pointer ─▶ none +``` + +Session directory: + +``` +~/.agent-desktop/ +├── current_session # pointer, written ONLY by `session start` (clobber-guarded) +└── sessions/run-42/ + ├── session.json # id, name(validated), created_at, ended_at?, trace: on|off + ├── refstore.lock # (unchanged) — also the gc liveness signal + ├── snapshots/<id>/refmap.json # 512-cap prune applies HERE only + └── trace/ # pre-created by `session start`; NEVER pruned + ├── 1837-2291.jsonl # agent A (pid+procTs, memoized once/process) + └── 1904-2295.jsonl # agent B +``` + +--- + +## Implementation Units + +### U1. Expose the session directory from `RefStore` + +**Goal:** Give trace wiring and the session module a way to locate a session's directory. +**Requirements:** R1, R2. +**Dependencies:** none. +**Files:** +- `crates/core/src/refs_store.rs` — add `pub(crate) fn base_dir(&self) -> &Path` and `pub(crate) fn trace_dir(&self) -> PathBuf` (`base_dir/trace`). `base_dir` is currently a private field with no accessor. +- `crates/core/src/refs_store_tests.rs` — accessor tests. +**Approach:** Pure additive path accessors, no filesystem side effects (no dir creation). `trace_dir()` on the default-root store is defined for symmetry but is unreachable under the sink-selection logic (no session ⇒ no trace). +**Patterns to follow:** existing private-path helpers (`snapshots_dir`, `lock_path`). +**Test scenarios:** +- `for_session(Some("run-42")).trace_dir()` returns `.../sessions/run-42/trace`. +- Accessors create no directories (assert no side effects). +**Verification:** `cargo tree -p agent-desktop-core` clean; store tests pass. + +### U2. Segment trace writer: per-process filename, lazy open, atomic line, `seq` + +**Goal:** Let `TraceConfig` write a per-process segment in a directory, opened lazily, with atomic lines and a tie-break counter. +**Requirements:** R2, R3, R9. +**Dependencies:** none. +**Files:** +- `crates/core/src/trace.rs` — add a `TraceSink` enum (`File(path)` | `SegmentDir(dir)`). Memoize `(pid, proc_start_ts)` in a process-wide `OnceLock` so all `TraceConfig`s in one process resolve to the same segment filename. **Defer the file open until first `write_event`** (store the path/dir at construction; open on first emit) so no empty segment is created for `version`/`skills`/help invocations. In the segment branch, **create the parent `trace/` dir** (recursive, `0700`) at first open if absent — the current `open_trace_file` does not `mkdir` the parent. Fix `write_event` to serialize the full line (event + `ts_ms` + per-process monotonic `seq` + redacted fields + `session_id`) into a buffer and issue one `write_all`. Preserve `0600`/`O_NOFOLLOW`/oversize hardening and `sanitize_trace_value` for both sink kinds. +- `crates/core/src/trace_tests.rs` — segment/atomicity/seq/lazy-open tests. +**Approach:** Redaction and `session_id` injection are unchanged (run in `write_event`). `--trace-strict` semantics preserved. The `seq` is a per-`OnceLock` `AtomicU64`. +**Patterns to follow:** current `open_trace_file`/`write_event`; the `Arc<Mutex<File>>` in-process guard stays for batch sharing. +**Test scenarios:** +- Two `TraceConfig`s constructed in the same process → same segment filename (OnceLock memoization). +- A fresh session dir with no `trace/` → first `write_event` creates `trace/` and a non-empty segment (the silent-hole regression). +- A `version`/no-op invocation writes **no** segment file (lazy open). +- One `write_event` → exactly one well-formed JSONL line; body and newline never split. +- `seq` increments per event within a process; present on every line. +- Redaction, `session_id`, permission + oversize guards identical to single-file path. +- Reader tolerance: a manually-truncated final line does not make the segment unparseable line-by-line. +**Verification:** trace unit tests pass; each segment is valid JSONL line-by-line. + +### U3. Manifest-gated auto-trace in `CommandContext::new` + snapshot coupling + batch + +**Goal:** Wire the session sink, gated by the manifest; handle batch session overrides; document/test the snapshot coupling. +**Requirements:** R1, R3, R9, R11. +**Dependencies:** U1, U2, U5. +**Files:** +- `crates/core/src/context.rs` — in `CommandContext::new`, select the sink: explicit `trace_path` → single file; else if the active session's manifest has `trace: on` → `RefStore::for_session(session_id)?.trace_dir()` segment sink; else → no trace. In `for_batch_item`, if the item overrides to a **different** session while tracing, re-derive the sink for that session (not just swap `session_id`), or reject the override — never write an item's events into the parent session's segment. +- `crates/core/src/context_tests.rs` — sink-selection + batch + coupling tests. If `context.rs` approaches the 400-LOC limit, extract inline tests to `context_tests.rs` first. +**Approach:** The manifest read goes through the U5 module. `RefStore::for_session` here is a path computation (no writes). Segment opens lazily (U2). +**Patterns to follow:** existing `CommandContext::new`; `with_headed`/`with_wait_selector`. +**Test scenarios:** +- `trace: on` session, no `--trace` → events in `<session>/trace/<pid>-*.jsonl`. +- Bare `--session foo` with **no** manifest → namespace selected, **no** trace (existing-caller regression). +- `session start --no-trace` session → no trace; snapshots still namespaced to it. +- `--trace <path>` overrides regardless of manifest. +- **Coupling:** a snapshot taken before `session start`, then `click @ref` after with implicit latest → resolves against the new session (documents R11); the same ref with explicit `--snapshot <old-id>` still resolves (cross-session). +- Batch item overriding to a different session does not write into the parent's segment. +**Verification:** `cargo test -p agent-desktop` + core context tests pass. + +### U4. FFI: session-bound structured trace (verification + opt-out) + +**Goal:** FFI consumers get the trace via a `trace: on` session; setting a session id alone does not write files. +**Requirements:** R5. +**Dependencies:** U3. +**Files:** +- `crates/ffi/src/adapter.rs` — `ad_adapter_create_with_session` already validates + stores `session_id`, and `command_context()` already passes it to `CommandContext::new`; with U3's manifest gate this yields trace only for a `trace: on` session. **No ABI change** — verification + a smoke test. Confirm an FFI consumer can create a `trace: on` session (via the CLI or a documented call) and that a plain session-id set stays trace-off. +- `crates/ffi/` C-ABI tests — smoke: FFI call under a `trace: on` session writes a segment; under a plain session writes none. +**Approach:** Behavior falls out of U3 + KTD1a; scope is verification + tests. Because the OnceLock memoizes per process (U2), a long-lived FFI host writes one segment, not one per call. +**Patterns to follow:** existing FFI context construction; header/drift tests. +**Test scenarios:** +- FFI call under a `trace: on` session → one segment for the process, structured events. +- FFI call with a plain (no-manifest / `--no-trace`) session → no trace files. +- Header/codegen drift gates green (no ABI change). +**Verification:** `cargo test -p agent-desktop-ffi --tests`; drift gates green. + +### U5. `session` core module: manifest, pointer, resolution, liveness, gc + +**Goal:** One core module owning the session on-disk contract, activation resolution, and safe gc. +**Requirements:** R4, R6, R8, R8a, R9, R10. +**Dependencies:** U1. +**Files:** +- `crates/core/src/session/mod.rs` (new) — `session.json` manifest (id, validated `name`, created_at, ended_at?, `trace: on|off`); `current_session` pointer read/write; `resolve_active_session(explicit, env) -> Option<String>` (flag > env > pointer > none, env above pointer); `is_live(session)` (active `refstore.lock` pid via `RefStoreLock`'s liveness, or recent `trace/` mtime); `list()` (manifest fields only); `gc()` (remove ended + provably-stale-and-not-live; symlink-safe removal mirroring `refs_store_prune`). Validate/scrub `name` before persistence (it bypasses trace redaction). +- `crates/core/src/session/session_tests.rs` (new). +- `crates/core/src/lib.rs` — register `session` (pub boundary). +**Approach:** Resolution is a pure function. `gc` never removes a live or pointer-referenced session. `list` is manifest-only (no subtree walk) to stay within R6's scope. +**Patterns to follow:** `refs_store.rs` path/private-file conventions; `RefStoreLock` pid-liveness; `refs_store_prune` symlink-safe removal; `validate_session_id`. +**Test scenarios:** +- Precedence: explicit > env > pointer > none; env beats a *different* pointer. +- Pointer absent → none (bare command → default behavior). +- Manifest round-trips with/without `ended_at`; `--name` with a control char is scrubbed/rejected. +- `gc` removes ended + stale; **leaves a session with a live `refstore.lock` pid**; leaves a session with recent `trace/` mtime; never removes the pointer-referenced session; refuses to follow a symlinked session dir. +- `list` reports manifest fields without walking `snapshots/`. +**Verification:** core session tests pass; `cargo tree` clean. + +### U6. `session` CLI commands (`start` / `end` / `list` / `gc`) + +**Goal:** The user-facing lifecycle, with the clobber guard and trace-dir pre-create. +**Requirements:** R6, R8a, R9. +**Dependencies:** U5. +**Files:** +- `crates/core/src/commands/session.rs` (new) — `execute()` over the four subactions. `start` creates the session dir + `trace/`, writes the manifest (`trace: on` unless `--no-trace`) and the pointer, prints the id; **refuses to clobber a live pointer without `--force`** (KTD3b). `end` seals + clears the pointer. `list`/`gc` render U5 results. +- `crates/core/src/commands/mod.rs` — register. +- `src/cli/mod.rs` + `src/cli_args/` — `Session` subcommand: `start [--name] [--no-trace] [--force]`, `end [id]`, `list`, `gc [--older-than] [--ended]`. +- `src/dispatch/mod.rs` — dispatch arm. +- `src/cli/contract_tests.rs` — CLI contract. +**Approach:** Follows the Extensibility Pattern (new command file + cli variant + dispatch arm). `start` pre-creating `trace/` removes any first-write dir race. +**Patterns to follow:** a multi-action command (`skills`) + dispatch registration. +**Test scenarios:** +- `session start` creates dir + `trace/` + manifest + pointer, prints a valid id; a subsequent bare command traces. +- `session start` over a **live** pointer without `--force` → refused (loud); with `--force` → overrides. +- `session start --no-trace` → session exists, bare commands do not trace. +- `session end` clears the pointer; subsequent bare command no longer traces/attaches. +- `--session X` explicit overrides an active pointer. +- Envelope/exit-code contract per subaction. +**Verification:** `cargo test -p agent-desktop` pass; `--help` shows `session`. + +### U7. Activation wiring, retention guard, and docs + +**Goal:** Resolve the active session once in the binary, guarantee trace survives pruning, and document the model. +**Requirements:** R4, R7, R8, R8a, R11. +**Dependencies:** U3, U5, U6. +**Files:** +- `src/main.rs` — resolve the active session once via `session::resolve_active_session(cli.session, env)` and thread the resolved id into `CommandContext::new` (batch inherits; not re-resolved per item). +- `crates/core/src/refs_store_prune.rs` — a test asserting prune scans only `snapshots/` and never `trace/` (prune already scopes to `snapshots/`; this pins it). +- `skills/agent-desktop/references/*.md`, `src/cli/help_after.txt`, `CLAUDE.md` — document: session owns trace **and** snapshots (R11 coupling + the explicit-`--snapshot` escape); manifest-gated trace; activation precedence + `AGENT_DESKTOP_SESSION`; per-process segments + `seq`; the shared-vs-independent contracts (R8/R8a); the clobber guard; that trace is opt-in per run and `status` shows tracing state. +**Approach:** Activation resolution lives at the binary edge so batch inherits one resolved id. The prune guard is a test, not new logic. +**Patterns to follow:** `--headed` global-flag threading; existing prune tests. +**Test scenarios:** +- A session past the 512-snapshot cap keeps every `trace/*.jsonl`. +- `AGENT_DESKTOP_SESSION` set + no `--session` → attaches to that session (resolved in `main`). +- `--session` explicit overrides env and pointer. +- Docs updated: `Test expectation: none`. +**Verification:** prune retention test passes; full gate set green. + +--- + +## Scope Boundaries + +**In scope:** manifest-gated session-owned trace, per-process segments (memoized filename, `seq`, lazy open, atomic line, reader-tolerance), `--trace` override, activation (flag/env/pointer), snapshot-coupling documentation, FFI opt-in trace, `session start/end/list/gc` with clobber guard + gc liveness, retention guard, docs. + +### Deferred to Follow-Up Work +- **Trace viewer** — the timeline/tree/activation-chain UI that reads a session's segments and **merges them by `ts_ms`+`seq`** (same-ms ties, cross-process clock skew, truncated-final-line tolerance are the viewer's problems). Separate plan; this plan is its foundation. +- **Per-step artifacts for replay** — `screenshot_id`/`tree_snapshot_id` per action + copying per-step refmaps into the trace (recorded decision: **copy**, Playwright-style). With the viewer. +- **Ambient default-on tracing** — a bounded, auto-gc'd default-session trace that eliminates the forget-`session start` case entirely; electable later if opt-in proves insufficient (KTD3). +- **Session-level trace budget** — a total-bytes cap across a session's segments (per-file 64MB still applies); add if unbounded growth between `gc` runs proves a problem. +- **Session zip/bundle export**, and **legacy `last_refmap.json` shim removal** (low-risk, separable). + +### Out of scope (not this product) +- **Swapping `RefStoreLock` for `flock`** — `flock` is unreliable over NFS and `~/.agent-desktop` can be a network home; the PID+token lock may exist for that portability. Its own PR if ever revisited. +- **Cross-OS-user session isolation / ownership tokens** — sessions are per-OS-user (`0700` home); management commands are trusted within that user. Multi-tenant shared-`$HOME` hardening is a separate concern. +- **A session daemon/server** — the filesystem-session model is the stateless-CLI equivalent. + +--- + +## System-Wide Impact + +- **CLI contract:** new `session` subcommand; new sticky behavior *after* `session start`. Bare-command and bare-`--session` behavior for non-adopters is unchanged (KTD1a). `--trace`/`--trace-strict` preserved. +- **Snapshot resolution:** activation relocates the snapshot namespace, not just trace (R11/KTD1b) — documented + tested; explicit `--snapshot` resolves cross-session. +- **FFI:** structured tracing activates only for `trace: on` sessions (no surprise data-at-rest); no ABI change. +- **Batch:** the resolved session id flows through `for_batch_item`; a per-item session override re-derives its own sink. +- **Two-PR seam:** U1–U4 (manifest-gated session trace + FFI) is the shippable core that kills the footgun; U5–U7 (lifecycle + gc + docs) can land as a second PR. `ce-work` may split there; the plan stays one document. + +--- + +## Risks & Dependencies + +- **R: activation silently relocates snapshot state** across a `session start` boundary. Mitigated: named as intentional (KTD1b/R11), documented, and covered by the U3 coupling test; explicit `--snapshot` still resolves. +- **R: existing `--session` / FFI callers get surprise trace files.** Mitigated by KTD1a — trace-on requires a manifest, which only `session start` writes. +- **R: FFI segment fragmentation** (one segment per call in a long process). Mitigated by the per-process `OnceLock` filename memoization (U2). +- **R: silent trace hole from a missing `trace/` dir.** Mitigated: `session start` pre-creates it and U2's segment branch recursively creates it on first open; regression test. +- **R: `current_session` pointer clobbering** cross-contaminates two agents. Mitigated by KTD3b (`session start` refuses a live pointer without `--force`) + env precedence (KTD3a) + the resolver test. +- **R: `gc` reaps a live env-bound session.** Mitigated by R10 liveness (active lock pid / recent `trace/` mtime), not creation-age alone. +- **R: opt-in-per-run does not eliminate the forget case.** Acknowledged (KTD3); `status` surfaces tracing state; ambient-default deferred as electable. +- **R: crash/NFS truncates the final segment line.** Mitigated: one `write_all` per event + a stated reader skip-and-warn tolerance (KTD2). +- **R: files near the 400-LOC limit** (`context.rs` ~368, `trace.rs` ~322, `refs_store_tests.rs` ~369). Mitigated: extract inline tests to sibling files before adding production code (U2/U3). +- **Sequencing note:** this is core/CLI/FFI plumbing that does not block or compete with the Phase 2 Windows/Linux adapter work (different surface, parallelizable). +- **Dependency:** none external; reuses `RefStore`, `RefStoreLock` (liveness), `TraceConfig`, `write_private_file`, `validate_session_id`, `refs_store_prune`. + +--- + +## Definition of Done + +- A `trace: on` session ⇒ trace records automatically to per-process segments (memoized filename, `seq`, lazy open, one `write_all`); `--trace` overrides to a single atomic file; a bare `--session`, `--no-trace` session, or no session ⇒ no trace. +- Activation resolves flag > env > pointer > none; pointer written only by `session start`; `session start` refuses a live-pointer clobber without `--force`. +- Activating a session relocates snapshot namespace too (documented + tested); explicit `--snapshot` resolves cross-session. +- FFI under a `trace: on` session produces one segment per process; a plain session writes nothing; drift gates green. +- `session start/end/list/gc` work; `gc` never reaps a live or pointer-referenced session and is symlink-safe; `list` is manifest-only. +- A session past the 512-snapshot cap retains all trace segments. +- Contracts documented (shared vs independent sessions; snapshot coupling; opt-in tracing). +- Gates: `cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`, `cargo test --lib --workspace`, `cargo test -p agent-desktop`, `cargo test -p agent-desktop-ffi --tests`, `cargo tree -p agent-desktop-core` (no platform crates), and E2E green. + +--- + +## Sources & Research + +- Investigation (this session): session/snapshot store map, trace subsystem map, over-engineering/debt audit, external-pattern research, and a six-persona doc review — evidence for every problem and decision. +- Current code: `crates/core/src/{refs_store.rs, refs_store_prune.rs, refs_lock.rs, trace.rs, context.rs, snapshot.rs, commands/helpers.rs, commands/batch.rs}`, `crates/ffi/src/adapter.rs`, `src/cli/mod.rs`, `src/main.rs`. +- External patterns (load-bearing): Playwright `context.tracing.start/stop`; Appium/WebDriver session lifecycle; CDP `Tracing.start/end` + Target/session; kubectl/`docker context`/ssh-agent set-once active-context. These shaped KTD1/KTD1a (session-bounded, manifest-gated trace), KTD2 (segments), and the activation model (R4). +- Debt note: a `ponytail:`/TODO/FIXME harvest found **zero** in-source markers — deferrals are undocumented, not absent. diff --git a/docs/plans/2026-07-01-001-feat-trace-viewer-replay-artifacts-plan.md b/docs/plans/2026-07-01-001-feat-trace-viewer-replay-artifacts-plan.md new file mode 100644 index 0000000..0f99dcf --- /dev/null +++ b/docs/plans/2026-07-01-001-feat-trace-viewer-replay-artifacts-plan.md @@ -0,0 +1,382 @@ +--- +title: Trace Viewer and Replay Artifacts - Plan +type: feat +date: 2026-07-01 +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-plan-bootstrap +execution: code +--- + +# Trace Viewer and Replay Artifacts - Plan + +## Goal Capsule + +- **Objective:** Build the trace read-and-replay layer on the session-first foundation: deterministic merged timelines (R1–R5), a versioned and tolerant format contract (R6–R7), replay-complete event enrichment (R8–R9), opt-in screenshot and refmap artifacts (R10–R13), a single-file HTML viewer (R14–R17), and additive, backward-compatible contract safety (R18–R19). +- **Authority:** This plan takes precedence over general repo conventions where the two conflict; repo conventions take precedence over implementer judgment; `CLAUDE.md` rules (the 400 LOC cap, zero `unwrap()`, no inline comments, core/platform isolation) remain non-negotiable regardless of what any unit implies. +- **Execution profile:** Units U1 and U5 are test-first — their enumerated test scenarios are written as failing tests before the implementation they specify. Every unit lands with its full enumerated test list; no scenario is dropped as a shortcut. +- **Stop conditions:** Any change that would require a new `PlatformAdapter` trait method, a new `ErrorCode` variant, an `ENVELOPE_VERSION` bump, or a new external dependency is out of contract (KTD9). Hitting one mid-implementation means stop and surface, not improvise around it. + +--- + +## Product Contract + +### Summary + +Add the trace read-and-replay layer that the session-first trace architecture (`docs/plans/2026-06-30-001-refactor-session-first-trace-architecture-plan.md`) deferred. `trace show` merges every segment of a session into one deterministic timeline, returned as a bounded JSON envelope agents can consume without exhausting their context window. `trace export` renders that timeline into a single self-contained HTML file — embedded JSON, embedded screenshots, an inline viewer — that opens from `file://` with no server and no network calls. + +Two enrichments make replay complete. Every command now emits a `command.start`/`command.end` boundary pair regardless of how it fails, and an opt-in `artifacts: full` session mode captures pre- and post-action screenshots and refmap copies around every ref action. The format itself carries a per-segment `trace.meta` header and an additive-only evolution rule, so a schema-0 trace from v0.4.6 and a future schema-N trace both read cleanly through the same reader. + +This plan touches no `PlatformAdapter` trait method, adds no error code, and does not bump `ENVELOPE_VERSION`. Both new commands are additive JSON on top of the existing envelope, wired through CLI, batch, and FFI via the same core command functions. + +### Problem Frame + +The session-first foundation gives every process a per-segment, lock-free trace sink, but nothing reads it back. Today's event catalog is thin and write-path-only: `ref.resolve.start/entry/ok/error` (`crates/core/src/commands/helpers.rs`), `actionability.check.start/ok/error` plus `action.dispatch.start/ok` (`crates/core/src/ref_action.rs` — `action.dispatch.ok` already carries the full `ActionResult`, including `post_state` and the activation-chain `steps: Vec<ActionStep>`), `input.focus_app` (`crates/core/src/commands/point_resolve.rs`), and `snapshot.root.saved` (`crates/core/src/snapshot_ref.rs`). + +There is no `command.start`/`command.end` pair, so a command that fails before ref resolution — a bad argument, a policy denial, a missing session — leaves zero trace of the attempt and no duration. The main snapshot path never emits `snapshot.saved`, so a trace cannot even enumerate which snapshots a session produced. Segments carry no self-description: nothing marks their schema or the binary version that wrote them, so a future format change has no way to warn a reader instead of silently misparsing. + +No per-step visual state is captured, so a human reviewing a trace sees JSON event names with no picture of what was on screen. And nothing merges segments into a timeline at all — not a reader, not a viewer, not even a sort-and-concatenate script. A session's trace directory is, today, a pile of unrelated JSONL files. + +### Requirements + +**Timeline reading** + +- R1. `trace show` merges every segment of a session into one deterministic timeline and returns it in the standard JSON envelope, honoring session activation precedence (flag > `AGENT_DESKTOP_SESSION` > `current_session` pointer) and requiring no accessibility or screen-recording permission. +- R2. Merge order is ascending `ts_ms`; equal `ts_ms` breaks ties by `(writer pid, in-file position)`. Within one segment, file order (the `seq` order) is never violated even when `ts_ms` regresses from a wall-clock adjustment — per-process causality is authoritative, and cross-process same-millisecond order is best-effort but deterministic. Output is independent of segment discovery order. Cross-process ordering is guaranteed only for same-millisecond ties; a wall-clock step large enough to separate causally-ordered events in different processes by more than a tie is an undefended, documented best-effort boundary. +- R3. Reader tolerance is a hard contract: a truncated final line from a mid-write crash is skipped and counted; a corrupt line mid-file is skipped and counted; non-object JSON lines are skipped and counted; files in `trace/` that don't match the `<pid>-<procStartTs>.jsonl` segment naming (or `*.tmp`) are ignored with a warning; a line longer than 8MiB is rejected and counted; a segment file that cannot be opened, or whose name resolves to a symlink, is skipped with a counted warning (segment opens use `O_NOFOLLOW`, mirroring the refmap store's open discipline); a `command.start` with no matching `command.end` (a killed process defeats the `Drop` guard) is surfaced as a counted warning and rendered as an open, incomplete group rather than dropped. The reader never errors on malformed content — malformed content degrades to counted warnings in the response. Warnings are machine-readable: each entry carries a closed kind (`foreign_file`, `unreadable_segment`, `symlinked_segment`, `schema_unknown`, `unpaired_command`) plus a human message, so agents branch on kind, not free text. +- R4. Each merged event is annotated with provenance: `writer_pid` and `segment` (the filename stem). The existing `pid` field on events like `ref.resolve.entry` — the target app's pid — is never overwritten. +- R5. Output is bounded for agent context windows: `--limit N` returns the last N events of the merged timeline (default 500; `--limit 0` means all); the response carries `total_events`, `returned_events`, and `truncated: true|false`. `--event <prefix>` filters by event-name prefix before the limit is applied. The tail slice is positional and may split a `command.start`/`command.end` pair; the envelope's `truncated` flag plus the `unpaired_command` warning kind mark the cut, and the viewer renders the affected group as open-incomplete rather than pretending completeness. + +**Format contract** + +- R6. Every segment opens with a `trace.meta` header event: `schema: 1`, binary version, `os`, `pid`, `proc_start_ms`, `session_id`. Segments without one — every trace from v0.4.6 and earlier — are read as schema 0, fully supported. A schema greater than the reader's known maximum produces a warning and a best-effort parse, never an error. +- R7. Trace format evolution is additive-only: new event types and new optional fields may appear, but existing field meanings never change. Readers ignore unknown event types and unknown fields, passing them through to output verbatim. + +**Replay completeness (event enrichment)** + +- R8. Every command run through the binary dispatch — CLI and each batch item — and through the five generated FFI command entrypoints (execute-by-ref, snapshot, status, version, wait) emits `command.start` `{command}` and `command.end` `{command, ok, duration_ms}`, plus `code` and `message` on failure. A command that fails preflight, policy, or resolution still yields its boundary pair. If a command panics or is interrupted after start, a guard emits `command.end` with `ok: false, code: INTERNAL` on unwind where recoverable. Hand-written FFI command surfaces stay uninstrumented at the command-boundary level in this plan (see R-G); per-action events still fire for FFI ref actions routed through the shared ref-action seam. +- R9. Snapshot creation is traced: the main snapshot path and wait-produced snapshots emit `snapshot.saved` `{snapshot_id, ref_count, app when known}`; the existing `snapshot.root.saved` on the drill-down path is unchanged, per the additive-only rule. + +**Replay artifacts (opt-in)** + +- R10. `session start --screenshots` records `artifacts: full` in the session manifest (default `events`). An old binary reading a new manifest ignores the field — `SessionManifest` carries no `deny_unknown_fields`. A new binary reading an old manifest defaults to `events` via `#[serde(default)]`. `status` surfaces the artifacts mode for the active session. +- R11. When tracing is active and `artifacts: full`, every ref action captures a pre-action and post-action screenshot of the acted-on app (`ScreenshotTarget::Window(entry.pid)`, PNG as returned by the existing adapter method), written under `<session>/trace/screens/` with process-collision-free names, as `0600` files in a symlink-guarded `0700` directory. A per-process budget of 128MiB and 200 captures bounds each process's contribution; a session driven by many separate invocations accumulates one allowance per process, so the session-level footprint is not bounded by this mechanism (see System-Wide Impact). Capture failure, budget exhaustion, missing screen-recording permission, or an adapter without screenshot support (the Windows and Linux stubs) skips the capture with a machine-readable reason — it never fails or slows the action beyond the capture attempt itself. +- R12. Each ref action with artifacts enabled emits `action.artifacts` `{ref, screenshot_pre, screenshot_post, skipped reasons when applicable}`, with paths relative to the session's trace directory. +- R13. When `artifacts: full`, every refmap save — a new snapshot, a drill-down re-save, a wait-produced snapshot — also copies the refmap JSON to `<session>/trace/refmaps/<snapshot_id>.json`. The copy is idempotent (first-write-wins, atomic tmp-then-rename) and shares a 64MiB per-process budget with skip-and-count on exhaustion. Snapshot pruning (the 512 cap) can therefore never break replay for a step whose refmap was already copied; a refmap that was budget-skipped and later pruned is gone, and the reader and viewer surface a placeholder plus a count for it, mirroring the screenshot embed-budget handling (R16). Refmap copies are intentionally unredacted, matching Playwright's full-fidelity replay — this is exactly why artifacts are opt-in. + +**Human-viewable export** + +- R14. `trace export [--out <path>] [--limit N]` writes one self-contained HTML file, defaulting to `--limit 5000` (ten times `trace show`'s tail default, serving the comprehensive human artifact) with `--limit 0` embedding the full timeline — no network fetches, no external files, works from `file://` — with the merged timeline embedded as JSON, screenshots embedded as base64 data URIs, and a viewer UI with inline CSS and JS. The default output path is `trace-<session_id>.html` in the current directory. +- R15. Export is XSS-safe against trace-controlled content — app window titles, element names, and error messages are all attacker-influenceable. Trace JSON is embedded in a `<script type="application/json">` block with `<`, `>`, `&`, U+2028, and U+2029 escaped as `\uXXXX` in the serialized JSON. The viewer JS renders all data via `textContent`/DOM text nodes, never `innerHTML` with data. Screenshot data URIs are validated against the base64 charset before assignment. +- R16. Export is bounded: total embedded screenshot bytes cap at 100MiB (beyond the cap: a placeholder plus a count in the response); total serialized JSON is guarded at 200MiB (beyond the guard: `INVALID_ARGS` with a suggestion to use `--limit`). Export output is byte-deterministic for identical inputs — no timestamps or randomness are injected. +- R17. Redacted event fields — written as `{"redacted": true}` by the foundation — render distinctly in the viewer (for example `⟨redacted⟩`), never as raw JSON noise. + +**Contract safety** + +- R18. Both commands are additive: no existing envelope shape changes, `ENVELOPE_VERSION` does not bump (per `docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md`), and no new error codes are introduced — a missing session or trace maps to `INVALID_ARGS` with a suggestion, mirroring `session end`. The FFI error-code ABI and header asserts stay untouched. +- R19. `trace show` and `trace export` work over FFI through the same core command functions — the generated FFI entrypoints extend through the `crates/ffi/build.rs` generator, never by hand-editing `@generated` files — and both are available in batch mode. + +### Scope Boundaries + +**Deferred to follow-up work:** + +- A served or interactive viewer app (web server, TUI) — the no-GUI identity holds; revisit only on demand. +- Zip or portable bundle export — the single HTML file is the shareable artifact. +- Video or screencast recording — per-action stills suffice. +- JPEG or scaled screenshot encoding — needs adapter API changes (the macOS implementation is PNG-only today, `crates/macos/src/system/screenshot.rs`); the byte budget caps volume instead. Revisit if budgets prove tight in practice. +- Per-action full accessibility-tree capture — `post_state` and the activation-chain `steps` already in `action.dispatch.ok`, together with screenshots and refmap copies, reconstruct the step without a second tree walk, and a per-action tree capture would visibly slow every action on desktop AX. +- Screenshots for non-ref physical input commands (`mouse-click --xy`, a bare `press`) — there is no resolved target pid at a shared seam; capture stays scoped to ref actions in v1. +- Live tailing (`trace show --follow`). + +**Outside this product's identity:** embedding an LLM analysis of traces; a GUI mode in the binary. + +### Acceptance Examples + +- AE1. Given a segment whose final line is cut mid-JSON with no trailing newline, when `trace show` runs, then the response succeeds, the event is absent, and the segment's `skipped_lines` is 1. +- AE2. Given two segments where process A wrote seq 5 at ts 1000 and seq 6 at ts 999 (a clock regression) and process B wrote at ts 999, when merged, then A's seq 5 still precedes A's seq 6, and B's ts-999 event orders relative to A's by the deterministic `(ts, pid, position)` rule. +- AE3. Given a session traced by a hypothetical newer binary writing `schema: 2` meta plus unknown event types and fields, when `trace show` runs, then events pass through verbatim, a schema warning is present, and the exit code is 0. +- AE4. Given a v0.4.6-era trace directory with no `trace.meta` lines, when `trace show` or `trace export` runs, then the full timeline is produced with no warnings about missing meta — schema 0 is first-class. +- AE5. Given `artifacts: full` and a fixture app whose window title is `<script>alert(1)</script><img src=x onerror=alert(2)>`, when `trace export` runs and the file is opened, then no script executes and the title renders as literal text. +- AE6. Given the 200-capture budget is exhausted mid-session, when a further click runs, then the click succeeds, `action.artifacts` carries `skipped: "budget"`, and `command.end.ok` is true. +- AE7. Given screen-recording permission is denied, or the platform adapter returns not-supported, when a ref action runs with `artifacts: full`, then the action behaves exactly as with `artifacts: events` except for the `action.artifacts` skip reason. +- AE8. Given no active session — no flag, env var, or pointer — when `trace show` runs, then `INVALID_ARGS` returns with a suggestion naming `session start` or `--session`. +- AE9. Given the same session read twice, with segments listed by the OS in different orders, when `trace export` runs twice, then the two HTML files are byte-identical. + +--- + +## Planning Contract + +### Key Technical Decisions + +- KTD1 — Viewer is the CLI merged timeline plus a single-file HTML export; no server, no GUI mode. `trace show` is the agent surface: envelope JSON bounded by `--limit` tail semantics and a `truncated` flag, so agents never blow their context window. `trace export` is the human surface: one static HTML file, openable anywhere, attachable to an issue. Playwright ships a web-app viewer, but this repo's identity is machine-facing — no GUI, no TUI — so the shareable artifact here is a file, not a program. Playwright 1.59's `npx playwright trace` (`actions --grep`, `action <n>`, `snapshot <n>`) is a stdout-only, no-browser trace reader built for CI and agent loops, and validates `trace show`'s list/filter shape as proven prior art. +- KTD2 — Merge is a k-way heap merge keyed on `(ts_ms, pid, file-position)`; per-segment file order is inviolable. Same-machine processes share a clock, so "skew" reduces to non-monotonic wall time; `seq`/file order preserves in-process causality, and the `(pid, position)` tie-break makes cross-process ties deterministic rather than pretending microsecond truth. Segments sort by filename before merging, which makes the result independent of segment discovery order (R2, AE9). +- KTD3 — Reader tolerance is a hard contract, not best-effort politeness (R3): every malformed shape degrades to a counted warning. The foundation plan's KTD2 promised that readers must tolerate a truncated final line; this plan turns the full tolerance matrix — truncated tail, corrupt line, foreign file, oversized line, non-object line — into an explicit, tested guarantee, because multi-process append-only files will exhibit all of these in real crashes. +- KTD4 — Format versioning is a `trace.meta` header event per segment plus additive-only evolution (R6/R7). An event, rather than a filename suffix or a sidecar file, keeps the format single-channel, survives file copies, costs one line, and lets v0.4.6 traces remain schema 0 with no migration. The reader accepts an unknown schema with a warning — forward-lenient, because traces are diagnostic artifacts and refusing to read one is strictly worse than a best-effort parse. Playwright has no trace-format version contract at all: compatibility is empirically "mostly forward," a 1.32 viewer once rendered old traces as a silent blank panel, and the request for a version-mismatch warning (playwright#21898) is still open. `trace.meta`'s schema field and its warn-not-fail behavior is the direct fix for a gap Playwright users have paid for. +- KTD5 — Command boundary events go through a `CommandContext::command_scope` guard wired at the binary's `dispatch::dispatch` — which covers CLI and every batch item, since batch items reuse the same dispatch with their own child contexts — and at the FFI generated entrypoints (a generator template change in `crates/ffi/build.rs`). One core implementation, two thin integration points; a `Drop` safety net emits a failure `command.end` if a scope is abandoned. `command.start` carries only the command name — the command's own events carry the specifics, and including arguments would drag sensitive payloads through redaction for no replay value. +- KTD6 — Screenshots are opt-in via the manifest's `artifacts: full` (set by `session start --screenshots`), captured only around ref actions at the single shared seam `ref_action::execute_resolved` — pre-capture after the actionability check (before activation — the activation chain, including scroll-into-view, runs inside `execute_action`), post-capture after execution on both the success and failure paths — targeting `ScreenshotTarget::Window(entry.pid)`, the acted-on app. Window-scoped beats full-screen: smaller PNGs (the adapter is PNG-only, and no new image dependency is allowed), less privacy exposure, and capture resolves through the platform's pid-to-window heuristic — on macOS the largest visible window for the pid, not the frontmost as the trait doc-comment stale-claims (U5 corrects the comment) — so smaller same-pid surfaces (menus, sheets, popovers) and the non-largest windows of multi-window apps can be missed (R-C). Cross-app side effects are a documented v1 miss. Budgets of 128MiB and 200 captures per process, enforced by process-local atomic counters, bound volume; every skip carries a reason in `action.artifacts` (R11/R12). Capture is best-effort by construction — an action must never fail because a screenshot did. +- KTD7 — Refmap copies happen at snapshot-save time, not action time. Copying `snapshots/<id>/refmap.json` to `trace/refmaps/<id>.json` when a refmap is saved is naturally idempotent, catches every snapshot an action could later reference, and decouples replay from the 512-snapshot prune — the foundation plan's recorded "copy, Playwright-style" decision, now executed. Copies are raw and unredacted; element names are the replay value, the same way Playwright copies the full DOM, which is exactly why `artifacts: full` is opt-in and the sensitivity is documented (R13). Playwright itself ships no content redaction despite multi-year demand — a request for password-protected traces (playwright#28934) was closed "not planned," and only screenshot masking exists — so agent-desktop's write-time field redaction is already ahead of that prior art; this plan keeps redaction intact for `events` mode and documents the opt-in, unredacted artifacts explicitly rather than following Playwright's precedent of no redaction at all. A per-action full accessibility-tree capture would also visibly slow every action on desktop AX, so this plan skips that: `post_state` plus the activation-chain `steps` already in `action.dispatch.ok`, together with screenshots and refmap copies, reconstruct the step without a second tree walk. +- KTD8 — HTML export embeds data as a JSON `<script type="application/json">` island with `<`-style escaping, and the viewer renders exclusively via `textContent` — the Lighthouse report-generator pattern, which sanitizes embedded JSON by escaping `<` to its `\uXXXX` code point (plus U+2028 and U+2029) so `</script>` becomes unrepresentable inside the payload. This makes XSS structurally impossible rather than sanitization-dependent (R15). Viewer assets are three `include_str!` files (HTML, CSS, JS), following the `commands/skills.rs` embedding precedent, with zero new dependencies (core already carries `serde_json` and `base64`). Playwright's own HTML report claims to be "one file" — a base64-zip inside a `<script id="playwrightReportBase64">` block — yet breaks when opened via `file://`, because its embedded viewer leans on service-worker and virtual-filesystem fetch interception, which refuses to register on a `file:` origin; Playwright ships a companion server (`show-report` on localhost:9323) to work around it. That failure is the decisive evidence for this plan's shape: resolving everything at generation time — an inline JSON island and inline base64 images, no runtime fetch or service-worker indirection — is the only single-file design that works from `file://`. +- KTD9 — Zero new error codes, zero envelope-version bump, zero `PlatformAdapter` trait changes. A missing session or trace maps to `INVALID_ARGS` with a suggestion, mirroring `session end`; the new commands are additive data, per the envelope-bump solution doc; capture reuses the existing `screenshot(ScreenshotTarget) -> ImageBuffer` adapter method with its default `not_supported` on stub platforms. This keeps the FFI ABI (error discriminant pins, header asserts) and `adapter.rs` (397 of 400 LOC) untouched. +- KTD10 — LOC-cap pre-splits are named up front. `crates/core/src/trace.rs` (389 of 400 LOC) sheds its sanitizer into `crates/core/src/trace_sanitize.rs` (re-exported; the FFI `log_callback` depends on `sanitize_trace_value`) before gaining `trace.meta` emission. `src/dispatch/mod.rs` (394 of 400 LOC) extracts the session and trace match arms into a sibling dispatch module before gaining the trace arm. New reader and artifact code lives in new modules (`crates/core/src/trace_read/`, `crates/core/src/trace_artifacts.rs`) rather than being squeezed into capped files. +- KTD11 — `trace` joins the permissionless policy class: the `Commands::Version | Skills | Session => None` arm in `src/command_policy/mod.rs` gains a `Trace(_)` case. Reading or exporting traces must work on a machine with zero permissions granted — CI, or post-hoc analysis of a copied session directory. + +### High-Level Technical Design + +The write path fans out across concurrent processes into per-process segments, plus artifacts alongside them; the reader discovers, tolerantly parses, and merges those segments back into one timeline that both new commands consume. + +```mermaid +flowchart TB + P["N processes: CLI, batch, FFI"] --> SEG["Segment pid-ts.jsonl"] + SEG --> SCR["screens/*.png"] + SEG --> RFM["refmaps/id.json"] + SEG --> RD["trace_read: parse + merge"] + SCR --> RD + RFM --> RD + RD --> SHOW["trace show (JSON envelope)"] + RD --> EXP["trace export (single HTML)"] +``` + +A single ref action with `artifacts: full` shows the capture path end to end; the refmap-copy path runs independently on every snapshot save. + +```mermaid +sequenceDiagram + participant Cmd as Command + participant RA as ref_action + participant Cap as Capture + participant Ad as Adapter + participant Tr as Trace + + Cmd->>RA: resolve ref + RA->>RA: actionability check + RA->>Cap: screenshot PRE + Cap-->>RA: PNG or skip reason + RA->>Ad: execute_action + Ad-->>RA: post_state, steps + RA->>Cap: screenshot POST + Cap-->>RA: PNG or skip reason + RA->>Tr: action.dispatch.ok + RA->>Tr: action.artifacts + + Note over Cmd,Tr: separately, on snapshot save + Cmd->>Tr: write refmap + Cmd->>Tr: copy to trace/refmaps (idempotent) +``` + +The write side stays lock-free: per-process segment files, process-local atomic budget counters, and first-write-wins tmp-then-rename copies need no cross-process coordination. The read side is pure `std` file reading inside `agent-desktop-core`, with no platform crate involvement, so `trace show` and `trace export` work identically on Windows and Linux the day those adapters land. U1–U3 (reader, `trace show`, enrichment) can land and prove the agent surface ahead of U4–U6; U2 and U3 both edit the dispatch module, so they land serially in that order. + +### System-Wide Impact + +- FFI consumers gain command boundary events and two new generated entrypoints. The ABI addition is purely additive: the header regenerates via the existing script, and no error code changes. +- Windows and Linux (Phase 2): the reader, `trace show`, and `trace export` are pure core, so they work on day one. Capture degrades to a reasoned skip through the adapter's default `not_supported`, proven by unit U5's stub-adapter test. +- Privacy posture changes under `artifacts: full`: raw pixels and unredacted refmap copies live under the session directory (`0600`/`0700`) and flow into exports. Documentation must say plainly that an exported HTML file should be treated like a screenshot of the screen. Event-field redaction is unchanged for `events` mode. One documented exception: `command.end`'s `message` is free text from the failing command's error string, not subject to key-based redaction, so caller-supplied predicate text (a wait selector, a window title) can surface verbatim in events-mode traces and their exports. +- Trace disk figures (64MiB per segment, 128MiB screenshots, 64MiB refmap copies) are per-process ceilings, not session bounds: every new process gets a fresh allowance, so an active long-running session's total footprint is unbounded across many invocations. Periodic session rotation — end the session and start a new one, then `session gc` — is the current mitigation; session-wide accounting is deliberately out of scope (lock-free write side). + +### Risks & Dependencies + +- R-A. `dispatch/mod.rs` and `trace.rs` are already at the 400-LOC cap. The named pre-splits (KTD10) are the first task inside U2 and U3 respectively; the splits are mechanical — moving match arms and the sanitizer to sibling files, not rewriting logic. +- R-B. PNG-only screenshots on a retina display can exhaust the 128MiB capture budget in long sessions. Budgets are fixed constants, so the consequence is skipped captures with reasons, never a failed action; JPEG or scaled encoding is deferred as an adapter change. Trace and single-file-report ecosystems run large by default — Playwright traces commonly land 10-50MB and often exceed 100MB with no built-in cap (playwright#8263, playwright#29218), and Allure's single-file mode inflates roughly 50% via base64 with a practical ~500MB browser ceiling — which is why this plan sets explicit, enforced budgets (128MiB/200 captures for capture, 100MiB/200MiB for export) rather than leaving either unbounded. +- R-C. `Window(pid)` capture can miss the acted-on surface two ways: cross-app effects (a system dialog, another process's window), and same-app misses — the macOS resolution picks the largest visible window for the pid, so a smaller menu, sheet, or popover, or the non-largest window of a multi-window app, may not be captured. Both are documented v1 boundaries (Scope Boundaries); U5 asserts the resolution behavior as a stated, tested contract and corrects the trait doc-comment's stale "frontmost" claim. +- R-D. The viewer JS must stay under 400 lines. Feature scope for U6 is pinned in its Approach; growth pressure routes to the deferred served-viewer follow-up, not into this file. +- R-E. Multiple `trace.meta` lines can appear in a shared explicit `--trace` file, since each writer opens with its own meta line. The reader rule is pinned by U1's test scenario 17: the first meta line wins, later ones pass through as regular events. +- R-F. The e2e hostile-window-title case depends on the fixture app's ability to set a window title containing markup. A fallback is documented in U7; unit-level AE5 coverage remains the guarantee regardless. +- R-G. Command-boundary events over FFI cover only the five generated entrypoints; the hand-written FFI command files (apps, windows, notifications, observation, screenshot, input, surfaces) stay uninstrumented at that level. Follow-up work; per-action dispatch events still cover FFI ref actions. +- R-H. `trace show` and `trace export` output can contain content the reader did not generate — a copied or tampered trace directory is a first-class use case (KTD11) — and R7 passes unknown fields through verbatim. Sanitizing trace-derived content against prompt injection remains the calling agent's responsibility, consistent with the repo's Non-Goals; the reader's job is tolerance and provenance, not trust. + +### Sources & Research + +- Foundation plan `docs/plans/2026-06-30-001-refactor-session-first-trace-architecture-plan.md` — its own R2 (per-process segments with a per-line `seq`), R7 (the trace log survives snapshot pruning), KTD2 (per-process segments), and KTD5 (the trace log decoupled from the snapshot cache) define the merge and copy contracts this plan executes; its "Deferred to Follow-Up Work" section is this plan's scope. +- `docs/solutions/best-practices/playwright-grade-desktop-reliability-2026-06-02.md` — the reliability contract touching trace output and FFI parity; read before implementing U3 and U5. +- `docs/solutions/best-practices/envelope-version-bump-contract-2026-05-13.md` — grounds KTD9's no-bump call. +- Code anchors: `crates/core/src/ref_action.rs` (the `execute_resolved` seam; `action.dispatch.ok` already carries `post_state` and `steps`), `crates/core/src/trace.rs` (`WriterState`, the 64MiB cap, segment naming), `crates/core/src/refs_store.rs` (the 512-snapshot prune cap, the 1MiB refmap cap, snapshot paths), `crates/core/src/session/manifest.rs` (serde compatibility posture), `src/command_policy/mod.rs` (the permissionless arm), `src/cli_args/session.rs` and `src/batch/mod.rs` (wiring mirrors for the new `trace` surface), `crates/core/src/commands/skills.rs` (the `include_str!` embedding precedent), `crates/macos/src/system/screenshot.rs` (evidence the adapter is PNG-only today), `crates/ffi/build.rs` and `crates/ffi/src/commands/generated.rs` (the FFI codegen this plan extends). +- External prior art — Playwright and the single-file-report ecosystem: + - Playwright's HTML report claims to be "one file" (a base64-zip inside a `<script id="playwrightReportBase64">` block) but breaks when opened via `file://`, because its embedded viewer depends on service-worker and virtual-filesystem fetch interception that refuses to register on a `file:` origin — Playwright ships a companion `show-report` server on localhost:9323 to work around it. Shapes KTD8. + - Playwright has no trace-format version contract: compatibility is empirically "mostly forward," a 1.32 viewer once rendered old traces as a silent blank panel, and the request to warn on a version mismatch (playwright#21898) is still open. Shapes KTD4. + - Playwright 1.59 added `npx playwright trace` (`actions --grep`, `action <n>`, `snapshot <n>`), a stdout-only, no-browser trace reader built for CI and agent loops. Validates KTD1's `trace show` shape as proven prior art. + - Playwright ships no content redaction despite multi-year demand — a request for password-protected traces (playwright#28934) was closed "not planned," and only screenshot masking exists. Shapes KTD7's decision to keep redaction intact for `events` mode while documenting the opt-in, unredacted artifacts explicitly. + - Trace sizes in the wild run 10-50MB typical and 100MB+ common with no built-in caps (playwright#8263, playwright#29218); Playwright screencast frames are JPEG-scaled to fit 800×800; Allure's single-file mode reports roughly 50% base64 inflation and a practical ~500MB browser ceiling. Calibrates R16's budgets and the R-B risk mitigation. + - Lighthouse's report generator sanitizes embedded JSON by escaping `<` (plus U+2028 and U+2029) to `\uXXXX` code points so `</script>` is unrepresentable inside the payload. This is the precise mechanism R15 mandates and KTD8 adopts. + - Sources: playwright.dev/docs/api/class-tracing; playwright.dev trace-viewer documentation; deepwiki.com/microsoft/playwright (trace.zip and HTML report internals); playwright.dev/docs/test-reporters; github.com/microsoft/playwright issues #21898, #28934, #19992, #29218, #8263; github.com/GoogleChrome/lighthouse `report-generator.js`; the allure-framework discussion #2854; the pytest-html user guide (`self-contained-html` limits). + +--- + +## Implementation Units + +### U1. Trace reader engine (`trace_read`) + +- **Goal:** A deterministic, tolerant segment discovery and merge engine, as a pure-core module returning a typed result — events as `serde_json::Value` plus per-segment stats and warnings. +- **Requirements:** R1 (engine half), R2, R3, R4, R6 (read side), R7. +- **Dependencies:** None. +- **Files:** NEW `crates/core/src/trace_read/mod.rs` (public API `read_merged(store_trace_dir, ReadOptions) -> MergedTrace`); NEW `crates/core/src/trace_read/segment.rs` (filename parsing for `<pid>-<ts>`, a tolerant line iterator, per-segment stats); NEW `crates/core/src/trace_read/merge.rs` (k-way merge, provenance annotation, event-prefix filter, tail limit); sibling tests `segment_tests.rs` and `merge_tests.rs`; register the module in `crates/core/src/lib.rs`. +- **Approach:** Read lines with `BufRead`, checking explicitly for a final-line trailing newline to detect truncation; guard any line over 8MiB; sort segments by filename, then merge them through a `BinaryHeap` keyed on `(ts_ms, pid, position)`; a missing `ts_ms` sorts as 0 for schema-0 tolerance; `writer_pid`/`segment` annotations are added without clobbering existing event fields; schema is read from a leading `trace.meta` line when present, else defaults to 0; a schema above the reader's known maximum produces a warning string rather than an error. Segment files open with `O_NOFOLLOW` (symlink refusal) mirroring `open_refstore_file`; an unreadable or symlinked segment degrades to a counted warning, never an error. +- **Execution note:** Test-first. Encode R2 and R3 as failing tests before writing the merge implementation. +- **Patterns to follow:** Sibling test files via the `#[path]` attribute pattern, consistent with how `agent-desktop-core` splits large modules from their tests elsewhere. +- **Test scenarios:** + 1. Two segments interleave strictly by `ts_ms`. + 2. A same-millisecond tie across processes orders by `(pid, position)` and is stable across runs. + 3. An in-process `ts_ms` regression keeps `seq` order (AE2). + 4. Discovery-order independence: the same result with the segment list reversed, fed via renamed copies. + 5. A truncated final line — no trailing newline, cut mid-JSON — yields `skipped_lines: 1` with remaining events intact (AE1). + 6. A corrupt middle line is skipped; subsequent lines still parse. + 7. A non-object JSON line (`[1,2]`, `"str"`) is skipped. + 8. An empty file and an empty trace directory yield an empty timeline with zero warnings; a missing trace directory yields a typed error for the command layer to map. + 9. A foreign file (`notes.txt`) and a `123-9.jsonl.tmp` file in `trace/` are ignored — the foreign file with a warning, the `.tmp` file silently. + 10. An oversized line over 8MiB is counted, not loaded. + 11. A `trace.meta` line with `schema: 1` parses into segment stats; an absent meta line reads as schema 0 with no warning (AE4). + 12. A `schema: 2` meta line produces a warning while events still return (AE3). + 13. Unknown event types and unknown fields pass through verbatim (R7). + 14. The `writer_pid` annotation does not clobber an existing `pid` field on `ref.resolve.entry`-shaped events (R4). + 15. The event-prefix filter (`action.`) combines with tail `--limit` semantics: the last N events after filtering, with `total_events` vs `returned_events` and the `truncated` flag both correct. + 16. Filename parsing: a valid `4242-1719900000000.jsonl` is accepted; `abc-1.jsonl` and `1.jsonl` are rejected as foreign. + 17. Multiple `trace.meta` lines mid-file pass through as regular events; only the first line counts toward the segment's schema (supports a shared explicit `--trace` file opened by more than one writer; see R-E). + 18. A segment-named symlink inside `trace/` is skipped with a `symlinked_segment` warning; its target is never read. + 19. A permission-denied segment file is skipped with an `unreadable_segment` warning; remaining segments still merge. +- **Verification:** `cargo test --lib -p agent-desktop-core` green; every new file under 400 LOC; `cargo tree -p agent-desktop-core` unchanged (zero platform imports). + +### U2. `trace show` command and full CLI, batch, and FFI wiring + +- **Goal:** Expose the reader as `agent-desktop trace show` everywhere other commands exist. +- **Requirements:** R1, R5, R18, R19, KTD11. +- **Dependencies:** U1. +- **Files:** NEW `crates/core/src/commands/trace.rs` (core `TraceAction::Show { limit, event }`, resolving `RefStore::for_session(context.session_id()).trace_dir()` and mapping a missing session or directory to `INVALID_ARGS` with a suggestion) plus sibling `trace_tests.rs`; NEW `src/cli_args/trace.rs` (clap `TraceArgs`/`TraceAction::Show(...)`); `src/cli/mod.rs` (the new variant and its name mapping); `src/dispatch/mod.rs` plus a new sibling extracted arm module (the KTD10 split moves the Session and Trace arms out, keeping dispatch under 400 LOC); `src/command_policy/mod.rs` (add `Trace(_)` to the permissionless `None` arm and to the ref-policy exemption arm); `src/batch/mod.rs` (a `"trace"` parse path with a `deny_unknown_fields` `BatchTraceArgs`); `crates/ffi/build.rs` (a descriptor for `trace show`, regenerated via `scripts/update-ffi-header.sh` if the surface changes); `src/cli/contract_tests.rs` plus a regenerated `src/cli/help_after.txt` golden; `src/tests/conformance.rs`; an FFI conformance test file enumerating commands. +- **Approach:** The envelope's `data` shape is `{session_id, segments: [{segment, pid, schema, event_count, skipped_lines}], total_events, returned_events, truncated, warnings: [], events: [...]}`, with empty `warnings` omitted per the serialization rules. The command works with a bare `--session <id>`: a manifest-less session still has a trace directory only if it was traced, and an absent directory returns the `INVALID_ARGS` guidance. `segments[].event_count` is an integer; per-segment event bodies are never duplicated — only the top-level `events` array (post-filter, post-limit) carries payloads. `warnings` entries are `{kind, message}` objects with the closed kind set from R3. +- **Patterns to follow:** `src/cli_args/session.rs` for the clap argument shape; the existing `session` batch-args handling for `deny_unknown_fields` parsing. +- **Test scenarios:** + 1. Happy path: a seeded session directory (real segment fixtures via a test helper) produces an envelope whose shape is asserted field by field. + 2. No active session anywhere returns `INVALID_ARGS` with a suggestion (AE8). + 3. A session that exists but was never traced (no trace directory) returns `INVALID_ARGS` suggesting `session start`, not a crash. + 4. Session resolution precedence: explicit `--session` beats the env var, which beats the pointer (reusing the `HomeGuard` test pattern). + 5. `--limit`/`--event` flow through to the reader; the default of 500 is documented in help text and asserted in tests. + 6. Policy: `trace show` passes preflight with every permission denied (a `command_policy` unit test). + 7. Batch: `{"command":"trace","args":{"action":"show"}}` runs; an unknown batch arg field is rejected via `deny_unknown_fields`. + 8. CLI contract: `trace` appears in the command list; the help golden is regenerated; argument-parse tests cover `--limit 0` and `--event action.`. + 9. FFI conformance: `trace show` is callable and returns an envelope, mirroring existing FFI command tests. + 10. A committed golden fixture — a mini-session with two segments, one containing a truncated line — produces stable merged JSON in `tests/fixtures/`; one of the two segments carries no `trace.meta` line, so the same committed fixture doubles as the schema-0/v0.4.6 compatibility proof the Definition of Done names. + 11. A tail window that cuts a `command.start`/`command.end` pair yields `truncated: true` plus an `unpaired_command` warning, with the surviving member still present in `events`. +- **Verification:** The full binary contract suite and FFI tests are green; `help_after.txt` is regenerated deliberately, with the diff reviewed rather than accidental. + +### U3. Event enrichment: command boundaries, snapshot linkage, segment meta + +- **Goal:** A trace alone reconstructs every command: boundaries, duration, outcome, and the snapshot ids it touched. +- **Requirements:** R6 (write side), R8, R9. +- **Dependencies:** None — runs in parallel with U1. U2's `trace show` output benefits once U3 lands, but does not require it first. +- **Files:** The KTD10 split happens first: NEW `crates/core/src/trace_sanitize.rs` (moves `sanitize_trace_value` and its key-token helpers out of `trace.rs`, with a `pub use` in `lib.rs` for the FFI `log_callback` consumer). Then: `crates/core/src/trace.rs` (emits `trace.meta` as the first line of every new segment, including explicit `--trace` files, so the reader treats both sink kinds identically); `crates/core/src/context.rs` (a `command_scope(name) -> CommandScope` guard — an eager start event, a `complete(&Result)` call emitting the end event with `duration_ms` via `Instant`, and a `Drop` that emits an `INTERNAL` end if not completed) plus `context_tests.rs`; `src/dispatch/mod.rs` (wraps the dispatch body in the scope, covering CLI and batch items from one place); `crates/ffi/build.rs` (generated entrypoints open and complete the scope); `crates/core/src/commands/snapshot.rs` (emits `snapshot.saved` after save, including `ref_count` and `app` when known); the other `save_new_snapshot` call sites (`snapshot_ref.rs` already emits `root.saved`; the wait paths gain `snapshot.saved` too) plus their tests. +- **Approach:** Events flow through the existing redaction pipeline unchanged — the message field passes, sensitive keys redact, already proven by the current context tests. `command.start` carries only `{command}` (KTD5). In batch, the outer `batch` command gets its own scope, and each item gets its own through the shared dispatch path, so nesting is reconstructable from `seq` order. Over FFI, boundary events cover the five generated entrypoints only; hand-written FFI surfaces are a documented follow-up (R-G). +- **Patterns to follow:** The existing `write_event`/redaction pipeline already used by `ref.resolve.*` and `actionability.*` events; `snapshot_ref.rs`'s existing `snapshot.root.saved` emission as the template for the new `snapshot.saved` call. +- **Test scenarios:** + 1. Scope happy path: a start/end pair with `duration_ms` present and sane, `ok: true`. + 2. A failing command's end event carries `ok: false` plus `code` and `message` (for example, a forced `INVALID_ARGS`). + 3. A drop without `complete()` emits an `INTERNAL` end exactly once — no double-emit when the scope completes normally. + 4. A no-sink context makes the scope a no-op: no error, no file. + 5. `trace.meta` is the first line of a fresh segment, with schema 1, `pid`, and `session_id` present; an explicit `--trace` file also opens with meta. This adds test scenario 17 to U1: multiple meta lines mid-file are passthrough events, and only the first counts as the segment's schema. + 6. A batch of 3 items produces 1 outer plus 3 inner start/end pairs, with distinct commands and correct nesting by `seq`. + 7. The snapshot command emits `snapshot.saved` carrying the id the envelope returned; the wait-with-snapshot path emits it too. + 8. FFI: a generated entrypoint produces a start/end pair in the session segment, extending the existing FFI trace verification test. + 9. Redaction: an end message containing a window title passes through as a diagnostic message under documented semantics; sensitive keys in any future fields still redact, and the sanitizer's own unit tests survive the file move unchanged. +- **Verification:** Core, binary, and FFI suites are green; `trace.rs` and `dispatch/mod.rs` are both under 400 LOC after their splits; the generated FFI file is regenerated via the script, never hand-edited. + +### U4. Artifacts mode: manifest, `session start --screenshots`, and status + +- **Goal:** One opt-in knob recorded in the manifest, surfaced everywhere session state is visible. +- **Requirements:** R10. +- **Dependencies:** None. +- **Files:** `crates/core/src/session/manifest.rs` (`#[serde(default)] artifacts: ArtifactsMode` enum with `Full` and `#[default] Events`, plus an `artifacts_full()` helper that respects `ended_at`); `crates/core/src/commands/session.rs` (`Start` gains a `screenshots: bool` mapped to `ArtifactsMode`); `crates/core/src/session/mod.rs` (`StartSessionOptions` gains the field); `src/cli_args/session.rs` (a `--screenshots` flag with help text warning about sensitivity); `src/batch/mod.rs` (`BatchSessionArgs` gains the field); `src/dispatch/mod.rs` (arm plumbing); `crates/core/src/context.rs` (resolves the artifacts mode at construction alongside trace gating, from one manifest read that returns both); `crates/core/src/commands/status.rs` (surfaces `artifacts` for the active session); tests in `session_tests.rs`, `status_tests.rs`, `batch/tests.rs`, plus CLI contract tests and a help golden update. +- **Approach:** `--screenshots` implies nothing about tracing on its own — `--no-trace --screenshots` together is `INVALID_ARGS`, since artifacts require tracing, validated at session start with a clear message. +- **Patterns to follow:** `artifacts_full()` mirrors the existing `trace_enabled()` end-of-session check; `--screenshots` follows the same boolean-flag-to-manifest-field wiring already used for `trace: on`. +- **Test scenarios:** + 1. `session start --screenshots` produces a manifest with `"artifacts":"full"`; without the flag, the field still serializes explicitly as `"events"` (always-serialize, for simpler golden fixtures). + 2. An old manifest JSON with no `artifacts` key deserializes to `Events` (backward compatible). + 3. A manifest with an unknown future key still parses — no `deny_unknown_fields` regression. + 4. `--no-trace --screenshots` together returns `INVALID_ARGS` with a suggestion. + 5. An ended session reports `artifacts_full()` as false after `session end`, mirroring `trace_enabled`. + 6. `status` shows the artifacts mode; an absent session omits the field. + 7. Batch `session start` accepts `screenshots: true`; an unknown field is still rejected. + 8. CLI: the flag parses correctly, and the help golden is updated. +- **Verification:** Core and binary suites are green; a v0.4.6-era session directory round-trips through `session list` untouched. + +### U5. Capture pipeline: screenshots and refmap copies (`trace_artifacts`) + +- **Goal:** The opt-in artifacts are actually captured, budgeted, and linked from events. +- **Requirements:** R11, R12, R13. +- **Dependencies:** U4 (mode). U1 only for shared naming conventions — a soft dependency, not a hard ordering requirement. +- **Files:** NEW `crates/core/src/trace_artifacts.rs` (hardens `screens/` and `refmaps/` by calling `ensure_trace_dir` — promoted to `pub(crate)` — rather than re-deriving the `0700`/symlink-guard logic; process-local atomic budgets and a capture sequence counter; `capture_action_screenshot(context, adapter, pid, phase) -> ArtifactOutcome` and `copy_refmap_if_full(context, store, snapshot_id)`, both writing through the existing `write_private_file` primitive — which already carries `0600`, `O_NOFOLLOW`, and atomic tmp-then-rename — with a pre-existence check supplying first-write-wins idempotency; no new file-write logic) plus `trace_artifacts_tests.rs`; `crates/core/src/ref_action.rs` (pre/post capture around execution, plus the `action.artifacts` event; stays under 400 LOC — currently 135); refmap-copy calls at every `save_new_snapshot`/`save_existing_snapshot` command seam (`crates/core/src/commands/snapshot.rs`, `snapshot_ref.rs`, and the wait snapshot-save paths — grep for the call sites); `crates/core/src/refs.rs` only if a raw-bytes read helper turns out to be needed; `crates/core/src/adapter.rs` (doc-comment correction only: `ScreenshotTarget::Window` documents largest-visible-window resolution, not "frontmost"). +- **Approach:** Capture runs only when `context.trace_enabled() && context.artifacts_full()`. Screenshots go through the existing `adapter.screenshot(Window(entry.pid))`; post-capture fires on the action's success and failure paths alike (`inspect_err`-symmetric, mirroring `check_actionability_with_trace` in the same file) — a failed action is when the screenshot matters most. Every failure path returns a reason string (`"budget"`, `"count_budget"`, `"adapter: <code>"`) that lands in `action.artifacts.skipped`; artifact writes never propagate errors into the action result. The mock adapter gains a screenshot stub returning a small fixed PNG. +- **Execution note:** Test-first for budget and skip semantics — they are the contract. +- **Patterns to follow:** `ensure_trace_dir` (promoted to `pub(crate)`) for directory hardening and `write_private_file` for every artifact write — reuse the proven primitives, do not re-derive them; the existing `serialize_with_size_check` for refmap reserialization rather than a raw-bytes copy; `check_actionability_with_trace`'s `inspect_err` symmetry for failure-path capture. +- **Test scenarios:** + 1. With artifacts full and the mock adapter, pre and post PNGs exist on disk (magic bytes `\x89PNG`), as `0600` files, named with pid, process-start timestamp, sequence, and phase; `action.artifacts` paths resolve relative to the trace directory. + 2. With artifacts at the default `events` mode, zero files and zero `action.artifacts` events are produced, and the action result is identical to today. + 3. Trace off with artifacts full still captures nothing — the trace gate wins. + 4. An adapter screenshot error still lets the action succeed, with skip reason `adapter: ...` (AE7). + 5. Byte budget exhaustion (a tiny test budget via a `cfg(test)` constructor) skips with reason `"budget"`; counters never wrap, using saturating arithmetic. + 6. Count budget exhaustion skips with reason `"count_budget"` (AE6). + 7. A symlinked `screens/` directory makes capture refuse, with a skip reason, never writing through the symlink. + 8. Refmap copy: a snapshot save with artifacts full produces `trace/refmaps/<id>.json` byte-equal to the source refmap; a second save of the same id produces a single copy (idempotent); a concurrent first-write race between two tmp files leaves exactly one winner and no error. + 9. Refmap copy budget exhaustion skips and counts; pruning the source snapshot afterward leaves the copy intact, so replay survives prune (the R13 guarantee). + 10. On an adapter with the default `not_supported` screenshot method (the Windows and Linux stubs), capture cleanly skips — proving cross-platform safety from day one. + 11. Two threads capturing through the same process counters produce distinct filenames, since the capture sequence is atomic. + 12. A failing ref action (`execute_action` returns an error) still produces the post-action screenshot and an `action.artifacts` event when artifacts are full. + 13. Against a multi-window mock, capture resolves per the platform heuristic (largest visible window for the pid) — asserted as the stated contract from R-C. +- **Verification:** Core suite is green; no adapter trait change (`adapter.rs` diff-free); capture adds zero overhead when the mode is `events` (asserted by no directory creation). + +### U6. `trace export`: single-file HTML viewer + +- **Goal:** The human-facing artifact: timeline, screenshots, and a detail pane in one XSS-safe static file. +- **Requirements:** R14, R15, R16, R17, R19 (export over FFI and batch). +- **Dependencies:** U1, U2 (the command file and its wiring already exist). +- **Files:** NEW `crates/core/src/trace_read/html.rs` (the export builder: merged timeline to an embedded JSON island with `<` escaping; screenshot files to base64 data URIs under the 100MiB embed budget, each resolved path canonicalized and contained within the session trace directory before any read — a path that escapes (traversal, absolute, or symlink) degrades to the placeholder-plus-count path, never an embedded read; the 200MiB JSON guard; deterministic output) plus `html_tests.rs`; NEW assets `crates/core/src/trace_read/viewer.html`, `viewer.css`, `viewer.js` (each under 400 lines, composed via `include_str!`); `crates/core/src/commands/trace.rs` (an `Export` action taking `--out` and `--limit`); the corresponding wiring increments in `src/cli_args/trace.rs`, the batch parser, the FFI descriptor, CLI contract tests, and the help golden. +- **Approach:** Viewer scope is fixed — resist feature creep. A chronological event list uses `command.start` rows as group headers with duration badges and ok/error coloring; a click opens a detail pane with pretty JSON rendered via `textContent`; `action.artifacts` rows show pre/post thumbnails that expand to full size on click; a text filter matches on event name; redacted fields render as `⟨redacted⟩`; a warnings banner surfaces skipped lines, schema warnings, and embed skips; zero network requests; vanilla JS only. The event list renders an explicit empty-state message both for an empty timeline and for a filter matching zero events. A skipped or missing screenshot renders a distinct non-image "screenshot unavailable" placeholder following R17's redacted-field pattern — never a broken `img`. A pruned, budget-skipped refmap renders the same placeholder-plus-count shape (R13). Command status pairs the ok/error coloring with a non-color glyph or text label, so status never relies on color alone. A command group whose start or end fell outside the tail window, or whose writer died before `command.end`, renders as an open, incomplete group labeled as such (R3/R5). An explicit `--limit 0` export is not virtualized and may render slowly for very large sessions — an accepted, documented tradeoff mirroring the JPEG deferral. +- **Patterns to follow:** The `commands/skills.rs` `include_str!` asset-embedding precedent for the three viewer asset files; the existing trace-file hygiene of refusing to write through a symlink, applied to `--out`. +- **Test scenarios:** + 1. Export writes exactly one file; the response `data` reports path, event count, screenshots embedded, and byte size. + 2. The output contains no `src="http`, no `<link href`, and no other external reference (structural assertions). + 3. Hostile strings — the AE5 payloads in a window title, an element name, and an error message — appear only `<`-escaped inside the JSON island; a raw `<script>alert` is absent outside it. + 4. The JSON island round-trips: extracting the island text and parsing it with `serde_json::from_str` reproduces the timeline, proving the escaping didn't corrupt it. + 5. Screenshots embed as `data:image/png;base64,` with a valid base64 charset; a screenshot file missing from disk produces a placeholder entry plus a count, not an error. + 6. Embed budget: an oversized screenshot set skips the later screenshots, counts them in `screenshots_skipped`, and the export still succeeds. + 7. The 200MiB JSON guard returns `INVALID_ARGS` with a `--limit` suggestion. + 8. Determinism: two exports of the same session are byte-identical (AE9). + 9. The default output path is `trace-<session>.html`; an explicit `--out` is honored; the write is plain create-or-truncate but refuses to follow a symlink, matching existing trace-file hygiene. + 10. A redacted field renders its flag correctly: the island JSON keeps `{"redacted": true}` intact, and the viewer maps it — the test asserts the data is intact. + 11. Export with `--limit` embeds only the tail, with a truncated marker in the island's metadata. + 12. A screenshot path containing traversal (`../`) or an absolute path, or resolving through a symlink, yields the placeholder plus a count — the target file's contents never appear in the HTML (companion to scenario 5). + 13. An export whose filter or timeline yields zero events renders the explicit empty-state message. + 14. An unpaired `command.start` renders as an open-incomplete group, not a blank or broken one. +- **Verification:** Core suite is green; asset files are each under 400 lines; the e2e-produced file is opened manually once during implementation as a human smoke test, with automated structural assertions thereafter. + +### U7. E2E scenario and docs + +- **Goal:** End-to-end proof by independent observation, with every doc surface telling the truth. +- **Requirements:** Closes the loop on R1–R17; updates the documented command count from 55 to 56 — `trace` counts as one command with two actions, consistent with how `session` (four actions: start/end/list/gc) is already counted as one command. +- **Dependencies:** U1–U6. +- **Files:** `tests/e2e/run.sh` gains a scenario: `session start --screenshots`, snapshot the fixture app, click and type via refs, then `trace show` asserts the command pairs, `action.artifacts`, `snapshot.saved`, and that the artifact files exist with PNG magic bytes; then `trace export` gets structural HTML assertions, including the hostile-title fixture case if the fixture app can set a window title containing markup (if not, the unit-level AE5 coverage suffices, and the gap is noted); `skills/agent-desktop/SKILL.md` (the command count in two places, plus Quick Reference rows for `trace show`, `trace export`, and `session start --screenshots`); `skills/agent-desktop/references/commands-system.md` (the full trace command reference: flags, envelope fields, tolerance semantics, and an artifact-sensitivity warning); `CONCEPTS.md` (the Coordination section gains "Trace Timeline", "Trace Schema", and "Replay Artifacts" entries; "Trace Segment" is updated for `trace.meta`); `README.md` (a trace viewer section with the one-line flow: `session start --screenshots`, do the work, `trace export`). Every doc stays consistent with the `artifacts: full|events` naming from the Planning Contract. +- **Test scenarios:** The e2e assertions are the scenarios. Per the repo's observation rule, no step trusts a command's own `ok: true` — the PNGs are stat'd, the HTML is grepped, and the JSON envelope is parsed independently. +- **Verification:** `bash tests/e2e/run.sh` is green in both the headless and `--headed` legs (release build plus AX permission); `agent-desktop skills get` output reflects the new docs (skills are `include_str!`'d, so a rebuild is required); the binary size check still passes under 15MB. + +--- + +## Verification Contract + +| Gate | Command | Applies | +|---|---|---| +| Format | `cargo fmt --all -- --check` | all units | +| Lint | `cargo clippy --all-targets -- -D warnings` | all units | +| Core tests | `cargo test --lib -p agent-desktop-core` | U1, U2, U3, U4, U5, U6 | +| Full unit suite | `cargo test --lib --workspace` | all units | +| Binary contract | `cargo test -p agent-desktop` | U2, U3, U4, U6 | +| FFI | `cargo test -p agent-desktop-ffi --tests` | U2, U3, U6 | +| Core isolation | `cargo tree -p agent-desktop-core` contains no platform crates | U1, U2, U3, U4, U5, U6 | +| Size | release binary under 15MB (CI gate) | U6, U7 (embedded assets and skills) | +| E2E | `bash tests/e2e/run.sh` (release build, AX permission) | U7 (and any unit changing action behavior) | + +--- + +## Definition of Done + +- All 7 units land with their enumerated test scenarios implemented in full, not a subset; every suite in the Verification Contract is green. +- Every new or changed file is under 400 LOC; no inline comments; zero `unwrap()` outside tests; only `lib.rs` re-exports. +- `trace show` and `trace export` are proven over CLI, batch, and FFI; the permissionless preflight is proven by test. +- A v0.4.6-era session trace reads cleanly, proven by a committed schema-0 fixture. +- Docs (skills, `CONCEPTS.md`, `README.md`) are updated and the command count is consistent; the help golden is regenerated deliberately, not accidentally. +- No abandoned experimental code remains in the final diff. +- The e2e scenario passes in both headless and `--headed` mode. + diff --git a/docs/plans/2026-07-03-001-feat-foundation-playwright-grade-contract-plan.md b/docs/plans/2026-07-03-001-feat-foundation-playwright-grade-contract-plan.md new file mode 100644 index 0000000..a14b2d7 --- /dev/null +++ b/docs/plans/2026-07-03-001-feat-foundation-playwright-grade-contract-plan.md @@ -0,0 +1,512 @@ +--- +title: Playwright-Grade Foundation Contract - Plan +type: feat +date: 2026-07-03 +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-plan-bootstrap +execution: code +--- + +# Playwright-Grade Foundation Contract - Plan + +## Goal Capsule + +- **Objective:** Land the Tier-1 FOUNDATION-NOW contract from the Playwright-grade gap analysis in `crates/core` — plus the three live correctness defects — so the Windows (UIA) and Linux (AT-SPI2) adapters implement a settled reliability contract instead of redesigning it. +- **Authority hierarchy:** repo `CLAUDE.md` invariants (400 LOC/file, no inline comments, zero `unwrap()` outside tests, core never imports platform crates, conventional commits, additive `not_supported()` trait defaults) override this plan; this plan overrides implementer improvisation; `docs/solutions/` learnings cited per unit are binding constraints. +- **Execution profile:** one feature branch (`feat/foundation-playwright-grade-contract`), one conventional commit per implementation unit, dependency order per the Unit Index. All gates in the Verification Contract green before any unit is considered done. +- **Stop conditions:** stop and surface — do not guess — if (a) a unit requires core to import a platform crate, (b) a wire-contract change goes beyond additive optional fields anywhere other than U8/U14's documented changes, (c) the U0 restructure cannot preserve `&dyn PlatformAdapter` call sites unchanged, or (d) any file cannot stay under 400 LOC without violating the one-command-per-file rule. +- **Tail ownership:** the implementer owns fixture updates, FFI header regeneration via `scripts/update-ffi-header.sh`, and doc touch-ups (`CLAUDE.md` error-code list, `docs/phases.md` pulled-forward notes) inside the unit that causes them. + +--- + +## Product Contract + +### Summary + +agent-desktop's core contract has three foundational gaps that cap it below best-in-class desktop automation: element identity is snapshot-bound rather than re-resolvable, actionability is checked once instead of awaited, and the role/state/naming vocabulary is too thin for a second adapter to implement compatibly. This plan fixes the three live correctness defects and lands the eighteen Tier-1 foundation items (two of which — display enumeration and the automation-permission wiring — are themselves the fixes behind defect requirements R2/R3) as core contract — new `PlatformAdapter` capability methods (all defaulting to `not_supported()`), new core types, and an enforced vocabulary — with macOS implementing each where it has immediate value. Windows and Linux then inherit reliability by implementing settled shapes. + +### Problem Frame + +A 15-agent gap analysis (verified against the repo at `52705af`) found: `is --property visible` is unconditionally true because it checks a `"hidden"` state token no code produces, and against snapshot-time state rather than live evidence; `PermissionReport.automation` is hardcoded `NotRequired` while `close_app`'s osascript fallback depends on the Automation TCC gate; `ScreenshotTarget::Screen(usize)` is dead code with no CLI reachability and an index-ignoring macOS impl. Beyond defects: refs go stale with no re-resolving locator alternative, `check_live` runs exactly once per action (`crates/core/src/ref_action.rs`), no occlusion hit-test exists anywhere, the states vocabulary has 5 producer tokens against the 17 both UIA and AT-SPI2 support natively, accessible-name computation is a private macOS function core never sees, three window operations re-resolve by `(pid, title)` even when an unambiguous id was supplied, and the trait has no process-liveness, display-enumeration, launch-environment, session-affinity, or event-baseline concepts. Building Phase 2/3 adapters against this trait would force each to invent incompatible answers — the exact failure the dependency-inversion architecture exists to prevent. + +### Requirements + +Live correctness defects: + +- R1. `is --property visible` reports real visibility: live bounds evidence plus canonical `hidden`/`offscreen` state tokens; an off-screen or hidden element reports `false`. +- R2. Display capture is a complete, honest contract: displays are enumerable, `--screen N` targets a real display, an out-of-range index returns `INVALID_ARGS` naming the available displays, and captures report their `scale_factor` so point↔pixel math (Retina 2×, mixed-scale multi-monitor) is computable from the result rather than guessed. +- R3. `permissions`/`status` report the Automation permission truthfully, without triggering a TCC prompt; osascript-backed paths reclassify authorization failures to `PERM_DENIED` with a recovery suggestion. + +Identity spine: + +- R4. Elements carry the native automation id (`AXIdentifier` / UIA `AutomationId`) when present, and ref re-identification prioritizes it above name/value/description text. +- R5. Window-scoped operations treat `WindowInfo.id` as the primary key; title is fallback evidence only. Two same-titled windows never cause an id-addressed operation to act on the wrong one. +- R6. A serializable `LocatorQuery` (role, name, description, native id, exactness, state predicates, containment filters, ordinal selection) resolves against the live tree through one adapter method, with the existing 0/1/N strict-resolution classification applied to its results. + +Actionability: + +- R7. Every ref-addressed action auto-waits for actionability by default under a single bounded budget (CLI default 5000 ms, `--timeout-ms 0` restores single-shot), retrying transient states and propagating permanent errors immediately; budget expiry returns `TIMEOUT` with the last actionability report and a `kind` discriminant. +- R8. Actions on occluded elements are detected before dispatch where the platform can hit-test; unavailable evidence reports `unknown`, never a false failure. +- R9. Element-targeted actions scroll the target into view (best-effort) before the visibility check, uniformly in core rather than per-platform-chain. + +Cross-platform contract: + +- R10. Every new adapter capability defaults to `Err(not_supported())`; macOS-only surface concepts (`Sheet`, `Popover`) are ratified explicitly via `supported_surfaces()` introspection rather than silently assumed portable. +- R11. Role and state tokens come from canonical core vocabulary modules; conformance tests fail any adapter emitting a token outside the vocabulary. The `is` property set and its evidence sourcing move onto that vocabulary. +- R12. Accessible name/description computation is a core-owned algorithm over adapter-supplied `NameEvidence`; adapters never invent their own precedence. +- R19. Action steps carry a typed delivery tier (`SemanticApi` vs `PhysicalSynthetic`) and a verified flag, so callers can programmatically ask "was this delivered semantically and independently confirmed?" + +Lifecycle and environment: + +- R13. Process liveness is classifiable (`Running`/`Exited`/`Crashed`/`Unresponsive`); persistent AX unresponsiveness surfaces as a new `APP_UNRESPONSIVE` error code, and ref/app resolution errors carry process state in `details`. +- R14. `launch_app` accepts arguments, environment variables, working directory, and an attach-vs-fail-if-running policy. +- R15. A session-affinity lifecycle hook (`open_session` returning an `AdapterSession`) exists on the trait — defaulted, uncalled by the CLI today — so Windows COM-MTA and Linux D-Bus connection state have a landing zone before Phase 2 starts. + +Signals and input vocabulary: + +- R16. UI-event detection works by baseline diff: snapshot observable signals, act, diff — surfaced as `wait --event <kind>` (window opened/closed, app launched/terminated, focus changed, surface appeared) without requiring known titles. +- R17. Clipboard content is typed (`Text`/`Image`/`FileUrls`) at the trait and CLI, not string-only. +- R18. Mouse events accept modifier chords, and a wheel-delta primitive exists as a first-class command. + +### Acceptance Examples + +- AE1. **Covers R1.** Given a window with a zero-sized or `AXHidden` element that holds a ref, when `is --ref @e5 --property visible` runs, then `result` is `false` (today: unconditionally `true`). +- AE2. **Covers R7.** Given a button that becomes enabled 800 ms after a dialog opens, when `click --ref @e3` runs with defaults, then the click succeeds without an explicit `wait` call; with `--timeout-ms 0` it fails immediately with the actionability report. +- AE3. **Covers R7.** Given a permanently disabled button, when `click --ref @e3 --timeout-ms 2000` runs, then the command fails at ~2 s with `TIMEOUT`, `details.kind = "actionability_timeout"`, and the last per-check report. +- AE4. **Covers R5.** Given two windows titled "Untitled" in one app, when a window operation targets the second window's id, then the operation acts on that window (today: first title match wins). +- AE5. **Covers R3.** Given Automation permission denied for System Events, when `close_app` falls back to osascript, then the error is `PERM_DENIED` with a System Settings suggestion, not a generic failure; and `permissions` reports `automation: denied` without prompting. +- AE6. **Covers R16.** Given a click that opens a dialog with unknown title, when `wait --event surface-appeared --app TextEdit` runs after the click, then the event reports the new surface without the caller naming it. +- AE7. **Covers R7.** Given a permanently disabled button, when `click --ref @e3` runs with no timeout flag at all, then the command fails at ~5 s (the untouched default) with `TIMEOUT`, `details.kind = "actionability_timeout"`, and the last per-check report — the exact experience a caller gets by doing nothing. + +### Scope Boundaries + +**In scope:** the three live defects; the eighteen Tier-1 items (two of them — `list_displays` and the automation-permission wiring — double as the defect fixes behind R2/R3, which is why the non-defect requirement groups sum to sixteen); the `adapter.rs` capability-trait restructure they force; macOS implementations where each unit names them; FFI parity where existing FFI surface is affected; fixture/doc updates caused by these changes. + +**Deferred to Follow-Up Work** (Tier-2/3 of the gap analysis — not this plan): visual diff/baselines and any image-codec dependency decision; the shared settled-debounce primitive and `stability_check` Fail-arm redesign; tri-state assertion-path resolution (`ProvenAbsent`); unified negatable `is`/`wait` predicate vocabulary; `toMatchAriaSnapshot`-style tree assertions; `level`/`pos_in_set`/`set_size`, `ValueRange`, relations, text-range actions; canonical key-name vocabulary module; `DragFiles` payload vocabulary; TCC/tray/launcher surfaces; tree-exposure-quality signal and Electron AX forcing; default per-invocation session isolation; `LocatorQuery.relative` steps and action-by-locator CLI; persisted cross-invocation `SignalBaseline` state (v1 is in-invocation only) and FFI exposure of the `wait --event` mode; `--env-file`/stdin secret passing for `launch`; FFI exposure of locator queries and new Family-B commands; `subscribe_events` push delivery (daemon-gated); per-rung trace streaming; window-id threading onto the five `WindowOp` commands. + +**Outside this product's identity** (stated non-goals, reaffirmed): no embedded LLM, no GUI/TUI, no browser automation, no macro record/replay (the trace-replay compiler stays rejected — it would also reopen the trace-redaction posture), no daemon in this plan. + +--- + +## Planning Contract + +### Key Technical Decisions + +- KTD1. **Capability supertraits, forced by the 400-LOC cap.** `crates/core/src/adapter.rs` is at 397/400 LOC with 38 trait methods; this plan adds ~12 methods and ~9 types. Restructure into `crates/core/src/adapter/` — `mod.rs` (`pub trait PlatformAdapter: ObservationOps + ActionOps + InputOps + SystemOps` **plus the blanket `impl<T: ObservationOps + ActionOps + InputOps + SystemOps> PlatformAdapter for T {}` that actually confers the composed trait** — supertraits alone are bounds, not implementations) plus one file per capability trait, mirroring the platform crates' `tree/actions/input/system` foldering. Supertrait methods remain callable on `&dyn PlatformAdapter`, so zero call-site churn; the compiler proves behavior preservation. Supporting types move to their own files per the one-type-per-file convention. +- KTD2. **The vocabulary module is the `visible` defect fix, not a point patch.** The root cause is unenforced vocabulary (a consumed token with zero producers, undetected indefinitely). `crates/core/src/state.rs` (17 canonical tokens + `STATE_VOCABULARY`) and `role.rs` (canonical `Role` enum, `from_str` → `Unknown`, never an error) plus conformance helpers close the class. `is --property visible` moves to live evidence: live bounds non-empty AND not `hidden` AND not `offscreen` (`is` currently reads snapshot-time `RefEntry.states` via `state_from_ref_entry` — a second latent defect this fixes). Wire format stays `Vec<String>`; the enum is the vocabulary authority, not a serialization change. +- KTD3. **Auto-wait defaults ON at the command layer, OFF at the type layer.** `ActionRequest.timeout_ms: Option<u64>` with serde default `None` preserves every existing constructor and wire payload (single-shot). The CLI ref-action surface defaults `--timeout-ms 5000`; FFI `ad_execute_by_ref` adopts the same 5000 ms default per the FFI/CLI-parity learning, with additive `ad_execute_by_ref_timeout(..., i64)` (−1 default, 0 single-shot) for callers needing opt-out. **Why default-on, and why 5000:** the gap analysis's P0 finding is precisely that retry-until-actionable is opt-in today — an opt-in default would preserve the gap; Playwright's equivalent default is 30 s, which is wrong for an LLM-agent consumer whose per-step latency budget is seconds, so 5000 ms covers the common enable/animation/dialog-settle transitions while capping a hard failure's cost at one step. The consumer-owned observe→decide→act loop is unaffected: the loop decides *what* to do next, the budget only gates *whether this one action's target is ready*, and permanent errors (`PERM_DENIED`, `APP_NOT_FOUND`, `ACTION_NOT_SUPPORTED`, `INVALID_ARGS`, `POLICY_DENIED`) still fail instantly, so callers with their own retry wrappers pay the budget only on genuinely transient/ambiguous states — and can set `--timeout-ms 0` to restore fail-fast wholesale. The poll loop re-runs resolve→`check_live` each tick (100 ms) until Pass or deadline; `STALE_REF`/`AMBIGUOUS_TARGET`/actionability-Fail retry within budget; `PERM_DENIED`/`APP_NOT_FOUND`/`ACTION_NOT_SUPPORTED`/`INVALID_ARGS`/`POLICY_DENIED` propagate immediately (mirrors the wait command's established retryable/permanent split). Budget expiry: `TIMEOUT` + `details.kind = "actionability_timeout"` + last `ActionabilityReport`, joining the existing `"wait_timeout"`/`"chain_deadline"` kind convention. The loop wraps resolve+preflight only — it never chooses `InteractionPolicy` and never skips post-condition verification (policy-preservation learning). The macOS chain's internal `AGENT_DESKTOP_CHAIN_TIMEOUT_MS` (post-dispatch) is untouched; `WAIT_RESOLVE_ATTEMPT` (750 ms) becomes the loop's per-tick resolve budget rather than a separate user-facing concept. +- KTD4. **Hit-testing ships as `hit_test(target, point) → HitTestResult`, not raw `element_at_point`.** `HitTestResult { ReachesTarget, InterceptedBy { role, name, bounds }, Unknown }` keeps ancestor-walk identity comparison (macOS `CFEqual` chain walk from `AXUIElementCopyElementAtPosition`) adapter-private, gives core a clean tri-state, and lets AT-SPI2 return `not_supported` honestly. The occluder's `name` rides under the `name` key so trace redaction covers it automatically. Consumed by a new `receives_events` actionability check (Pass/Fail/Unknown) and by ref-targeted pointer actions (`Hover`, `Drag`) which today skip `check_live` entirely; raw coordinate `mouse-*` commands stay raw by design. +- KTD5. **Scroll-into-view is core policy, adapter mechanics.** `Action::requires_scroll_into_view()` (element-targeted variants: `Click*`, `SetValue`, `TypeText`, `Toggle`, `Check`/`Uncheck`, `Select`, `Expand`/`Collapse`, `Clear`, `Hover`, `SetFocus`) + `ActionOps::scroll_into_view(&NativeHandle)` defaulting `not_supported`. Called best-effort before the actionability visibility check; failure or `not_supported` degrades to today's behavior, never fails the action. macOS promotes its existing `AXScrollToVisible` usage (7 occurrences, 5 files) into the trait impl. +- KTD6. **Accessible naming: core algorithm, adapter evidence, principled precedence.** `crates/core/src/accname.rs` computes name/description from `NameEvidence` with documented precedence (explicit label → labelled-by text → native title → static-role value → aggregated child label → placeholder → description last). macOS's private `resolve_element_name` becomes an evidence supplier. This may change some computed names: golden fixtures update consciously via characterization tests written before the migration (execution note on U11). Accepted consequence, stated plainly: a refmap written by a pre-upgrade binary stores names computed by the old algorithm, so acting on those refs after a mid-session upgrade may surface `STALE_REF` until the caller re-snapshots — bounded (one re-snapshot heals it) and accepted, not eliminated. +- KTD7. **`SnapshotSurface` ratified, not generalized.** `Sheet`/`Popover` stay as explicitly macOS-native variants; `SystemOps::supported_surfaces()` (default: `Window, Focused, Menu, Menubar, Alert`; macOS overrides adding `Sheet, Popover`) makes support introspectable via `status`, and requesting an unsupported surface returns `PLATFORM_NOT_SUPPORTED` naming the supported set. Honest vocabulary beats lossy force-fit; Windows/Linux declare what they mean, not what Cocoa meant. +- KTD8. **`ProcessState` is honest about platform limits.** `Exited { code: Option<i32> }` because macOS cannot read exit codes of non-child processes (`open -g -a` detaches); `Crashed` stays in the contract for adapters with real evidence (Windows `GetExitCodeProcess`), macOS emits `Running`/`Exited{None}`/`Unresponsive` initially. `Unresponsive` classification comes from a bounded AX probe (`kAXErrorCannotComplete` persisting beyond one retry). New `ErrorCode::AppUnresponsive` (16th code) → `ENVELOPE_VERSION` bumps (the envelope learning: bump for error-code changes callers branch on), asserted through the constant, and the `CLAUDE.md` error-code list updates in the same unit. Naming reconciliation: this supersedes `docs/phases.md`'s planned Phase-2 `AxMessagingTimeout` at the process-classification level (transport-level timeout classification remains a Phase-2 concern). +- KTD9. **Automation permission via `AEDeterminePermissionToAutomateTarget` with `askUserIfNeeded=false`** — the only probe that answers without triggering a TCC prompt. Mapping: permitted → Granted, not-permitted (−1743) → Denied, System Events not running → Unknown. The osascript fallback path in `close_app` reclassifies −1743 stderr to `PERM_DENIED` following the proven `map_screencapture_error` convention. No new error code (reuse-existing-codes learning); `docs/phases.md`'s planned `AutomationPermissionDenied` is thereby superseded. +- KTD10. **`native_id` (the gap report's name) supersedes `docs/phases.md`'s planned `identifier` field.** macOS reads `kAXIdentifierAttribute` in the existing batch attribute read; auto-generated AppKit identifiers (prefix `_NS`) are treated as absent. Identity priority: `native_id` equality is the strongest match signal (above name/value/description); two present-but-different `native_id`s are a hard non-match. Real-world macOS coverage is unverified (gap report Part VIII) — the field is `Option<String>` and degrades gracefully; its full payoff arrives with UIA `AutomationId`. +- KTD11. **`LocatorQuery` v1 ships without `relative` steps and without action-by-locator.** Fields: role, name, description, native_id, exact, state predicates (vocabulary tokens + bool), has/has_not (boxed subqueries), has_text, nth/first/last. `ObservationOps::resolve_query` returns live handles; core owns `classify_query_result` (0 → not-found, 1 → accept, 2+ → ambiguous-with-candidates), reusing the existing strict-resolution outcome contract. Consumer v1 is the `find` command (whose `FindQuery` becomes a `LocatorQuery` subset); action-by-locator and FFI exposure are deferred with the follow-up list. +- KTD12. **`SignalBaseline` generalizes the proven `NotificationFingerprint` pattern.** Plain-data baseline + pure `diff_signals` in core (testable with adapter doubles, no daemon); `SystemOps::snapshot_signals(&SignalFilter)` supplies windows/apps/focus cheaply, plus alert/sheet surface presence when the filter names an app (bounded cost). Consumer v1 is `wait --event <kind>` (baseline at wait start, poll-diff at the existing wait cadence) in a new `wait_event.rs`; persisted cross-invocation baselines are deferred. Event payloads carry titles under `title` keys — redaction-safe by construction. +- KTD13. **Clipboard trait migrates to typed content; string methods are removed, not wrapped.** `ClipboardContent { Text(String), Image(ImageBuffer), FileUrls(Vec<String>) }`; `get_clipboard_content`/`set_clipboard_content` replace `get_clipboard`/`set_clipboard` (repo-internal trait, all impls updated in one unit). CLI `clipboard-get` gains `--format auto|text|image|file-urls`; image content writes to `--out <path>` (JSON carries path + dimensions, keeping the envelope small). macOS implements all three via `NSPasteboard`, reusing the pasteboard-restore machinery's existing type round-tripping. +- KTD14. **`launch_app` signature changes directly to `(&self, id, &LaunchOptions)`.** The trait is repo-internal and pre-1.0; a `launch_app_with_options` shim would be permanent noise. `LaunchOptions { args, env, cwd, timeout_ms, attach_if_running }`; `attach_if_running: true` preserves today's `open -g -a` semantics; `false` fails with a structured error if the app is already running. CLI gains `--arg` (repeatable), `--env KEY=VAL` (repeatable), `--cwd`, `--no-attach`. FFI `ad_launch_app` keeps its current C signature (constructs default `LaunchOptions` internally — zero ABI change). **Env values are secrets by assumption:** no error message, trace event, or `details` object may ever carry a raw env value — a malformed `--env` entry reports its argument position and at most the key name, never the value (per `docs/solutions/conventions/keep-raw-arguments-out-of-trace-reachable-error-messages.md`); env passing via argv is an inherent CLI exposure (`ps`/shell history), documented on the flag, with an `--env-file`/stdin alternative deferred to follow-up. +- KTD15. **`open_session` lands as a contract-only hook.** `SystemOps::open_session(&SessionAffinity) → Box<dyn AdapterSession>` (Send + Sync, `close(self)`), default `not_supported`, called by nothing in this plan. Its doc comment names the intended owners: Windows COM-MTA apartment thread, Linux D-Bus connection. This resolves the statelessness tension architecturally: the hook exists from day one, behavior changes only when a persistent host (FFI/daemon) opts in. +- KTD16. **`ActionStep` gains `mechanism: Option<StepMechanism>` + `verified: Option<bool>`** (`SemanticApi | PhysicalSynthetic`), serde-skipped when absent — additive for every existing trace/JSON consumer. macOS chain rungs populate mechanism from what each rung already knows (AX action vs CGEvent) and `verified` from the chain's existing effect-verification outcomes. Both fields are shapes, not content — redaction-safe by construction. +- KTD17. **Open questions resolved by fiat (documented, revisitable):** AT-SPI2 hit-testing may ship `not_supported` (the contract permits it); Wayland wheel-as-buttons is an adapter translation problem, so the `Wheel` contract uses line-deltas with an explicit X11/Wayland translation note; sandbox/hardened-runtime AX denial and secure-field synthetic-input behavior stay open questions carried to Tier-2 (they inform `TreeQuality`, which is deferred); the image-codec dependency decision is moot in this plan (visual diff deferred). + +### High-Level Technical Design + +Contract layering after this plan — core owns vocabulary, types, and policy; adapters own native evidence; consumers are unchanged in shape: + +```mermaid +flowchart TB + subgraph CORE["crates/core"] + VOCAB["role.rs / state.rs / capability.rs\ncanonical vocabulary + conformance"] + TYPES["LocatorQuery / HitTestResult / ProcessState\nDisplayInfo / LaunchOptions / ClipboardContent\nSignalBaseline / SessionAffinity / StepMechanism"] + TRAIT["adapter/mod.rs\nPlatformAdapter:\nObservationOps + ActionOps + InputOps + SystemOps"] + POLICY["ref_action.rs auto-wait poll loop\nactionability checks (+receives_events)\naccname.rs / classify_query_result / diff_signals"] + end + MAC["crates/macos\nAX evidence: AXIdentifier, hit-test,\nAXScrollToVisible, NSPasteboard, displays,\nprocess probe, signal snapshot"] + WIN["crates/windows (Phase 2)\nimplements settled shapes"] + LIN["crates/linux (Phase 3)\nimplements settled shapes"] + CLI["src/ CLI + batch"] + FFI["crates/ffi C ABI"] + VOCAB --> POLICY + TYPES --> TRAIT + TRAIT --> MAC + TRAIT -.-> WIN + TRAIT -.-> LIN + POLICY --> CLI + POLICY --> FFI +``` + +Auto-wait pre-action gate (U8) — the loop wraps resolve+preflight only; dispatch and post-condition verification are untouched: + +```mermaid +sequenceDiagram + participant C as command (click --ref @e3) + participant L as core poll loop + participant A as adapter + C->>L: ActionRequest{action, policy, timeout_ms} + loop per iteration: 100ms sleep; worst case ~850ms when resolve is slow + L->>A: resolve_element_strict_with_timeout(entry, tick) + alt permanent error (PERM_DENIED, APP_NOT_FOUND, ...) + A-->>C: propagate immediately + else STALE_REF / AMBIGUOUS_TARGET + A-->>L: retry next tick + else resolved + L->>A: scroll_into_view (best-effort, if action requires) + L->>A: check_live (visibility, stability, enabled, receives_events, ...) + alt all Pass + L->>A: execute_action (policy + post-verify unchanged) + A-->>C: ActionResult + else Fail + A-->>L: retry next tick + end + end + end + L-->>C: TIMEOUT + kind:"actionability_timeout" + last report +``` + +`ProcessState` classification (U14): + +```mermaid +stateDiagram-v2 + [*] --> Running: pid alive, AX responsive + Running --> Unresponsive: AX probe CannotComplete persists + Unresponsive --> Running: probe recovers + Running --> Exited: pid gone (code None on macOS) + Running --> Crashed: adapter has crash evidence (Win/Linux) +``` + +### Assumptions + +- A1. Scope equals the user-supplied Tier-1 list plus the three defects; Tier-2/3 items referenced by Tier-1 designs (settled-debounce, tri-state resolution) are deferred even where a unit touches adjacent code. +- A2. CLI surface additions are in scope exactly where they make a Tier-1 contract observable and testable (`--screen`, `--timeout-ms`, `find` query fields, `wait --event`, clipboard formats, launch flags, `mouse-wheel`, `list-displays`); broader ergonomics are not. +- A3. Flipping auto-wait ON by default is the intended behavior change (it is the P0 finding), acceptable pre-1.0 as a `feat:`; failure paths get slower by design, success paths are unchanged. +- A4. FFI parity obligations are scoped by the alignment learning: behavior parity for the action path (U8), no new Family-B commands, additive Family-A functions only where a unit names them. +- A5. The gap analysis (session artifact, verified against `52705af`) is the requirements source; where scouts found its citations drifted (`FOCUS_CONFIRMATIONS`, `activate.rs`, `resolve_classify.rs::proved_absent`), this plan's citations are the corrected ones. + +--- + +## Implementation Units + +### Unit Index + +| U-ID | Title | Key files | Depends on | +|---|---|---|---| +| U0 | Capability-supertrait restructure | `crates/core/src/adapter/` (new), `crates/macos/src/adapter.rs` | — | +| U1 | Role/state vocabulary + live `is visible` fix | `crates/core/src/{role,state}.rs` (new), `commands/is_check.rs` | U0 | +| U2 | macOS state-producer expansion | `crates/macos/src/tree/{builder,state_reader}.rs`, `.../post_state.rs` | U1 | +| U3 | Displays: `list_displays` + `--screen` + scale factor | `crates/core/src/display_info.rs` (new), `commands/list_displays.rs` (new), `crates/macos/src/system/{screenshot,display}.rs` | U0 | +| U4 | Automation permission truthfulness | `crates/macos/src/system/permissions.rs`, `.../app_ops.rs` | — | +| U5 | `native_id` end-to-end | `crates/core/src/{refs,node,ref_identity}.rs`, `crates/macos/src/tree/attributes.rs`, `crates/ffi/src/types/ref_entry.rs` | U0 | +| U6 | Window identity as primary key | `crates/core/src/adapter/system.rs`, `crates/macos/src/{adapter.rs,system/window_ops.rs,system/app_ops.rs}` | U0 | +| U7 | `LocatorQuery` + `resolve_query` + `find` | `crates/core/src/locator.rs` (new), `commands/{query,find}.rs`, `crates/macos/src/tree/query.rs` (new) | U0, U1, U5 | +| U8 | Auto-wait pre-action gate | `crates/core/src/{action_request,ref_action}.rs`, `src/cli_args/`, `crates/ffi/src/actions/` | U0 | +| U9 | `hit_test` + `receives_events` | `crates/core/src/hit_test.rs` (new), `adapter/observation.rs`, `actionability/*`, `crates/macos/src/tree/hit_test.rs` (new) | U0, U8 | +| U10 | `scroll_into_view` contract | `crates/core/src/{action,adapter/actions.rs}`, `crates/macos/src/actions/` | U0, U8 | +| U11 | `accname.rs` core naming | `crates/core/src/accname.rs` (new), `crates/macos/src/tree/element.rs` | U0 | +| U12 | `SnapshotSurface` ratification | `crates/core/src/adapter/system.rs`, `commands/status.rs` | U0 | +| U13 | `ActionStep.mechanism`/`verified` | `crates/core/src/action_step.rs`, `crates/macos/src/actions/chain*.rs`, `crates/ffi/src/types/action_step.rs` | — | +| U14 | `ProcessState` + `APP_UNRESPONSIVE` | `crates/core/src/{process_state.rs (new),error.rs}`, `crates/macos/src/system/process.rs` (new) | U0 | +| U15 | `LaunchOptions` | `crates/core/src/launch_options.rs` (new), `crates/macos/src/system/app_ops.rs`, `src/cli_args/` | U0 | +| U16 | `open_session` affinity hook | `crates/core/src/adapter/system.rs`, `session_affinity.rs` (new) | U0 | +| U17 | `SignalBaseline` + `wait --event` | `crates/core/src/{signals.rs (new), commands/wait_event.rs (new)}`, `crates/macos/src/system/signals.rs` (new) | U0 | +| U18 | Typed clipboard content | `crates/core/src/clipboard_content.rs` (new), `crates/macos/src/input/clipboard.rs`, `crates/ffi/src/input/clipboard.rs` | U0 | +| U19 | Mouse modifiers + wheel | `crates/core/src/action.rs`, `crates/macos/src/input/mouse.rs`, `crates/core/src/commands/mouse_wheel.rs` (new) | U0 | + +**Phase A — headroom + live defects (U0–U4).** Everything else builds on U0's file-budget headroom and U1's vocabulary — except U4 and U13, which are dependency-free and can land any time. + +### U0. Capability-supertrait restructure of the adapter contract + +- **Goal:** Split the 397-LOC `crates/core/src/adapter.rs` into `crates/core/src/adapter/` with `PlatformAdapter: ObservationOps + ActionOps + InputOps + SystemOps`, creating file-budget headroom for the ~12 new methods this plan adds. Zero behavior change. +- **Requirements:** enables R4–R18 (every unit that adds adapter surface); enforces the repo 400-LOC invariant. R19 (U13) and R3 (U4) are independent of this restructure. +- **Dependencies:** none. +- **Files:** `crates/core/src/adapter/mod.rs` (new — composed trait + re-exports so `crate::adapter::PlatformAdapter` paths survive), `adapter/observation.rs`, `adapter/actions.rs`, `adapter/input.rs`, `adapter/system.rs` (new — trait per capability, existing 38 methods distributed by domain), supporting types move to `crates/core/src/{screenshot_target,image_buffer,window_filter,live_element}.rs` (new) as needed to keep each file <400; `crates/macos/src/adapter.rs` (impl blocks split per trait, delegating into the existing `tree/actions/input/system` modules); `crates/windows/src/adapter.rs`, `crates/linux/src/adapter.rs` (empty impls become four empty impls each); `src/tests/conformance.rs` and every test double (~55 across 33 files) — each single `impl PlatformAdapter for X {}` block becomes up to four capability-trait impls (only the traits whose methods the double overrides; empty ones where needed), and **no `impl PlatformAdapter` block remains anywhere** — real adapters and doubles alike get the composed trait from the blanket impl, so their existing method overrides move onto the capability traits without conflict. +- **Approach:** method distribution mirrors the platform-crate foldering: tree/element reads + resolution + queries → `ObservationOps`; `execute_action`, live reads, bounds, release → `ActionOps`; mouse/key/drag/clipboard → `InputOps`; windows/apps/permissions/screenshot/notifications/surfaces/wait → `SystemOps`. All defaults keep their exact current bodies. **The composing piece is a blanket impl** — supertraits are bounds, not implementations, so `adapter/mod.rs` must define `impl<T: ObservationOps + ActionOps + InputOps + SystemOps> PlatformAdapter for T {}` (legal: all five traits are crate-local, no orphan-rule conflict). That blanket impl is what makes the four-per-trait impl pattern at every adapter and test-double site satisfy `PlatformAdapter` for `&dyn`/`impl PlatformAdapter` call sites — no fifth impl anywhere, and without it every one of the ~58 construction sites fails E0277. `lib.rs` re-exports preserve every public path. +- **Patterns to follow:** the macOS crate's `tree/actions/input/system` module split; `node.rs`'s grouped-small-types precedent for the extracted type files. +- **Test scenarios:** (1) full workspace test suite passes unchanged — the compiler plus existing tests are the behavior proof; (2) a new conformance test asserts `&dyn PlatformAdapter` exposes every method from all four supertraits (compile-time usage test in `src/tests/conformance.rs`); (3) `cargo tree -p agent-desktop-core` still names no platform crate; (4) every new file <400 LOC (existing CI/file conventions). +- **Verification:** all Verification Contract gates green with zero test-expectation edits outside mechanical `impl` splitting. + +### U1. Canonical role/state vocabulary + real `is --property visible` + +- **Goal:** Create the enforced role/state vocabulary and fix the `visible` defect by moving `is` onto live evidence. +- **Requirements:** R1, R11. +- **Dependencies:** U0. +- **Files:** `crates/core/src/state.rs` (new — 5 existing + 12 new token constants, `STATE_VOCABULARY`, `assert_states_in_vocabulary` conformance helper), `crates/core/src/role.rs` (new — `Role` enum with `as_str`/`from_str`/`is_interactive`; `roles.rs`'s `INTERACTIVE_ROLES` and `is_toggleable_role`/`is_expandable_role` rebase onto it, string API preserved), `crates/core/src/commands/is_check.rs` + `is_check_tests.rs`, `crates/core/src/commands/wait_predicate.rs` (its `Visible` arm adopts the same evidence). +- **Approach:** `is` resolves the ref and reads live evidence (`get_live_state` + `get_element_bounds`) instead of `state_from_ref_entry`'s snapshot clone; `visible` = bounds present, non-zero, and neither `hidden` nor `offscreen` token; unknown evidence → `applicable: false`-style honest reporting, never a false pass. All token literals in core (`is_check.rs:46-52` match arms, `wait_predicate` checks, capability applicability sets) move to `state::` constants. `IsProperty`/`ElementPredicate` unification stays deferred; both consume the same constants. +- **Patterns to follow:** `capability.rs`'s constants + membership-helper shape (the gap analysis names it the repo's best vocabulary precedent); the existing live-read usage in actionability checks. +- **Test scenarios:** (1) element with `hidden` state → `visible: false`; (2) zero-sized bounds → `false`; (3) `offscreen` token → `false`; (4) visible element → `true`; (5) live-read `not_supported` → applicability honestly degraded, not `true` (per-test adapter double); (6) conformance: every token consumed anywhere in core is in `STATE_VOCABULARY` (grep-derived guard, exhaustiveness-guard learning); (7) `Role::from_str("bogus")` → `Unknown`, `is_interactive` parity with the current 16-role list; (8) `enabled`/`checked`/`focused`/`expanded` behavior unchanged on live state (regression). +- **Verification:** AE1 lands in the e2e harness (`tests/e2e/`, the SwiftUI fixture's home); gates green. + +### U2. macOS state-producer expansion onto the vocabulary + +- **Goal:** One shared macOS state-reader emits the vocabulary tokens with direct AX evidence, replacing the two hand-duplicated producer sites. +- **Requirements:** R1, R11. +- **Dependencies:** U1. +- **Files:** `crates/macos/src/tree/state_reader.rs` (new — single producer), `crates/macos/src/tree/builder.rs` (inline states assembly at 152-176 delegates), `crates/macos/src/actions/post_state.rs` (`element_state_from_attrs`, lines 65-91, delegates), `crates/macos/src/tree/attributes.rs` (batch read gains the new AX attributes — both `AXUIElementCopyMultipleAttributeValues` call sites live in `element.rs:77-82`/`attributes.rs:154-159`), sibling `_tests.rs` files. +- **Approach:** existing tokens (`focused, disabled, secure, expanded, checked`) plus new producers with direct AX evidence: `selected` (AXSelected), `hidden` (AXHidden), `busy` (AXElementBusy), `modal` (AXModal), `required` (AXRequired where exposed), `indeterminate` (mixed AXValue on toggle roles), `pressed` (button role + boolean AXValue true), `readonly` (editable role whose value attribute is not settable — reuse the capabilities module's settability read), `offscreen` (computed: live bounds fail to intersect the owning window's bounds). `invalid`/`multiselectable`/`haspopup` stay vocabulary-only on macOS (no clean AX evidence — documented in `state.rs`). New attributes join the existing `AXUIElementCopyMultipleAttributeValues` batch (`attributes.rs:154-159`) — no per-attribute fetches. +- **Patterns to follow:** the batch-attribute-read gotcha in `CLAUDE.md`; `state_reader` unification follows the ref-alloc config-struct dedup learning (one shared body, not `_with_X` copies). +- **Test scenarios:** (1) conformance: every emitted token ∈ `STATE_VOCABULARY` (uses U1 helper); (2) both call sites produce identical tokens for identical evidence (dedup regression); (3) mixed-state checkbox → `indeterminate` + not `checked`; (4) hidden element → `hidden`; (5) window-clipped element → `offscreen`; (6) macOS integration: fixture app's known controls produce expected token sets; (7) golden snapshot fixtures updated only where new tokens appear (diff reviewed, no token disappears). +- **Verification:** `cargo test --lib -p agent-desktop-macos` + fixture integration green; AE1 remains green end-to-end. + +### U3. Display contract: `list_displays`, honest `--screen`, scale factor + +- **Goal:** Complete the dead display-targeting contract end-to-end. +- **Requirements:** R2. +- **Dependencies:** U0. +- **Files:** `crates/core/src/display_info.rs` (new — `DisplayInfo { id, bounds, is_primary, scale }`), `adapter/system.rs` (`list_displays` default `not_supported`), `crates/core/src/image_buffer.rs` (`scale_factor: f64`, serde default 1.0), `crates/core/src/commands/list_displays.rs` + `_tests.rs` (new command), `crates/core/src/commands/screenshot.rs` (validate index), `crates/macos/src/system/display.rs` (new — CoreGraphics active-display enumeration: bounds, main flag, points-vs-pixels scale), `crates/macos/src/system/screenshot.rs` (both `capture_screen` impls pass `-D <n>` to `/usr/sbin/screencapture`; populate `scale_factor`), `src/cli/mod.rs` + `src/cli_args/mod.rs` (`ScreenshotArgs` lives there at lines 154-165 — add `--screen N`; new `list-displays` subcommand args in `system.rs`), `src/dispatch/`; note `resolve_target()` (`commands/screenshot.rs:33-63`) currently never produces `Screen` — the new flag wires it. +- **Approach:** deterministic ordering (primary first, then stable display-id order) documented on the trait method; `--screen` out of range → `INVALID_ARGS` with available displays in `details` (count + ids — shapes, not content). `ImageBuffer.scale_factor` rides this unit deliberately: AX bounds are in points while captures are pixel-scaled, and without the factor on the result every consumer's crop/coordinate math silently breaks on Retina and mixed-scale multi-monitor setups — the exact evidence `list_displays` exists to supply (R2); `AdImageBuffer` is opaque, so FFI exposure is one accessor, no ABI pin. Honest framing: `Screen(usize)` was CLI-unreachable dead code, so this is contract completion, not a behavior fix; the gap report's "wrong screenshot" claim is corrected in the unit's commit message. +- **Patterns to follow:** `list-windows`/`list-apps` command shape (one command per file, registration checklist in `CLAUDE.md`'s Extensibility Pattern); `map_screencapture_error` for shell-error mapping. +- **Test scenarios:** (1) `list-displays` JSON envelope with ≥1 display, primary flagged; (2) `--screen 99` → `INVALID_ARGS` naming available count; (3) `--screen 0` targets primary (integration, macOS); (4) `scale_factor` present and ≥1.0 on captures; (5) adapter double returning 2 displays → command output ordering deterministic; (6) `list_displays` default → `PLATFORM_NOT_SUPPORTED` mapping (stub adapters); (7) FFI: `AdImageBuffer` is deliberately opaque (accessor-only, no `repr(C)` pin) — add an `ad_image_buffer_scale_factor` accessor, no pin sequence fires. +- **Verification:** new command passes CLI contract tests (`src/cli/contract_tests.rs`); gates green. + +### U4. Truthful Automation permission + +- **Goal:** `PermissionReport.automation` reflects reality; osascript fallbacks classify TCC denials as `PERM_DENIED`. +- **Requirements:** R3. +- **Dependencies:** none. +- **Files:** `crates/macos/src/system/permissions.rs` (add `automation_state()` via `AEDeterminePermissionToAutomateTarget(askUserIfNeeded=false)` against System Events; wire at the two construction sites, lines 63/74), `crates/macos/src/system/app_ops.rs` (`close_app_impl` osascript fallback: reclassify −1743/`errAEEventNotPermitted` stderr → `PERM_DENIED` + System Settings suggestion), notification session paths sharing the osascript pattern, sibling tests. +- **Approach:** probe never prompts (that is the point of `askUserIfNeeded=false`); the call requires building an `AEAddressDesc` target descriptor for System Events first (typeApplicationBundleID) — isolate descriptor construction + probe behind one testable fn. System Events not running → `Unknown`, honestly. Core's `PermissionReport` shape is already correct — this is adapter wiring only. Suggestion text mirrors the screen-recording suggestion's shape. +- **Patterns to follow:** `map_screencapture_error` (`screenshot.rs:105-129`) — the proven stderr-reclassification convention; reuse-existing-error-code learning (no new code). +- **Test scenarios:** (1) report mapping unit tests for permitted/denied/unknown probe outcomes (probe injected via small trait or fn pointer for testability); (2) `close_app` fallback with −1743 stderr → `PERM_DENIED` + suggestion (double-driven); (3) `permissions` command JSON shows real state on macOS (integration, environment-dependent assertions: value ∈ {granted, denied, unknown}, never `not_required`); (4) no TCC prompt during `permissions` (manual verification note, one-time). +- **Verification:** AE5 double-driven test green; gates green. + +**Phase B — identity spine (U5–U7).** + +### U5. `native_id` end-to-end + +- **Goal:** Capture the native automation id and make it the strongest re-identification signal. +- **Requirements:** R4. +- **Dependencies:** U0. +- **Files:** `crates/core/src/node.rs` (`AccessibilityNode.native_id: Option<String>`, serde skip-none), `crates/core/src/refs.rs` (`RefEntry` 17th field, serde default — old refmaps deserialize; current struct has 16), `crates/core/src/ref_identity.rs` (+`_tests`), `crates/macos/src/tree/attributes.rs` (batch read `kAXIdentifierAttribute`; `_NS`-prefixed auto-generated ids → `None`), `crates/core/src/ref_alloc.rs` (thread the field), `crates/ffi/src/types/ref_entry.rs` (`AdRefEntry` is a pinned `repr(C)` mirror — add the field via the full 3-layer size-pin sequence: const assert, `ad_ref_entry_size`, `tests/c_abi_layout.rs` literal) + conversion code + header regen via `scripts/update-ffi-header.sh`, snapshot serialization tests + golden fixtures. +- **Approach:** identity precedence in `identity_matches`: equal present `native_id`s → strongest positive (with pid+role); differing present ids → hard non-match; absent on either side → existing evidence chain unchanged. `has_meaningful_identity` counts it. Snapshot JSON includes it (agents can address by it later; it is already redaction-relevant? — it is a developer-assigned identifier, a shape not user content; no redaction key needed). +- **Patterns to follow:** the existing 14-field `RefEntry` evidence conventions; identity-fingerprint learning ("plumb the handle/fingerprint, fail closed on mismatch"). +- **Test scenarios:** (1) same id → match survives name/value/bounds drift (the localization-resilience case); (2) different present ids → non-match even with identical name+role; (3) absent ids → behavior identical to today (regression on existing identity test corpus); (4) `_NS:123`-style id filtered to `None`; (5) old refmap JSON without the field loads (serde default); (6) fixture snapshot includes `native_id` where the SwiftUI fixture sets `accessibilityIdentifier`; (7) refmap size guard unaffected (<1MB write-side check); (8) FFI: `AdRefEntry` 3-layer pin updated in lockstep (const assert + runtime accessor + layout-test literal), FFI resolution path carries the field (CLI/FFI evidence parity). +- **Verification:** ref-identity unit suite + macOS resolve integration green; FFI header drift gate green after regen; gates green. + +### U6. Window identity as primary key + +- **Goal:** Window-scoped operations resolve by `WindowInfo.id` first; title is fallback evidence only. +- **Requirements:** R5. +- **Dependencies:** U0. +- **Files:** `crates/core/src/adapter/system.rs` (`resolve_window_strict(&self, id: &str)` default `not_supported`), `crates/macos/src/adapter.rs:51` (get_tree default surface), `crates/macos/src/system/window_ops.rs:26` (`window_op`), `crates/macos/src/system/app_ops.rs:90` (`focus_window_impl`) — all three re-resolve id-first via CGWindowList `kCGWindowNumber` match, `(pid, title)` only when the caller supplied no id; `src/tests/conformance.rs` + `tests/conformance/` (new shared contract test), macOS integration test. +- **Approach:** core documents the resolution obligation on the trait (`WindowInfo.id` is an opaque platform string — never assume numeric, per the AT-SPI D-Bus-path note in the gap analysis). **Id match alone is not trusted:** macOS `kCGWindowNumber` values are recycled after windows close, so `resolve_window_strict` corroborates the id-matched row against `pid` (always) and `title` (when the caller's `WindowInfo` carries one) and fails closed with `WINDOW_NOT_FOUND` on mismatch — the same optional-fingerprint-verify-before-acting shape `NotificationIdentity` already uses. The three macOS sites keep their title path only as explicit fallback for id-less flows. +- **Patterns to follow:** identity-fingerprint learning (window lists are its named example); `ContractAdapter` conformance-test shape. +- **Test scenarios:** (1) conformance: two same-titled, different-id windows — operation with id targets the id-matched one (adapter double); (2) id supplied but no longer present → structured `WINDOW_NOT_FOUND`, not silent first-title match; (3) **recycled id: id matches a live window but its pid differs from the caller's `WindowInfo.pid` → `WINDOW_NOT_FOUND` (fail-closed), never a silent action on the impostor window**; (4) title-only flow unchanged (regression); (5) macOS integration: two "Untitled" TextEdit windows, focus/resize by id acts on the right one; (6) `resolve_window_strict` default → `not_supported` on stub adapters. +- **Verification:** AE4 integration test green; gates green. + +### U7. `LocatorQuery` + `resolve_query` + `find` integration + +- **Goal:** A serializable, live-resolving element query as core contract, consumed by `find`. +- **Requirements:** R6. +- **Dependencies:** U0, U1, U5. +- **Files:** `crates/core/src/locator.rs` + `_tests.rs` (new — `LocatorQuery`, `StatePredicate`, `classify_query_result`), `crates/core/src/adapter/observation.rs` (`resolve_query(&self, &LocatorQuery, scope: Option<&NativeHandle>) → Vec<NativeHandle>` default `not_supported`), `crates/core/src/commands/query.rs` (`FindQuery` becomes a thin constructor of `LocatorQuery`; selector syntax unchanged), `crates/core/src/commands/find.rs` + `_tests.rs` (new flags: `--exact`, `--state token[=bool]` repeatable, `--has-text`, `--native-id`; existing `--nth/--first/--last/--count` map onto query ordinals), `crates/macos/src/tree/query.rs` (new — live traversal matcher reusing the builder's traversal guards and depth caps), `src/cli_args/`. +- **Approach:** matching semantics defined in core and unit-tested against tree fixtures: name/description substring case-insensitive by default, `exact` for equality; `state` predicates evaluate vocabulary tokens against live state; `has`/`has_not` are subtree containment (core-side walk over returned subtrees where the adapter returns candidates); `native_id` exact-match always. `classify_query_result` reuses the strict-resolution outcome contract (0/1/N) and the existing candidate-summary shape — implemented and tested now, wired to actions in Tier-2. `find` keeps snapshot/ref materialization behavior; only its matching backend changes. macOS `resolve_query` walks live AX with the same ancestor-path cycle guard as the snapshot builder. +- **Patterns to follow:** `parse_selector`/`validate_selector` conventions; progressive-snapshot learning (malformed input → `INVALID_ARGS`, never `STALE_REF`); the resolver-deadline sharing rule from the reliability learning. +- **Test scenarios:** (1) role+name exact vs substring; (2) `--state checked=true` filters live state (double with controllable live reads); (3) `has_text` containment matches the row-with-text case; (4) `has`/`has_not` subqueries; (5) `native_id` match ignores renamed labels; (6) `nth/first/last` ordinal determinism (document order); (7) `classify_query_result`: 0 → not-found shape, 1 → accept, 2+ → ambiguous with ≤10 candidate summaries; (8) invalid state token → `INVALID_ARGS` naming the vocabulary; (9) `find` CLI regression corpus unchanged for existing flags; (10) macOS integration: query against fixture app returns stable matches under UI idle. +- **Verification:** `find` conformance suite + macOS integration green; gates green. + +**Phase C — actionability (U8–U10).** + +### U8. Auto-wait pre-action gate, default on + +- **Goal:** Every ref-addressed action retries resolve+actionability under one bounded budget by default; the pre-action timeout fragmentation ends. +- **Requirements:** R7. +- **Dependencies:** U0. +- **Files:** `crates/core/src/action_request.rs` (`timeout_ms: Option<u64>`, serde default), `crates/core/src/ref_action.rs` + `_tests.rs` (the poll loop wraps `check_actionability_with_trace` + resolve inside `execute_resolved`/`execute_entry_with_context` — today `check_live` runs once at `ref_action.rs:66` before dispatch), `crates/core/src/commands/helpers.rs` (shared ref-action arg threading), `src/cli_args/mod.rs` (`RefArgs`, lines 209-219 — used verbatim by 12 ref commands) + `src/cli_args/actions.rs` (the payload-carrying structs that inline their own `ref_id`+`snapshot`: `TypeArgs`, `SetValueArgs`, `SelectArgs`, `ScrollArgs`, plus `HoverArgs`/`DragCliArgs`) — `--timeout-ms` clap default 5000 on each, `crates/core/src/commands/wait_element.rs` (`WAIT_RESOLVE_ATTEMPT` becomes the loop's per-tick resolve budget — one owner), `crates/ffi/src/actions/execute.rs` + `conversion.rs` (default 5000 parity; additive `ad_execute_by_ref_timeout`; `AdAction` pin untouched — timeout rides the new fn's parameter, never the struct), `crates/ffi/include/agent_desktop.h` via `scripts/update-ffi-header.sh`, e2e assertions. +- **Execution note:** start with failing core tests for the loop's retry/permanent/deadline matrix before touching dispatch (the policy-preservation learning makes this the highest-blast-radius unit). +- **Approach:** per KTD3. Iteration cadence, stated honestly: 100 ms sleep between iterations, and each iteration's resolve gets `remaining.min(WAIT_RESOLVE_ATTEMPT)` (750 ms) — so iterations cost ~100 ms when the element resolves fast but is not yet actionable, and up to ~850 ms when resolve itself is slow (element missing), meaning the 5000 ms default yields ~6 attempts in the slow-resolve regime, not 50. Retryable = `STALE_REF`, `AMBIGUOUS_TARGET`, actionability Fail, `TIMEOUT`-from-resolve; permanent = `PERM_DENIED`, `APP_NOT_FOUND`, `ACTION_NOT_SUPPORTED`, `INVALID_ARGS`, `POLICY_DENIED`. **Transient-ambiguity audit:** if any iteration observed `AMBIGUOUS_TARGET` and a later iteration resolves to one match and acts, the success result carries `details.transient_ambiguity: true` (a shape — redaction-safe) so callers can audit or reject an action whose target identity flickered. Deadline → `TIMEOUT`, `details.kind = "actionability_timeout"`, last `ActionabilityReport` attached. **CLI/batch default equivalence is enforced at the arg-struct layer, not assumed:** clap's `default_value_t` does not participate in serde deserialization, and batch decodes JSON into these same structs — so the new field follows the repo's established paired pattern (`ScrollArgs.direction`, `MouseClickArgs.count`): `#[arg(long = "timeout-ms", default_value_t = 5000)]` + `#[serde(default = "default_timeout_ms")]`, with `0` mapping to `ActionRequest.timeout_ms = None` at request build. Batch entries omitting the field therefore auto-wait at 5000 ms too — part of the same documented breaking change. The loop never selects policy and never bypasses per-command post-condition verification. Tests constructing `ActionRequest` directly stay single-shot (`None`); e2e failure-path cases pass `--timeout-ms 0`. +- **Patterns to follow:** wait command's retryable/permanent split and `TIMEOUT` `kind` convention (reliability learning); FFI/CLI policy-alignment learning (mandatory `crates/ffi/src/actions/` review pass). +- **Test scenarios:** (1) target actionable on first check → zero added latency, single check (call-count assert on double); (2) actionable on 3rd tick → succeeds, elapsed ≈ 200–400 ms; (3) permanently Fail → `TIMEOUT` at budget with `kind` + last report; (4) `PERM_DENIED` mid-loop → immediate propagation; (5) `STALE_REF` twice then resolves → succeeds; (6) `AMBIGUOUS_TARGET` persists → `TIMEOUT` carrying candidate summaries; (7) `AMBIGUOUS_TARGET` once then a clean single match → succeeds **with `details.transient_ambiguity: true`**; (8) slow-resolve double consuming its full per-attempt budget each call → attempt count within the 5000 ms budget asserted (~6, the honest worst case); (9) `timeout_ms: None` → exactly one check (wire/back-compat regression); (10) `--timeout-ms 0` maps to `None`; (11) batch JSON entry omitting `timeout_ms` deserializes to 5000 (serde-default parity — the clap default never fires for batch); (12) post-condition verification still runs after a waited success (policy-preservation regression); (13) FFI default parity: `ad_execute_by_ref` waits, `ad_execute_by_ref_timeout(0)` is single-shot; (14) envelope: no version bump (additive optional field only); (15) e2e: AE2, AE3, and AE7 against the fixture app in both headless and `--headed`. +- **Verification:** e2e suite green including the new AE2/AE3 cases; FFI header drift gate green after regeneration. + +### U9. `hit_test` + `receives_events` occlusion detection + +- **Goal:** Occluded targets are caught before dispatch; ref-targeted pointer actions stop bypassing actionability. +- **Requirements:** R8. +- **Dependencies:** U0, U8. +- **Files:** `crates/core/src/hit_test.rs` (new — `HitTestResult { ReachesTarget, InterceptedBy { role, name, bounds }, Unknown }`), `crates/core/src/adapter/observation.rs` (`hit_test(&self, target: &NativeHandle, point: Point)` default `not_supported`), the actionability check module (+`receives_events` check, Unknown on `not_supported`), `crates/core/src/ref_action.rs` (ref-targeted `Hover`/`Drag` route through `check_live` minus `supported_action`/`editable`), `crates/macos/src/tree/hit_test.rs` (new — `AXUIElementCopyElementAtPosition` + `CFEqual` ancestor walk), sibling tests. +- **Approach:** per KTD4. Check point = center of live bounds. Three-way classification: Pass iff hit element ∈ target's subtree (target or descendant); **hit on the target's own ancestor → Unknown, not Fail** — AX hit-testing commonly resolves to a coarser container on composited/custom-drawn views where no distinct child node is hit-testable, and treating that as occlusion would false-Fail working clicks (this is the Risks section's Unknown-on-weird-evidence rule made explicit); Fail with occluder summary (`role` + `name` keys — redaction-safe) is reserved for hits **outside the target's ancestor chain** (true occluders: modals, overlays, sibling panels); probe error or `not_supported` → Unknown (never false failure, per the reliability learning's evidence rule). Raw coordinate `mouse-*` commands unchanged by design (documented). +- **Patterns to follow:** existing actionability check structure (per-check Pass/Fail/Unknown with reason); ancestor-path traversal guard from the macOS builder. +- **Test scenarios:** (1) unobstructed target → Pass; (2) modal overlay covering center → Fail naming occluder role (out-of-chain hit); (3) hit lands on target's own text child → Pass (descendant rule); (4) hit on target's ancestor → **Unknown, action proceeds** (composited-container case — never a false occlusion Fail); (5) `not_supported` adapter → Unknown, action proceeds; (6) `hover --ref` on disabled/occluded element now fails preflight (previously dispatched blind); (7) occluder name redacted in trace output (`trace_sanitize` regression); (8) macOS integration: sheet over button → click blocked with occluder detail, sheet dismissed → click passes within one auto-wait budget (composes with U8). +- **Verification:** actionability suite + macOS integration green; gates green. + +### U10. `scroll_into_view` as core contract + +- **Goal:** Element-targeted actions scroll off-screen targets into view uniformly, best-effort, before the visibility check. +- **Requirements:** R9. +- **Dependencies:** U0, U8. +- **Files:** `crates/core/src/action.rs` (`requires_scroll_into_view()` alongside the existing per-variant policy helpers), `crates/core/src/adapter/actions.rs` (`scroll_into_view(&NativeHandle)` default `not_supported`), `crates/core/src/ref_action.rs` (pre-check call inside the U8 loop), `crates/macos/src/actions/` (promote `ax_helpers::ensure_visible` — the real `AXScrollToVisible` invocation, wired today only through `CLICK_CHAIN`'s `pre_scroll` at `chain.rs:42-46` — into the `ActionOps::scroll_into_view` impl; the chain's `pre_scroll` rung delegates to the same fn), sibling tests. +- **Approach:** called once per loop iteration before visibility when the action requires it; any error degrades silently to today's behavior (best-effort, never fails the action); macOS falls back to geometric scroll only where the AX action is unavailable on the container (reuse existing scroll semantics), keeping the fallback inside the adapter per the gesture-capability learning ("the command never decides"). +- **Patterns to follow:** `Action` policy-helper style (`requires_cursor_policy`-family); chain `pre_scroll` as the extraction source. +- **Test scenarios:** (1) `requires_scroll_into_view` truth table over all 21 `Action` variants (exhaustive match — compiler-enforced); (2) core calls it before visibility for `Click`, not for `PressKey` (call-order assert via double); (3) `not_supported` → action proceeds to visibility check unchanged; (4) adapter error → logged step, no failure; (5) macOS integration: off-screen row in a scroll area — `click --ref` succeeds where today it fails visibility; (6) chain regression: existing `CLICK_CHAIN` behavior unchanged for on-screen targets. +- **Verification:** e2e scroll-area case green in both modes; gates green. + +**Phase D — naming, surfaces, action reporting (U11–U13).** + +### U11. Core accessible-name computation + +- **Goal:** One core-owned name/description algorithm over adapter-supplied evidence. +- **Requirements:** R12. +- **Dependencies:** U0. +- **Execution note:** characterization-first — snapshot the fixture apps' current computed names into tests before migrating, then diff consciously. +- **Files:** `crates/core/src/accname.rs` + `_tests.rs` (new — `NameEvidence`, `compute_name`, `compute_description`, documented precedence), `crates/core/src/adapter/observation.rs` (`get_live_name_evidence` default `not_supported`), `crates/macos/src/tree/element.rs` (+`attributes.rs`: `resolve_element_name` becomes the evidence supplier feeding `compute_name`), golden fixtures under `tests/fixtures/`, macOS integration tests. +- **Approach:** per KTD6. Precedence: explicit label → labelled-by text → native title → static-role value → aggregated child label → placeholder → description. Evidence fields are all `Option<&str>`; core treats absent evidence as skip-to-next. The live path (`get_live_name_evidence`) serves future retrying name assertions; snapshot-time naming uses the same `compute_name` on evidence gathered during the batch read. +- **Patterns to follow:** vocabulary-module shape from U1 (algorithm + conformance in core, evidence at the adapter); UIA/AT-SPI mapping notes recorded as doc comments on `NameEvidence` fields. +- **Test scenarios:** (1) precedence table — one test per rung proving the earlier rung wins; (2) all-absent evidence → `None` (ref-ability unaffected elsewhere); (3) child-label aggregation joins in document order; (4) characterization: fixture-app names before == after for the common cases, intentional diffs enumerated and approved in the fixture update; (5) `get_live_name_evidence` default → `not_supported`; (6) macOS: evidence supplier returns raw attributes without applying its own precedence (grep-guard: `resolve_element_name` no longer contains fallback chains). +- **Verification:** fixture diffs reviewed and committed with the unit; ref-identity suite unaffected; gates green. + +### U12. `SnapshotSurface` platform-neutrality ratification + +- **Goal:** `Sheet`/`Popover` become explicitly platform-declared rather than silently assumed portable. +- **Requirements:** R10. +- **Dependencies:** U0. +- **Files:** `crates/core/src/adapter/system.rs` (`supported_surfaces() → &'static [SnapshotSurface]`, default `[Window, Focused, Menu, Menubar, Alert]`), `crates/macos/src/adapter.rs` (override adds `Sheet`, `Popover`), `crates/core/src/commands/snapshot.rs` (unsupported requested surface → `PLATFORM_NOT_SUPPORTED` naming the supported set), `crates/core/src/commands/status.rs` (surface list in `status` output), doc comments on the enum ratifying the semantics per variant + UIA/AT-SPI mapping notes. +- **Approach:** additive; macOS behavior unchanged. The enum stays `#[non_exhaustive]`; the contract text (which variants are universal, which are platform-native) lives on the enum as `///` docs — the ratification the gap analysis demanded before Phase 2. +- **Patterns to follow:** `permission_report`-style introspection surfaced through `status`. +- **Test scenarios:** (1) default trait impl excludes `Sheet`/`Popover`; (2) macOS reports all 7; (3) requesting `--surface sheet` against a double without it → `PLATFORM_NOT_SUPPORTED` with supported list in details; (4) `status` JSON includes `supported_surfaces`; (5) envelope: additive field only, no version bump (assert via `ENVELOPE_VERSION`). +- **Verification:** snapshot/status contract tests green; gates green. + +### U13. Typed `ActionStep` delivery tier + +- **Goal:** Callers can programmatically distinguish semantic-API delivery from physical synthesis, and whether the effect was verified. +- **Requirements:** R19. +- **Dependencies:** none (core type + macOS chain population). +- **Files:** `crates/core/src/action_step.rs` (+`StepMechanism { SemanticApi, PhysicalSynthetic }`, `mechanism: Option<StepMechanism>`, `verified: Option<bool>`, serde skip-none; note current shape is private `label: String` + `pub outcome: ActionStepOutcome` with `attempted/skipped/succeeded` constructors — extend via builder methods, not field surgery), `crates/macos/src/actions/chain*.rs` (each rung tags its mechanism; verification rungs set `verified` on the step they confirm), `crates/macos/src/actions/dispatch.rs` (direct non-chain paths tag too), `crates/ffi/src/types/action_step.rs` (`AdActionStep` is a pinned `repr(C)` mirror — expose the two fields via the full 3-layer size-pin sequence + header regen; result-shape parity is part of the FFI action path), trace fixture updates. +- **Approach:** labels stay (back-compat); the typed fields are additive. Population is mechanical: AX `AXUIElementPerformAction`/`SetAttributeValue` rungs → `SemanticApi`; CGEvent rungs → `PhysicalSynthetic`; the chain's existing effect-verification outcome writes `verified: Some(bool)` — no new verification logic, only surfacing what the chain already computes (policy-preservation learning: verification behavior itself untouched). +- **Patterns to follow:** existing `ActionStep{label, outcome}` conventions; trace-redaction learning (both new fields are shapes — enum + bool — safe by construction). +- **Test scenarios:** (1) headless click on AX-supporting control → all steps `SemanticApi`, final step `verified: true`; (2) `--headed` physical fallback → the fallback rung tagged `PhysicalSynthetic`; (3) serde: absent fields for legacy steps (round-trip old JSON); (4) trace export includes the fields un-redacted (shape fields, sanitizer regression); (5) chain outcome/step-count regression: no behavior change to rung execution; (6) FFI: `AdActionStep` pin triple updated (const assert + `ad_action_step_size` + layout literal), zeroed-read check extended per the size-pin learning. +- **Verification:** action-result serialization suite + trace viewer fixture green; FFI header drift gate green; gates green. + +**Phase E — process, launch, session lifecycle (U14–U16).** + +### U14. `ProcessState`, `APP_UNRESPONSIVE`, enriched errors + +- **Goal:** Process liveness/hang become classifiable evidence; persistent AX unresponsiveness gets its own error code; resolution errors carry process state. +- **Requirements:** R13. +- **Dependencies:** U0. +- **Files:** `crates/core/src/process_state.rs` (new — enum per KTD8), `crates/core/src/adapter/system.rs` (`process_state(&self, pid)` default `not_supported`), `crates/core/src/error.rs` (`AppUnresponsive` 16th variant + code string `APP_UNRESPONSIVE`), `crates/core/src/output.rs` (`retry_token_for_code` arm; envelope version constant bump), `crates/core/src/ref_action.rs` + app-resolution paths (attach `details.process_state` on `STALE_REF`/`APP_NOT_FOUND` when pid known, best-effort), `crates/macos/src/system/process.rs` (new — `kill(pid,0)` liveness + bounded AX responsiveness probe), macOS `ax_helpers` (persisting `kAXErrorCannotComplete` → `AppUnresponsive` classification), `CLAUDE.md` error-code list, `docs/phases.md` naming-reconciliation note, batch/main envelope tests. +- **Approach:** per KTD8. Enrichment is additive `details` only and never converts a success to failure; **it runs exactly once, when constructing the terminal caller-visible error — never on an internal auto-wait retry tick** (U8's loop and this enrichment share `ref_action.rs`; the probe must not multiply across ~6–50 iterations of an exhausted budget). Classification threshold: the existing one-retry on `CannotComplete` stays, a second consecutive failure within one command classifies. `ENVELOPE_VERSION` bump per the envelope learning, all assertions via the constant. +- **Patterns to follow:** envelope-version learning verbatim; the reliability learning's structured-error conventions (`suggestion` on the new code: "app may be hung; consider close-app --force"). +- **Test scenarios:** (1) live pid → `Running`; (2) exited pid → `Exited{code: None}`; (3) AX probe timing out twice → `Unresponsive` + `APP_UNRESPONSIVE` from the action path with suggestion; (4) `STALE_REF` against dead pid carries `details.process_state = "exited"`; (5) enrichment failure (probe errors) → original error unchanged, no panic; (6) probe call count is independent of auto-wait tick count (terminal-only enrichment, asserted via probe-counting double); (7) envelope tests updated through `ENVELOPE_VERSION` (main + batch + unit, per the learning); (8) `retry_token_for_code(APP_UNRESPONSIVE)` yields a sensible recovery token; (9) exit-code 2 arg-error path unaffected. +- **Verification:** full envelope/conformance suites green after the version bump; gates green. + +### U15. `LaunchOptions` + +- **Goal:** Launching accepts args/env/cwd and an explicit attach-vs-fail policy. +- **Requirements:** R14. +- **Dependencies:** U0. +- **Files:** `crates/core/src/launch_options.rs` (new), `crates/core/src/adapter/system.rs` (`launch_app(&self, id, &LaunchOptions)` — direct signature change per KTD14), `crates/core/src/commands/launch.rs`, `src/cli_args/` (`--arg`, `--env`, `--cwd`, `--no-attach`), `crates/macos/src/system/app_ops.rs` (`open -g -a` gains `--args` pass-through and env/cwd via spawn where `open` cannot carry them — adapter-internal choice), `crates/ffi/src/apps/launch.rs` (internal default construction, C signature unchanged), all launch test doubles. +- **Approach:** `attach_if_running: true` default preserves today's semantics exactly; `false` + already-running → structured error naming the running pid. macOS: plain `open -g -a` path when options are empty (zero regression risk); options present → `NSWorkspace`/spawn path. Struct stays ≤5 fields (God-object rule). +- **Patterns to follow:** config-struct-over-parameter-sprawl (repo naming conventions); `open -g` background-launch semantics preserved. +- **Test scenarios:** (1) empty options → byte-identical launch behavior (double asserts the same adapter call shape); (2) `--env KEY=VAL` parse + invalid form → `INVALID_ARGS` whose `message` and `details` carry position/key-name only — a regression test mirroring the existing `wait_text_timeout_message_omits_raw_text_from_trace_segment` guard asserts the raw value never reaches a trace segment; (3) `--no-attach` with running app → structured failure naming pid; (4) attach default with running app → success (today's behavior); (5) FFI `ad_launch_app` unchanged signature, defaults verified; (6) macOS integration: launch fixture app with a marker arg, assert received (fixture reads argv); (7) trace of a successful `launch` with env set carries no env value anywhere in the segment (sanitizer + message audit). +- **Verification:** launch suite + FFI compile + header drift gate green. + +### U16. `open_session` adapter-affinity hook + +- **Goal:** The stateful-adapter landing zone exists before Phase 2 starts; nothing calls it yet. +- **Requirements:** R15. +- **Dependencies:** U0. +- **Files:** `crates/core/src/session_affinity.rs` (new — `SessionAffinity { session_id: Option<String> }` extending the manifest vocabulary, and `AdapterSession: Send + Sync { fn close(self: Box<Self>) → Result<(), AdapterError>; }`), `crates/core/src/adapter/system.rs` (`open_session` default `not_supported` + doc naming the COM-MTA/D-Bus owners), conformance test. +- **Approach:** per KTD15 — contract only. The doc comment is the deliverable as much as the signature: it states what state may live inside a session (native connection affinity), what must not (resolved element handles — the RAII learning), and that the CLI path remains stateless. +- **Patterns to follow:** the trait's existing default-body convention; `resolve-then-release` RAII boundary documented in the gap analysis's strengths list. +- **Test scenarios:** (1) default returns `not_supported`; (2) a test double implementing a session (flag-setting `close`) proves the shape is implementable and object-safe; (3) `Box<dyn AdapterSession>` is `Send + Sync` (compile-time assertion); (4) no CLI/dispatch call sites exist (grep guard test, exhaustiveness-guard pattern). +- **Verification:** compile + conformance green; gates green. + +**Phase F — signals and input vocabulary (U17–U19).** + +### U17. `SignalBaseline`, `diff_signals`, `wait --event` + +- **Goal:** Title-agnostic "what appeared/changed" detection as core contract, generalizing the notification fingerprint pattern. +- **Requirements:** R16. +- **Dependencies:** U0. +- **Files:** `crates/core/src/signals.rs` + `_tests.rs` (new — `EventKind { WindowOpened, WindowClosed, AppLaunched, AppTerminated, FocusChangedWindow, SurfaceAppeared{kind}, SurfaceDismissed{kind} }`, `SignalFilter { app, pid }`, `SignalBaseline` plain data, pure `diff_signals`, `UiEvent`), `crates/core/src/adapter/system.rs` (`snapshot_signals` default `not_supported`), `crates/core/src/commands/wait_event.rs` + `_tests.rs` (new — `WaitMode::Event` variant in `wait_mode.rs`, registered in the `WAIT_SUPPORTED` allowlist at `src/main.rs`), `crates/macos/src/system/signals.rs` (new — windows via CGWindowList ids, apps via workspace list, focused window; alert/sheet presence when `filter.app` set), `src/cli_args/system.rs` (`--event <kind>` on wait). +- **Approach:** per KTD12. Baseline is counting-map-shaped like `NotificationFingerprint` (survives reordering); diff is pure and double-testable. `wait --event` snapshots at start, polls at the existing wait cadence, returns first matching events; timeout follows the `wait_timeout` kind convention with the baseline summary (counts only — shapes) in details. Event payloads put window/surface titles under `title` keys (redaction-covered). FFI: `ad_wait` keeps its existing modes — `AdWaitArgs` is a pinned `repr(C)` struct, and the Event mode's FFI exposure is explicitly deferred to the same follow-up batch as locator FFI (recorded in Scope Boundaries), so no pin changes fire in this unit. +- **Patterns to follow:** `wait_for_notification` baseline/diff loop (`wait.rs:233-329`) as the reference implementation; wait-file splitting convention (`wait.rs` is at 349 LOC — new mode lives in its own file); exhaustiveness-guard learning for the `WAIT_SUPPORTED` registration. +- **Test scenarios:** (1) `diff_signals` pure tests: new window id → `WindowOpened`; removed → `WindowClosed`; focus id change → `FocusChangedWindow`; app pid appears/disappears → launched/terminated; sheet appears under app filter → `SurfaceAppeared{Sheet}`; (2) reorder-only baselines → zero events (fingerprint property); (3) duplicate-title windows counted correctly (counting map); (4) `wait --event window-opened` end-to-end with double: appears on 3rd poll → success with event payload; (5) timeout → `TIMEOUT` + `kind:"wait_timeout"` + counts-only details; (6) unknown `--event` value → `INVALID_ARGS` naming kinds; (7) trace: event title redacted (sanitizer regression); (8) macOS integration: open TextEdit document window, `wait --event window-opened --app TextEdit` fires without knowing the title (AE6). +- **Verification:** AE6 integration green; gates green. + +### U18. Typed clipboard content + +- **Goal:** Clipboard round-trips text, images, and file lists through one typed contract. +- **Requirements:** R17. +- **Dependencies:** U0. +- **Files:** `crates/core/src/clipboard_content.rs` (new), `crates/core/src/adapter/input.rs` (replace `get_clipboard`/`set_clipboard`/`clear_clipboard`-adjacent string surface with `get_clipboard_content`/`set_clipboard_content`; `clear_clipboard` unchanged), `crates/core/src/commands/{clipboard_get,clipboard_set}.rs`, `src/cli_args/` (`--format`, `--out`, `--image`, `--file-url` repeatable), `crates/macos/src/input/clipboard.rs` (NSPasteboard string/PNG/fileURL read+write, reusing the restore machinery's type round-tripping), `crates/ffi/src/input/clipboard.rs` (existing text fns delegate through `ClipboardContent::Text` — C signatures unchanged), sibling tests. +- **Approach:** per KTD13. `clipboard-get --format auto` reports the richest available type; image bytes go to `--out` path (default: temp file under the session dir) written via the `write_private_file` pattern (0600, `O_NOFOLLOW`, atomic rename — clipboard images can hold copied secrets; screenshot's unguarded `std::fs::write` is the wrong precedent for this file class), JSON carries `{type, path, width, height}` — envelope stays small and redaction-safe (paths are shapes). Setting file URLs validates existence up front (`INVALID_ARGS` reporting count + entry indexes only — basenames themselves can be sensitive and `details` reaches traces). +- **Patterns to follow:** `refs.rs::write_private_file` for the image temp file; screenshot's `--out` flag ergonomics; pasteboard-restore machinery as the macOS marshaling reference. +- **Test scenarios:** (1) text round-trip through the typed API (regression vs old string behavior); (2) image set-then-get round-trips pixel dimensions (macOS integration); (3) file-urls set → Finder-paste-shaped pasteboard content (integration, assert via get); (4) `--format text` on image-only clipboard → structured empty/`NOT_FOUND`-style result, not a panic; (5) FFI text fns behave identically pre/post (parity regression); (6) core command tests with double covering all three variants + `auto` preference order; (7) missing `--file-url` path → `INVALID_ARGS` with count + entry-index details only (no path content); (8) default image temp file lands 0600 under the session dir (mode assertion, Unix). +- **Verification:** clipboard suite + FFI parity green; gates green. + +### U19. Mouse modifiers + wheel primitive + +- **Goal:** Chorded clicks and a first-class wheel-delta command. +- **Requirements:** R18. +- **Dependencies:** U0 (file moves only). +- **Files:** `crates/core/src/action.rs` (`MouseEvent.modifiers: Vec<Modifier>` serde-default-empty; `MouseEventKind::Wheel { delta_x: f64, delta_y: f64 }`), `crates/core/src/commands/mouse_wheel.rs` + `_tests.rs` (new command), existing mouse command files (`--modifiers` flag), `src/cli_args/`, `crates/macos/src/input/mouse.rs` (CGEventFlags from the existing `Modifier` mapping; wheel reuses `synthesize_scroll_at` — the `CGEventCreateScrollWheelEvent` call already in this file at lines 282-296 — invoked directly instead of only via `scroll`'s fallback), FFI `crates/ffi/src/input/mouse.rs` (additive fn carrying modifiers as a bitmask param + wheel deltas; `AdMouseEvent` — a pinned `repr(C)` mirror — stays untouched, so its pin triple stands; header regen), doc note on the enum: deltas are wheel lines; X11 button-4/5/6/7 translation is adapter-side (KTD17). +- **Approach:** modifiers apply to Down/Up/Click kinds by setting event flags for the synthetic event's duration (restore after — no sticky modifier leakage; mirror `KeyCombo` handling). `Wheel` ignores `button`. `mouse-wheel --dx N --dy N [--at X,Y]` is raw-input tier (no ref, no actionability — documented like the other `mouse-*` commands). +- **Patterns to follow:** `MouseEventKind::Click{count}` precedent for kind-level extension; keyboard modifier synthesis for flag handling; one-command-per-file registration checklist. +- **Test scenarios:** (1) serde: legacy `MouseEvent` JSON without `modifiers` deserializes (default empty); (2) `--modifiers cmd,shift` parse + unknown modifier → `INVALID_ARGS`; (3) wheel deltas serialize/dispatch with sign conventions documented (positive = up/left or down/right — pick and test one, document on the type); (4) macOS integration: cmd-click in fixture multi-select list selects additively; (5) wheel scroll moves fixture scroll area (integration, both directions); (6) modifier restore: subsequent unmodified click carries no stale flags; (7) FFI additive fn present in regenerated header; `AdMouseEvent`'s existing pin triple unchanged (asserted — the new fn takes params, not a struct change). +- **Verification:** input suite + e2e wheel/chord cases green; header drift gate green. + +--- + +## Verification Contract + +| Gate | Command | Applies to | +|---|---|---| +| Format | `cargo fmt --all -- --check` | every unit | +| Lint | `cargo clippy --all-targets -- -D warnings` | every unit | +| Core/workspace tests | `cargo test --lib --workspace` | every unit | +| Binary conformance | `cargo test -p agent-desktop` | every unit | +| FFI tests | `cargo test -p agent-desktop-ffi --tests` | U3, U8, U15, U18, U19 (and any unit touching `crates/ffi`) | +| Core isolation | `cargo tree -p agent-desktop-core` contains no platform crate names | every unit (CI-enforced) | +| FFI header drift | CI `ffi-header-drift` job (`cbindgen --verify` via pinned 0.29.4); regenerate with `scripts/update-ffi-header.sh` | U8, U19 (any header-affecting unit) | +| Binary size | release binary <15MB (CI check) | final | +| File budget | every touched file <400 LOC (except `@generated`) | every unit | +| E2E | `bash tests/e2e/run.sh` (release build + AX permission, headless and `--headed`) | U1, U2, U8, U9, U10, U17, U19 at minimum; full run before merge | +| Envelope discipline | version assertions via `ENVELOPE_VERSION` constant only; exactly one bump (U14) across the plan | U12, U14 | + +Behavioral acceptance: AE1–AE7 each map to a named integration/e2e test landed with its owning unit (AE1→U1/U2, AE2/AE3/AE7→U8, AE4→U6, AE5→U4, AE6→U17). + +--- + +## Definition of Done + +- All 20 units merged in dependency order with one conventional commit each (`feat:`/`fix:` per unit intent; U1/U4 carry `fix:`; **U8 carries a `BREAKING CHANGE:` footer** — the default-on auto-wait silently changes every existing untouched call's timing, which meets the repo's breaking bar and must cut a minor version pre-1.0, not a patch). +- Every Verification Contract gate green on the branch tip; full e2e pass in both headless and `--headed` modes. +- R1–R19 each traceable to at least one landed test; AE1–AE6 green. +- All new `PlatformAdapter` surface defaults to `not_supported` and is exercised by at least one default-behavior test (stub-adapter inheritance proof). +- No stray scaffolding: abandoned experiments, dead flags, or superseded helpers removed; `git grep "MockAdapter"` still matches only docs (or docs corrected). +- `CLAUDE.md` error-code list includes `APP_UNRESPONSIVE`; `docs/phases.md` reconciliation notes landed (U14); no other doc drift introduced. +- Follow-up issues filed (or a follow-up list appended to the gap-analysis notes) for every Deferred item this plan touches adjacent to. + +--- + +## System-Wide Impact + +- **Behavior change (deliberate):** ref-action failure paths slow to their wait budget by default (AE3); success paths unchanged. Batch inherits the same defaults through the shared dispatch path — batch entries may set `timeout_ms` per action. +- **Wire compatibility:** all new JSON fields are additive/optional; the single envelope bump is U14's new error code. Old refmaps load (serde defaults). +- **FFI ABI:** no existing function-signature changes; additive functions (`ad_execute_by_ref_timeout`, mouse additions) plus exactly two pinned-struct extensions — `AdRefEntry` (U5) and `AdActionStep` (U13) — each executed as the size-pin learning's 4-step sequence; `AdImageBuffer` is opaque so `scale_factor` (U3) is accessor-only. Header regenerated per affected unit via the maintainer script; CI drift gate proves it. +- **Phase 2/3 adapters:** implement against settled shapes — the point of the plan. `docs/phases.md`'s overlapping planned items (`identifier`, `AutomationPermissionDenied`, `AxMessagingTimeout`) are superseded by name here (U5, U4, U14). +- **Trace/redaction:** every new field audited: shapes (enums, bools, counts, ids) pass; content rides only under `SENSITIVE_KEYS`-covered keys (`name`, `title`). + +--- + +## Risks & Dependencies + +- **U8 blast radius (highest).** The poll loop wraps every ref action. Mitigations: type-layer default `None` keeps every existing constructor single-shot; the loop adds behavior only at the CLI/FFI arg layer; policy/post-verification untouched (test 9); e2e in both modes. +- **U0 mechanical churn.** ~55 test doubles re-split. Mitigation: compiler-driven, zero-logic edits; land first, alone. +- **U11 name drift.** Principled precedence may change computed names. Mitigation: characterization tests first; fixture diffs reviewed consciously. Residual (accepted): pre-upgrade refmaps may return `STALE_REF` after a mid-session binary upgrade until re-snapshot — one snapshot heals it. +- **`AXUIElementCopyElementAtPosition` fidelity (U9).** May return unexpected proxies on some apps. Mitigation: Unknown-on-weird-evidence keeps it advisory; integration tests on fixture app only assert the clear cases. +- **`AEDeterminePermissionToAutomateTarget` linkage (U4).** Carbon-era API via raw FFI. Mitigation: isolate in `permissions.rs` behind a probe fn; `Unknown` on any linkage/runtime failure. +- **Fixture app gaps.** Some integration cases (multi-select list, scroll area, `accessibilityIdentifier`) may need small SwiftUI fixture additions — allowed, they live under `tests/e2e/` fixture sources and follow its README. +- **rtk grep interception.** Verbatim code inspection during implementation must use `rtk proxy` or Read (scout-verified footgun). + +--- + +## Sources & Research + +- Gap analysis (requirements source): `agent-desktop-vs-playwright-gap-analysis.md` — session artifact produced 2026-07-02 by a 15-agent review, code-verified against `52705af`; its Tier-1 list + three defects define this plan's scope. This plan is self-contained: every load-bearing claim was re-verified in-repo by scouts on 2026-07-03. +- Scout-verified integration facts: `crates/core/src/adapter.rs` 397 LOC / 38 default-bodied methods; `ErrorCode` 15 variants (`error.rs:9-25`); `RefEntry` 16 fields (`refs.rs:22-50`); `is_check.rs` snapshot-state defect (`state_from_ref_entry`, lines 59-65) + dead `hidden` token (line 47); actionability = 6 ordered checks in `actionability/mod.rs:79-108`, Unknown never blocks, `check_live` runs once from `ref_action.rs:66`; `(pid,title)` sites `crates/macos/src/adapter.rs:51`, `system/window_ops.rs:26`, `system/app_ops.rs:90` (all via `tree::window_element_for`); both `capture_screen` impls shell `/usr/sbin/screencapture` (`screenshot.rs:46,211`); automation hardcoded `NotRequired` at all 5 sites (`permission_report.rs:51,68`, core `adapter.rs:214`, macOS `permissions.rs:63,74`); state producers `tree/builder.rs:152-176` + `actions/post_state.rs::element_state_from_attrs` (65-91); `AXScrollToVisible` at 5 sites, real invocation `ax_helpers::ensure_visible` via `CLICK_CHAIN.pre_scroll` (`chain.rs:42-46`); `ActionStep` = private `label` + `pub outcome` with `attempted/skipped/succeeded` constructors; `WAIT_RESOLVE_ATTEMPT` 750ms (`wait_element.rs:14`); two distinct 30s wait defaults (global `--wait-timeout` `src/cli/mod.rs:89-95`; `wait --timeout` `src/cli_args/system.rs:8`); chain env var + 10s default (`chain.rs:17,216-223`); `NotificationFingerprint` (`wait.rs:284-329`); wheel synthesis `input/mouse.rs::synthesize_scroll_at` (282-296); `RefArgs` (`cli_args/mod.rs:209-219`) used by 12 commands, payload actions inline their own ref fields; FFI: 75 exports, two families, codegen registry `crates/ffi/build.rs:49-68` + `EXPECTED_COMMANDS` drift gate, pinned `repr(C)` mirrors incl. `AdAction`/`AdRefEntry`/`AdActionStep`/`AdMouseEvent`/`AdWaitArgs` (3-layer pins, `tests/c_abi_layout.rs`), `AdImageBuffer` deliberately opaque (accessor-only), header via `scripts/update-ffi-header.sh` (cbindgen 0.29.4, banned from build graph per `deny.toml:21-23`); no `MockAdapter` exists — ~58 per-test doubles across 33 files, conformance via `src/tests/conformance.rs`. +- Binding institutional learnings (`docs/solutions/`): playwright-grade reliability contract; keep-FFI-action-policy-aligned-with-CLI; preserve-command-policy-semantics; envelope-version-bump contract; exhaustiveness-guards-over-catch-alls; identity-fingerprint-against-OS-reorder; keep-raw-arguments-out-of-trace-reachable-error-messages; deduplicate-ref-allocator-via-config-struct; macos-gesture-headless-capability; abort-state-guidance-multi-step-physical-input. +- External grounding (from the gap analysis, cited there): Playwright locator/auto-wait/assertion semantics; UIA ControlTypes/patterns/`AutomationId`; AT-SPI2 roles/interfaces/states and its hit-test gap. + diff --git a/docs/plans/2026-07-10-feat-gauntlet-evaluation-framework-plan.md b/docs/plans/2026-07-10-feat-gauntlet-evaluation-framework-plan.md new file mode 100644 index 0000000..c4a0a24 --- /dev/null +++ b/docs/plans/2026-07-10-feat-gauntlet-evaluation-framework-plan.md @@ -0,0 +1,489 @@ +# Gauntlet: An Open Evaluation Framework for Desktop Automation Tools + +**Status:** Proposed (v2 — restructured: standalone open-source framework; agent-desktop is the first driver) +**Repo:** `desktop-gauntlet` (separate repository, working name) +**First platform pack:** macOS · Windows/Linux packs follow the same contracts +**Author:** Lahfir · **Date:** 2026-07-10 + +--- + +## 1. What this is, in one page + +Gauntlet is a benchmark-and-evaluation harness for desktop automation tools — the +CLIs/servers that let agents observe and control real applications. It is **not** +tied to agent-desktop: it defines a small driver contract, ships its own fixture +apps and task corpus, and scores any tool that implements the contract. +agent-desktop is the first driver and the reason it exists. + +The whole framework is one machine with three slots: + +``` + TASK × POLICY × DRIVER + what to do who decides steps which tool executes + (corpus) (script | real LLM) (agent-desktop vX | other CLI) + │ + ▼ + runner executes task → policy → driver → real app + │ + ▼ + VERIFIER reads real state (files, AX tree, DOM) — never the tool's own ok:true + │ + ▼ + results.json → compare → scoreboard +``` + +Fix two slots, vary the third: + +| Vary | Question answered | Product | +|---|---|---| +| driver **version** (scripted policy) | did v0.4.8 regress vs v0.4.7? | tool reliability + regression engine | +| **policy** (model) | which LLM drives the tool best, at what cost, how consistently? | computer-use benchmark | +| **tool** | agent-desktop vs any other automation CLI | open leaderboard | + +```mermaid +flowchart LR + subgraph corpus["Task corpus (versioned data)"] + C1[conformance cases] + C2[grounding triples] + C3[multi-step scenarios] + C4[chaos scripts] + end + subgraph runner["gauntlet runner"] + P["policy\n(scripted | llm:claude | llm:gpt | ...)"] + D["driver\n(tool under test)"] + P --> D + end + corpus --> runner + D -->|CLI subprocess| T["desktop tool\n(agent-desktop vX, ...)"] + T -->|OS accessibility| A["fixture apps + real apps"] + A -.->|independent read| O["oracles\nfs · AX · CDP · inventory"] + O --> V{verifier} + V --> R[(results.json)] + R --> CMP["gauntlet compare (A vs B)"] + R --> REP["scoreboard / release report"] +``` + +**What Gauntlet is NOT responsible for:** the tool's own repo hygiene. Lint rules, +file-size limits, unit-test runners, mutation testing, JSON-schema snapshot gates — +those live in each tool's own CI (agent-desktop's are listed in §11 as a companion +work stream, because they matter, but they are not the framework). Gauntlet only +measures what it can observe through a tool's public surface. It also never uses +LLM judges for scoring (every mature benchmark that started with them ripped them +out) and never uses pixel-diff assertions (accessibility is the contract; +screenshots are evidence, not oracles). + +--- + +## 2. Why (the problem, from the agent-desktop trenches) + +- agent-desktop's merge gates run ~1,700 mock-based unit tests. Its excellent + fail-closed e2e suite requires macOS Accessibility permission, which CI runners + don't have — so **zero real-AX tests gate a merge**. Three total-breakage + regressions shipped green because of exactly this + (`docs/solutions/real-app-tests-are-the-platform-adapter-gate.md`). +- Two independent audits couldn't agree which commands have e2e coverage (~31 vs + ~35 of 58) — coverage isn't tracked mechanically. +- One latency assertion exists in the whole system; rich per-phase timing is + emitted to traces but never aggregated or gated. +- No flake accounting, no repeated-trial scoring, no multi-step task corpus, no + general cross-version comparison. + +Generalize the pain: **every desktop automation tool has this problem** — mocks +lie, real-AX CI is hard (permissions), and there is no shared benchmark for +"is this tool reliable enough for an agent to depend on." That's the gap Gauntlet +fills, and why it's worth open-sourcing rather than burying in one repo's tests. + +--- + +## 3. Repo split + +```mermaid +flowchart TB + subgraph OG["desktop-gauntlet (open source)"] + RN["runner · results schema · compare · report"] + CP["corpus/ (versioned task data)"] + FX["fixtures/ (SwiftUI · AppKit · Electron apps)"] + DRV["drivers/ (agent-desktop, ...)"] + POL["policies/ (scripted, llm-anthropic, llm-openai, ...)"] + INF["infra/ (Tart VM image recipe with pre-seeded AX permission)"] + end + subgraph AD["agent-desktop repo (the tool)"] + SRC["tool source + unit tests"] + CI2["its own CI: lint · unit · mutation · schema gates (§11)"] + end + CI2 -->|"invokes gauntlet as a required check"| RN + DRV -->|subprocess, JSON| SRC + OTHER["any other tool"] -.->|"implements driver contract"| DRV +``` + +- `desktop-gauntlet` owns everything reusable: runner, schemas, corpus, fixtures, + drivers, policies, the VM/permission recipe, the leaderboard. +- `agent-desktop` keeps its own repo CI and its local e2e acceptance suite (which + seeded many of Gauntlet's ideas), and adds one thin CI job: *run Gauntlet's + macOS pack against the freshly built binary; fail the PR on gate breaches.* +- Fixtures: the existing `AgentDeskFixture` SwiftUI app migrates into + `desktop-gauntlet/fixtures/swiftui/` (it was designed tool-agnostically — + "never tuned to make a command pass" — which is exactly a benchmark fixture). + +--- + +## 4. The components + +### 4.1 Driver contract + +A driver adapts one tool to three verbs. For CLI tools it's a manifest plus a +subprocess convention (no linking required — any language works): + +```yaml +# drivers/agent-desktop/driver.yaml +tool: agent-desktop +binary: ${GAUNTLET_TOOL_BINARY} +capabilities: [snapshot, click, type, scroll, drag, clipboard, windows, wait, batch] +verbs: + snapshot: ["snapshot", "--app", "{app}", "-i"] # → JSON on stdout + act: ["{action}", "{ref}", "--snapshot", "{snap}"] # → JSON envelope + probe: ["get", "{ref}", "--property", "{prop}"] # tool-side reads (weak oracle) +contract: + error_codes: [STALE_REF, AMBIGUOUS_TARGET, TIMEOUT, PERM_DENIED, ...] + envelope: json # every response parseable +``` + +The runner treats drivers as untrusted subprocesses: private hashed binary copy, +isolated HOME/state dir, own process group, absolute watchdog timeout, bounded +output capture — inherited from agent-desktop's `run.sh` harness-integrity work. +A tool without some capability skips those cases (recorded as `unsupported`, which +is itself a scoreboard column — coverage of the action vocabulary is a comparison +axis between tools). + +### 4.2 Policies (who decides the next step) + +- **`scripted`** — deterministic step lists from the corpus. No model anywhere. + This is the *only* policy used for reliability gates and version comparison. +- **`llm:<provider/model>`** — a real model (Claude, GPT, Gemini, local) receives + the task instruction and drives the tool through the driver, exactly like a + production agent would. Used for the computer-use benchmark tier: same tasks, + same tool, different models — or same model, two tool versions ("did the new + snapshot format make the model better at the task?"). Results carry cost, + tokens, steps, wall time, and pass^k, HAL-style (cost and consistency are + first-class columns, accuracy alone is a vanity metric). +- **`adversarial`** — a scripted policy that deliberately misuses the tool (stale + refs, cross-namespace refs, malformed batches, rapid-fire duplicates, concurrent + sessions). The contract under abuse: typed error from the documented set, + parseable envelope, no state corruption, bounded exit. This is "any model can + use it" made testable: the contract must hold even for the worst caller. + +Verification is identical for all policies — the verifier never knows or cares who +decided the steps. + +### 4.3 One case, end to end + +```mermaid +sequenceDiagram + participant R as runner + participant P as policy + participant D as driver + participant App as fixture/real app + participant O as oracle (independent) + R->>App: reset to pinned initial state + loop until done / budget exhausted + P->>D: next step (snapshot / click @e12 / type ...) + D->>App: execute via OS accessibility + D-->>P: JSON envelope (result | typed error) + end + R->>O: read real state (fs · AX · CDP · AppleScript · inventory hook) + O-->>R: observed state + R->>R: verdict + invariants (no hang · no crash · no wrong-target · store intact) + R->>R: append case → results.json (k trials → pass^k) +``` + +Oracles come in two strengths, stated honestly: **strong** oracles bypass the tool +entirely (file system, AppleScript app state, CDP DOM/AX, fixture inventory hook); +**weak** oracles read UI state through the tool itself but against an element +disjoint from the action target. Scenario verifiers prefer strong oracles; weak +ones are allowed for effect checks and are cross-audited by the fidelity suite. +When a verifier is found wrong, we fix the verifier and keep the task (score +continuity — the OSWorld-Verified rule), and we log every verifier change with its +measured false-negative impact (the WebArena-Verified meta-metric: they found +broken checkers in 299/812 tasks; auditing your own verifiers is not optional). + +### 4.4 Fixture apps (ground truth you own) + +Real third-party apps update themselves; fixtures don't — they're the +deterministic core, and each one **exports its own ground truth** so snapshots can +be scored for completeness (recall/precision), not just spot-checked. + +- **SwiftUI** (exists — migrated): buttons/toggles/sliders/pickers, ambiguous + twins, zero-bounds controls, delayed-enable, duplicate-titled windows, sheets/ + popovers/menus, drag canvas, removable rows. Added: an animated control + (stability gate), a churn card (timed subtree mutations at configurable Hz), a + static `inventory.json` manifest. +- **AppKit-classic** (new, small): NSTableView/NSOutlineView/NSToolbar/NSAlert — + what Finder-class apps are actually made of. +- **Electron** (new — the dense-tree gauntlet): eight pages, each aimed at a + researched failure mode — parameterized dense grid (1k–20k DOM nodes), + virtualized list **with a non-virtualized twin of the same data** (off-viewport + rows are structurally *absent* from every AX channel, so single-snapshot + "completeness" claims are only honest against the rendered inventory; full-data + completeness is a scroll-loop scenario), AG Grid with virtualization toggle, + Monaco (10k lines, accessibility mode on/off — two different code paths), canvas + regions (with/without fallback DOM — must report "no actionable children," not + junk), nested cross-origin iframes (tree-stitching races), a live-region ticker + (5–20 Hz churn), and a frameless custom-chrome window. Every page ships + `window.__gauntletInventory__` and exposes CDP for oracle reads. +- **Real-app pack** (nightly, informational): Finder, TextEdit, System Settings, + Safari, Chrome, VS Code, Slack — because hostile, non-conformant AX trees only + exist in the wild (UFO2 attributed 62% of its failures to control detection on + exactly such apps). + +### 4.5 Results schema and comparison + +Every run, any tier, any policy, emits one `gauntlet-result/v1` JSON: binary +identity (sha256/version/git), host class, **pinned corpus version**, per-case +attempts with wall time, phase spans (resolve/preflight/dispatch/verify), error +codes, outcome (`pass|fail|flaky|quarantined|unsupported`), invariant violations, +and aggregate metrics. Schema rules: additive-only evolution, hard version pinning +(a score is meaningless without the exact corpus that produced it — +terminal-bench's rule), committed JSON Schema. + +`gauntlet compare a.json b.json` → per-case outcome transitions, metric deltas +with noise margins, phase-span deltas ("resolve p95 +40% on dense trees" is one +line). `gauntlet run --binary A --binary B` runs balanced AB/BA pairs against one +pinned app state — the methodology already proven in agent-desktop's +`electron-live.sh` (interleaved ordering, separate HOMEs, immutable hashes, state +gate before/after every pair, paired diffs), promoted from one operation to the +whole corpus. + +**Metric definitions (fixed):** +- `pass^k` — per case, n trials, c successes: `C(c,k)/C(n,k)` (tau-bench's + estimator). The metric that exposes "works once, fails on retry" — models drop + from ~50% pass@1 to ~25% pass^8; a *tool* must hold ≥0.99 pass^5 on scenarios + and 1.0 on conformance. +- `flake_rate` — fail-then-pass is recorded `flaky` forever (Playwright/nextest + semantics); trailing budget <0.5%, breach → quarantine lane (cases keep running + and reporting, gate nothing, carry an issue URL, expire after 4 weeks). +- **Zero-tolerance invariants** — wrong-target actions (effect landed on an + element other than the addressed one — fixtures make this observable), crashes, + watchdog hangs, state-store corruption, secrets in traces: any occurrence fails + the run outright. +- `determinism_rate` — identical fixture state, two headless snapshots → identical + normalized tree and ref assignment. Must be 1.0. +- Fidelity — recall (oracle-interactive elements present with refs — a miss means + an agent literally cannot act), precision, hierarchy edge-F1, attribute accuracy + (role/name/value/bounds). Goes beyond Screen2AX's structure-only F1. +- Latency — nearest-rank p50/p95 per command × fixture class × mode; plus token + economy (snapshot bytes / est. tokens per fixture class — an agent-facing cost). +- `stale_recovery_rate` — churn → STALE_REF → re-snapshot+retry succeeds ≤1 cycle. + +--- + +## 5. The six suites + +| Suite | Question | Gates? | +|---|---|---| +| **Conformance** | does every documented command do exactly what it claims, under actionability rules, with typed errors — on real AX? | yes — 100%, per-PR (smoke) / merge (full) | +| **Fidelity** | is the snapshot complete and correct vs ground truth? | yes on fixtures (recall = 1.0); informational on real apps | +| **Grounding** | given a description, does the tool resolve the *right* element — or correctly refuse? | yes | +| **Scenarios** | do multi-step, real-workflow tasks succeed repeatedly (pass^k)? | nightly gate | +| **Chaos** | under perturbation, does it degrade into typed errors — never hangs/crashes/wrong clicks? | nightly gate, zero-tolerance invariants | +| **Performance** | latency/scaling/cost curves, version over version | thresholds vs pinned baselines | + +Condensed specs (full case inventories live in the corpus itself): + +- **Conformance** — per command × fixture pattern × headless/headed: effect + verified by observation exactly-once; per-check actionability cases (disabled → + typed timeout with last-report; moving → held until stable; occluded → occluder + named); an error-taxonomy case per documented code; determinism and recovery + cases. Generalizes agent-desktop's AE1–AE7 to the whole surface. A mechanical + coverage matrix (command × fixture × mode) is a committed artifact; empty rows + fail. Chromium-specific contract: correct AX enablement (`AXManualAccessibility` + for Electron), full-tree mode confirmed, post-mutation settle honored, and the + notification-incompleteness rule (Chromium's macOS AX event map is explicitly + incomplete — a node deleted with no notification must still vanish from the next + snapshot; incremental invalidation may only ever be an optimization over full + re-traversal). +- **Grounding** — ~150 labeled triples to start (every fixture control + surface + elements), scored exact-match, including must-refuse rows (AMBIGUOUS_TARGET on + twins; not-found on absent). ScreenSpot's design, plus the refusal classes it + lacks. +- **Scenarios** — 30–40 declarative YAML tasks v1 (fixture flows, TextEdit + save-to-disk with fs oracle, Finder folder ops, Settings toggle with `defaults + read` oracle, Electron form-fill with DOM oracle, clipboard round-trips, + notification post+dismiss). Scripted policy for gates; the same files run under + LLM policies for the benchmark tier. +- **Chaos** — injectors: window-move mid-action, focus steal, occluder appears, + element removed between preflight and dispatch, app SIGSTOP (→ typed + APP_UNRESPONSIVE, never a hang), app SIGKILL, 20 Hz churn storm, dense-tree + pressure, permission revoked mid-run, store fault injection, concurrent + invocations racing one app. +- **Performance** — dense scaling curves (the curve is the artifact — slope + regressions get caught even when small-N points pass); per-command wall-clock on + real Macs (hyperfine methodology: warmups, no shell, JSON export); phase-span + attribution from driver `--profile` output where the tool provides it; strategy + matrix — when a tool exposes alternative strategies behind flags, Gauntlet runs + the corpus per variant and ranks them. That's the "find the faster method" + requirement: land alternatives behind flags, let the harness pick the winner, + then make it the default. Calibration note: VoiceOver itself stalls 40–60s on + Gmail-class pages — dense-tree budgets are set from measured baselines, not + vibes. (Deterministic instruction-count gating of a tool's platform-independent + core is a *tool-repo* concern, §11 — Valgrind doesn't run on macOS.) + +--- + +## 6. Infrastructure: real AX permission in CI + +The blocker that keeps every tool's real tests local is macOS TCC (Accessibility +permission cannot be granted non-interactively through any supported API). The +production-proven answer, shipped as `infra/` in the framework repo: + +- **Tart golden VM image** — build on Cirrus Labs' own published recipe (their + production image pipeline scripts a SIP-disable inside the VM via recovery-mode + automation, then seeds `TCC.db` directly, then snapshots; now free after the + OpenAI relicense). Gauntlet's image adds a TCC row for a **fixed staging path** + (`/opt/gauntlet/bin/<tool>` — path-keyed grants survive binary changes) plus the + fixture apps. `tart clone` per CI job = clean state + permission, in seconds, on + any Apple-Silicon self-hosted box. +- Fallbacks, documented in order of decreasing solidity: persistent self-hosted + Mac / EC2 Mac dedicated host with a one-time manual grant; the user-level + `TCC.db` sqlite insert on GitHub-hosted runners (works as of 2026-04, unofficial + and revocable — stopgap only). + +Suggested cadence for a tool consuming Gauntlet (agent-desktop's plan): + +| Lane | When | What | +|---|---|---| +| PR (required) | every push | conformance smoke (headless, ~⅓ cases, 1 trial) on a Tart clone — ≤10 min | +| PR (label) | `gauntlet-full` | full conformance + fidelity + grounding | +| merge | main | full conformance both modes + scenarios (2 trials) + perf representative set | +| nightly | cron | scenarios pass^5 + chaos + perf curves + real-app pack + LLM-policy probe | +| release | tag | everything, macOS N/N-1 matrix + published reliability report | + +--- + +## 7. Meta-evaluation: proving the framework catches bugs + +A framework that can't catch known bugs has no business gating anything. + +- **Seeded-defect corpus** (mechanism generic, seeds per tool): `gauntlet + seed-defects` reverts a known-bug fix in a worktree, rebuilds, runs the mapped + suite, and asserts Gauntlet **fails**. agent-desktop's seeds come from its own + history — the mock-concealed regressions, the window-id fix, accessible-name + unification, the Instant-overflow panic. Detection rate is tracked per release; + target 100%. Every new production bug adds a seed before its fix merges. +- **Verifier audit log** — every verifier change records measured false-negative/ + false-positive impact, published in the release report. + +--- + +## 8. The open-source products + +1. **The conformance suite** — WPT-for-desktop-automation: any tool implements the + driver contract and self-certifies against the pinned corpus. +2. **The computer-use benchmark** — LLM policies × scenario corpus, with cost, + steps, and pass^k as first-class columns; results reproducible from the + results.json + pinned corpus + model version. +3. **The leaderboard** — terminal-bench mechanics: submissions are PRs of raw + per-trial results, a bot validates schema/structure, a human merges; every row + hard-pinned to corpus + tool + model versions. No self-reported unverifiable + numbers. +4. **The reliability report** — regenerated per release of a tool: case counts, + pass^k, flake trend, chaos survival, fidelity, perf deltas, seeded-defect + detection rate. (Playwright publishes no numbers and has no perf CI at all — + this artifact is the differentiator.) + +--- + +## 9. Build plan (honest sizing — this is phased work, not one PR) + +| Phase | Deliverable | Rough effort | +|---|---|---| +| **P0 — Scaffold** | `desktop-gauntlet` repo: runner skeleton, driver contract + agent-desktop driver, results schema + compare, SwiftUI fixture migrated, ~20 conformance cases running locally | 1–2 weeks | +| **P1 — Reality gate** | Tart image + self-hosted runner; agent-desktop PR lane goes required; conformance corpus covers all 58 commands; coverage matrix + failure bundles | 1–2 weeks | +| **P2 — Chromium depth** | Electron fixture (8 pages), fidelity suite with triangulated oracles (inventory + CDP + ariaSnapshot), dense perf curves, grounding corpus | ~2 weeks | +| **P3 — Brutality** | Scenario corpus (30–40 tasks), chaos injectors + invariant watchdog, adversarial policy, pass^k nightly, seeded-defect corpus, flake dashboard (a DuckDB file + static HTML — no SaaS) | ~2 weeks | +| **P4 — Publish** | LLM policies (Anthropic/OpenAI/local), leaderboard bot, docs site, first public reliability report, Windows/Linux pack scaffolds | 2+ weeks, ongoing | + +Each phase ships value on its own; P1 alone kills the "merged green, broke real +apps" failure mode. Phases get their own implementation specs when started — this +document is the architecture, not the work items. + +**Companion stream (agent-desktop repo, independent, days not weeks):** §11. + +--- + +## 10. What Gauntlet deliberately excludes + +- LLM judges in any gating tier (non-deterministic; the failure mode every + "Verified" benchmark overhaul existed to remove). +- Pixel-diff assertions (AX is the contract; screenshots are evidence). +- The tool's internal test infrastructure (unit tests, mutation, lint — §11). +- Hosting third-party model leaderboards before the tool-reliability core is + credible (products in §8 ship in order). + +--- + +## 11. Companion stream: agent-desktop's own CI hardening (NOT the framework) + +These came out of the same research and are worth doing regardless of Gauntlet — +but they are agent-desktop repo concerns, listed here so they don't get lost and +don't get confused with the framework again: + +- **JSON envelope schema gate** — schemars-derived schema for the envelope + all + 58 payloads, committed, insta-snapshot-gated; golden fixtures validated against + it (`jsonschema` crate). Additive-only evolution (Stripe/Zalando rules). +- **Mutation testing** — `cargo-mutants --in-diff` per PR (cost scales with the + diff); sharded full sweeps weekly. Directly quantifies "tests pass but catch + nothing" (~61k non-test LOC ⇒ full runs are hours; shard them). +- **nextest** as unit runner — process-per-test isolation for the AX/FFI blast + radius; `flaky-result="fail"` in PR profile. +- **proptest-state-machine** ref-model suite — `@ref → (pid, role, path, + stable_text, bounds_hash)` transitions asserting strict re-identification + semantics (the icechunk pattern). +- **`--profile` flag** — surface the already-collected `query_stats`/phase spans + in the response envelope; Gauntlet's phase attribution consumes it. +- **Doc-drift fixes** — CLAUDE.md's false CI claim, the four dead + `AGENT_DESKTOP_*_TIMEOUT_MS` env knobs, the stale fixtures README. +- Existing gates (400-LOC, clippy, core-isolation, binary size, FFI drift/panic) + stay exactly where they are: repo CI. + +--- + +## 12. Decision log (research verdicts, framework-relevant) + +| Decision | Verdict | Basis | +|---|---|---| +| Execution-based verifiers only; no LLM judges | ADOPT | OSWorld-Verified, WebArena-Verified (299/812 broken checkers fixed via backend state), AndroidWorld | +| Fix evaluators not tasks; semver-pinned corpus | ADOPT | OSWorld-Verified score continuity; terminal-bench pinning | +| pass^k (hypergeometric) + pass@1 | ADOPT | tau-bench; computer-use reliability follow-ups | +| Flaky as first-class outcome; quarantine with issue URLs | ADOPT | Playwright outcome semantics + fixme convention | +| Fixture self-inventory + CDP + ariaSnapshot oracle triangulation | ADOPT | each channel's blind spots documented; CDP tree is an unpruned superset; virtualized rows absent from all AX channels | +| Fidelity = recall/precision + edge-F1 + attributes | ADOPT (extended) | Screen2AX is structure-only | +| Balanced AB/BA paired comparison | ADOPT | agent-desktop `electron-live.sh` methodology | +| Tart golden image for TCC | ADOPT | Cirrus Labs' production recipe; free post-relicense | +| Hosted-runner TCC.db hack | STOPGAP | works 2026-04; unofficial, revocable | +| Cost/steps/consistency as first-class model-benchmark columns | ADOPT | HAL; "AI Agents That Matter" cost-controlled evaluation | +| PR-based leaderboard with bot validation | ADOPT | terminal-bench | +| Zero perf story in Playwright ⇒ perf tier is our differentiator | ADOPT | verified: no benchmark code/CI in microsoft/playwright | + +## 13. Source index (primary sources) + +Playwright: playwright.dev docs (actionability, writing-tests, test-retries, +aria-snapshots, trace-viewer); microsoft/playwright source (injectedScript.ts, +dom.ts/frames.ts/progress.ts, packages/trace/src, tests/, workflows incl. +fix-flakes.yml, browser_patches). Benchmarks: OSWorld arXiv:2404.07972 + +OSWorld-Verified (xlang.ai); WindowsAgentArena arXiv:2409.08264; macOSWorld +arXiv:2506.04135; AndroidWorld arXiv:2405.14573; WebArena + evaluators.py + +WebArena-Verified (OpenReview); VisualWebArena arXiv:2401.13649; UFO2 +arXiv:2504.14603; τ-bench arXiv:2406.12045 + τ²-bench arXiv:2506.07982 (pass^k +estimator, code-verified); terminal-bench/Harbor arXiv:2601.11868; ScreenSpot +family (arXiv:2401.10935, 2410.23218, 2504.07981); Screen2AX arXiv:2507.16704; +HAL arXiv:2510.11977; "AI Agents That Matter" arXiv:2407.01502; computer-use +reliability arXiv:2604.17849. Chromium/macOS AX: chromium.googlesource.com +accessibility docs (overview, how-a11y-works, offscreen, performance), +ax_platform_node_mac.mm event map, Electron a11y docs, CDP Accessibility domain, +WebKit AXIsolatedTree, VoiceOver perf threads, Lighthouse DOM-size guidance. +Tooling: mutants.rs, nexte.st, insta.rs, schemars/jsonschema, gungraun, hyperfine +source, criterion FAQ, rustc-perf, ripgrep tests/, git t/README + chainlint, +cargo-test-support. CI/TCC: actions/runner-images issues #1567/#3286/#8162/#8214/ +#9529, fopina/pyautogui-next#20, Apple PPPC guide, tccutil man page, +cirruslabs/macos-image-templates (disable-sip.pkr.hcl, update-tcc-database.sh, +monthly.yml), openai/tart discussion #1171, Veertu Anka docs, AWS EC2 Mac docs.