Creepso/agent-desktop
1
0
Fork 0
mirror of https://github.com/lahfir/agent-desktop.git synced 2026-07-26 17:12:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat!: implement Playwright-grade foundation contract

Browse source 
Settle the Playwright-grade reliability contract in agent-desktop-core
before the Windows/Linux adapters are built, so they inherit it instead
of redesigning it. Every command now observes, waits, verifies, and
reports honestly instead of firing blindly.

Highlights: capability-supertrait split of PlatformAdapter with
not_supported() defaults; canonical role/state vocabulary with live
`is --property visible`; display enumeration (`list-displays`) and honest
`--screen` with scale factor; truthful Automation permission; `native_id`
identity spine; window-id-first resolution; serializable `LocatorQuery`
with live `find`; default-on auto-wait before every ref action; three-way
`hit_test` occlusion gate; `scroll_into_view` in core; core accessible-name
precedence; typed `ActionStep` delivery tier; `ProcessState` and
`APP_UNRESPONSIVE`; `LaunchOptions`; baseline-diff desktop signals
(`wait --event`); typed clipboard (`Text`/`Image`/`FileUrls`); mouse
modifier chords and `mouse-wheel`. Hardened through a 35-reviewer pass with
independent validation and a green live e2e gate (109/0), plus a
head-vs-main performance comparison harness.

BREAKING CHANGE: default-on auto-wait changes the timing of every
previously-untouched ref-action call (bounded 5000 ms default; `--timeout-ms 0`
restores single-shot). `ENVELOPE_VERSION` is now `2.1` (adds the
`APP_UNRESPONSIVE` code and process state in error details). FFI ABI major
is `3` (append-only struct evolution; `wait --event` is intentionally not
exposed over FFI). The legacy string clipboard API is removed in favor of
typed content. `key-down`/`key-up` fail closed until daemon-owned held input
exists. `close-app` verifies termination and the osascript fallback path is
removed. `--text` matching is subtree containment: `find --text X --first`
returns the outermost matching container.
This commit is contained in:
Lahfir 2026-07-20 00:21:38 -07:00 committed by GitHub
parent 52705afbe1
commit 3f322728b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
996 changed files with 87819 additions and 23649 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
.githooks/pre-commit
View file

@ -51,54 +51,16 @@ run() {
fi
}
run "inline comment ban" bash -c '
mapfile -t files < <(git diff --cached --name-only --diff-filter=ACMR -- "*.rs")
if ((${#files[@]} == 0)); then
exit 0
fi
if grep -En "^\s*//[^/!]" "${files[@]}" 2>/dev/null; then
echo "" >&2
echo "pre-commit: inline // comments are forbidden; use /// or //! docstrings only." >&2
exit 1
fi
if grep -En "[^:\"'\''\`]\s+//[^/!]" "${files[@]}" 2>/dev/null; then
echo "" >&2
echo "pre-commit: end-of-line // comments are forbidden; use /// or //! docstrings only." >&2
exit 1
fi
'
run "Rust source rules" scripts/check-rust-file-size.sh
run "cargo fmt --all -- --check" cargo fmt --all -- --check
run "cargo clippy --all-targets -- -D warnings" cargo clippy --all-targets -- -D warnings
run "cargo test --lib --workspace" cargo test --lib --workspace
run "cargo test --lib --workspace" scripts/cargo-test-isolated-home.sh test --lib --workspace
# ── FFI-specific gates ────────────────────────────────────────────────────────
# Mirrors CI jobs: ffi-codegen-drift, ffi-passthrough, ffi-header-drift.
# Fast-skipped in full when no crates/ffi/** files are staged.
#
# Scope: only the fast structural gates run locally (codegen drift, stub
# passthrough, optional header drift). panic_spike (requires a release-ffi
# example build) and the Python smoke harness (requires a full cdylib build
# plus a Python runtime) are CI-only — jobs ffi-panic-guard and
# ffi-python-smoke — to keep commits fast.
# FFI-specific gates are skipped when no FFI files are staged. Runtime panic
# and Python smoke proofs remain CI-only to keep commits fast.
if [ "$HAS_FFI_CHANGES" -eq 1 ]; then
# Gate 1: Codegen drift — build triggers build.rs which regenerates
# generated.rs in-source; diff detects a stale staged copy.
run "FFI codegen build" cargo build --locked -p agent-desktop-ffi --features stub-adapter
if ! git diff --quiet crates/ffi/src/commands/generated.rs; then
printf '\033[1;31m✗ FFI codegen drift\033[0m\n' >&2
printf '\n' >&2
printf 'pre-commit: crates/ffi/src/commands/generated.rs is stale.\n' >&2
printf ' The hook already regenerated it — just stage the updated file:\n' >&2
printf ' git add crates/ffi/src/commands/generated.rs\n' >&2
printf '\npre-commit: refusing the commit. Fix the issues above or rerun with SKIP_PRECOMMIT=1 to bypass.\n' >&2
exit 1
fi
# Gate 2: Stub-adapter passthrough tests.
run "FFI stub passthrough" cargo test --locked -p agent-desktop-ffi --features stub-adapter --test c_abi_passthrough
# Gate 3: Header drift — graceful skip if cbindgen is absent or not 0.29.4.
# The hard gate is CI job ffi-header-drift; local absence is not a commit blocker.
export PATH="${HOME}/.cargo/bin:${PATH}"
_cbindgen_cmd=""
if command -v cbindgen >/dev/null 2>&1; then

3
.github/actionlint.yaml vendored Normal file
View file

@ -0,0 +1,3 @@
self-hosted-runner:
labels:
- agent-desktop-e2e

200
.github/workflows/ci.yml vendored
View file

@ -29,6 +29,51 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- run: rustup component add rustfmt
- run: cargo fmt --all -- --check
- name: Check E2E scripts without running native automation
run: |
scripts/check-bash3-compat.sh
shellcheck -x -e SC2034,SC2154 tests/e2e/run.sh tests/e2e/electron-live.sh tests/e2e/permission-contract.sh tests/e2e/guard-command.sh scripts/*.sh .githooks/pre-commit
PYTHONPYCACHEPREFIX=/tmp/agent-desktop-pycache python3 -m py_compile tests/e2e/*.py
PYTHONPYCACHEPREFIX=/tmp/agent-desktop-pycache python3 -m unittest discover -s tests/e2e -p 'test_*.py'
- name: Validate GitHub Actions workflows
run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.11
msrv:
name: Rust 1.89 MSRV
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- run: rustup toolchain install 1.89.0 --profile minimal
- run: cargo +1.89.0 check --locked -p agent-desktop-core -p agent-desktop-linux -p agent-desktop --all-targets
platform-check:
name: Native check (${{ matrix.platform }})
runs-on: ${{ matrix.os }}
timeout-minutes: 25
permissions:
contents: read
strategy:
fail-fast: false
matrix:
include:
- platform: Linux
os: ubuntu-latest
package: agent-desktop-linux
- platform: Windows
os: windows-latest
package: agent-desktop-windows
- platform: macOS
os: macos-latest
package: agent-desktop-macos
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Install pinned Rust toolchain
run: rustup show
- name: Check native adapter and binary
run: cargo check --locked -p agent-desktop-core -p ${{ matrix.package }} -p agent-desktop --all-targets
test:
name: Test
@ -40,7 +85,12 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Install Rust toolchain
run: rustup show
run: |
rustup show
rustup component add clippy
- name: Verify macOS Bash 3.2 compatibility
run: /bin/bash scripts/check-bash3-compat.sh
- name: Cache cargo registry
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@ -63,7 +113,8 @@ jobs:
- name: Check dependency isolation
run: |
if cargo tree --locked -p agent-desktop-core 2>/dev/null | grep -E 'agent-desktop-(macos|windows|linux)'; then
TREE=$(cargo tree --locked -p agent-desktop-core --edges normal,build)
if printf '%s\n' "$TREE" | grep -E 'agent-desktop-(macos|windows|linux)'; then
echo "FAIL: core crate depends on platform crates"
exit 1
fi
@ -72,68 +123,77 @@ jobs:
- name: Check release metadata consistency
run: scripts/check-release-consistency.sh
- name: Clippy
run: cargo clippy --locked --all-targets -- -D warnings
- name: Enforce Rust source rules
run: scripts/check-rust-file-size.sh
- name: Unit tests
run: cargo test --locked --lib --workspace
- name: Clippy
run: cargo clippy --locked -p agent-desktop-core -p agent-desktop-macos -p agent-desktop -p agent-desktop-ffi --all-targets -- -D warnings
- name: Core and macOS unit tests
run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-core -p agent-desktop-macos --lib
- name: Locator benchmark adapter contract
run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-core --example locator_benchmark
- name: Deterministic permission acceptance contract
run: bash tests/e2e/permission-contract.sh
- name: Binary command tests
run: cargo test --locked -p agent-desktop
run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop
# The FFI crate ships integration harnesses under crates/ffi/tests/ that
# exercise the C-ABI from outside the crate (raw extern "C" decls, enum
# fuzzing, out-param zeroing, last-error lifetimes). The workspace --lib
# run skips them, so wire them in explicitly so regressions fail PR CI.
- name: FFI integration tests
run: cargo test --locked -p agent-desktop-ffi --tests
run: scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi --tests
# Use the ci profile (no LTO, opt-level 1) — fast compile, still checks the binary builds.
- name: Build binary (ci profile)
run: cargo build --locked --profile ci
- name: Check binary size
- name: Build stripped release binary
run: |
SIZE=$(stat -f%z target/ci/agent-desktop)
LIMIT=$((15 * 1024 * 1024))
echo "Binary size: $(du -sh target/ci/agent-desktop | cut -f1)"
if [ "$SIZE" -gt "$LIMIT" ]; then
echo "FAIL: binary exceeds 15MB limit (${SIZE} bytes)"
cargo build --locked --release -p agent-desktop
cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper
- name: Verify shipped binary version flag
run: |
PACKAGE_ID=$(cargo pkgid -p agent-desktop)
EXPECTED_VERSION=${PACKAGE_ID##*@}
EXPECTED_OUTPUT="agent-desktop $EXPECTED_VERSION"
ACTUAL_OUTPUT=$(target/release/agent-desktop --version)
if [ "$ACTUAL_OUTPUT" != "$EXPECTED_OUTPUT" ]; then
echo "FAIL: expected '$EXPECTED_OUTPUT', got '$ACTUAL_OUTPUT'"
exit 1
fi
echo "OK: binary within 15MB limit"
- name: Check shipped binary size
run: |
SIZE=$(stat -f%z target/release/agent-desktop)
HELPER_SIZE=$(stat -f%z target/release/agent-desktop-macos-helper)
LIMIT=$((15 * 1024 * 1024))
echo "Binary size: $(du -sh target/release/agent-desktop | cut -f1)"
echo "Helper size: $(du -sh target/release/agent-desktop-macos-helper | cut -f1)"
if [ "$SIZE" -gt "$LIMIT" ] || [ "$HELPER_SIZE" -gt "$LIMIT" ]; then
echo "FAIL: shipped executable exceeds 15MB limit (binary=${SIZE}, helper=${HELPER_SIZE} bytes)"
exit 1
fi
echo "OK: shipped executables are within the 15MB per-file limit"
- name: FFI cdylib build (release-ffi profile)
run: cargo build --locked --profile release-ffi -p agent-desktop-ffi
- name: FFI dylib-adjacent helper discovery smoke
run: scripts/ci-ffi-helper-discovery-smoke.sh
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: '24'
- name: NPM package contents
run: node scripts/check-npm-package.js
run: |
node --test tests/npm/*.test.js
node scripts/check-npm-package.js
- name: NPM wrapper smoke
run: |
case "$(uname -s)-$(uname -m)" in
Darwin-arm64) NAME=agent-desktop-darwin-arm64 ;;
Darwin-x86_64) NAME=agent-desktop-darwin-x64 ;;
*)
echo "Unsupported smoke-test platform: $(uname -s)-$(uname -m)"
exit 1
;;
esac
trap 'rm -f "npm/bin/${NAME}"' EXIT
cp target/ci/agent-desktop "npm/bin/${NAME}"
chmod +x "npm/bin/${NAME}"
node npm/bin/agent-desktop.js version > /tmp/agent-desktop-version.json
node -e "
const out = require('fs').readFileSync('/tmp/agent-desktop-version.json', 'utf8');
const json = JSON.parse(out);
if (json.ok !== true || !json.data || typeof json.data.version !== 'string') {
throw new Error('agent-desktop npm wrapper did not return version JSON');
}
"
run: scripts/ci-npm-wrapper-smoke.sh
ffi-python-smoke:
name: FFI Python Smoke
@ -232,11 +292,9 @@ jobs:
# This job proves the contract in two steps:
# 1. Assert the Cargo.toml profile setting directly (simpler and more
# reliable than a build.rs env check for a dual cdylib+rlib crate).
# 2. Build and run crates/ffi/examples/panic_spike under the release-ffi
# profile: an extern-C fn panics, catch_unwind catches it, and the
# example prints "PANIC CAUGHT OK (code = -1)" and exits 0. If the
# profile were flipped to panic="abort" the process would SIGABRT
# instead, making this step a runtime proof — not just a string check.
# 2. Build the actual feature-gated cdylib panic entrypoint, load it with
# dlopen from a C process, and require the exported ABI boundary to
# return ErrInternal instead of aborting the host.
ffi-panic-guard:
name: FFI Panic Guard
runs-on: macos-latest
@ -268,8 +326,8 @@ jobs:
fi
echo "OK: release-ffi profile retains panic=\"unwind\""
- name: Prove catch_unwind survives the release-ffi profile
run: cargo run --locked --profile release-ffi --example panic_spike -p agent-desktop-ffi
- name: Prove the shipped cdylib catches an exported-boundary panic
run: bash crates/ffi/tests/run_cdylib_panic_probe.sh
# Gate 3 — Stub-adapter passthrough tests: the Family-B command-backed
# entrypoints (ad_snapshot, ad_status, ad_wait, ad_execute_by_ref,
@ -286,10 +344,9 @@ jobs:
# documented follow-up.
#
# Per-target build coverage:
# - PR CI: this job runs on Linux (no main-thread guard off macOS, so the
# stub adapter is reached directly) to assert PLATFORM_NOT_SUPPORTED for
# every covered entrypoint — directly proving the non-macOS parity
# U10 targets.
# - PR CI: this job runs on Linux and reaches the stub adapter directly to
# assert PLATFORM_NOT_SUPPORTED for every covered entrypoint, proving the
# non-macOS parity U10 targets.
# - Release: release.yml build-ffi matrix covers macOS×2 + Linux×2 +
# Windows×1 for the real cdylib. PR builds do not duplicate that
# cross-compile matrix to avoid bloating CI on every push.
@ -317,44 +374,5 @@ jobs:
- name: Run stub-adapter passthrough tests
run: >
cargo test --locked -p agent-desktop-ffi
scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi
--features stub-adapter --test c_abi_passthrough
# Gate 4 — Codegen drift: the committed src/commands/generated.rs must
# match what build.rs produces from the current templates. A stale committed
# file means the source diverged from the templates — either a hand-edit
# bypassed the generator or a template was changed without rebuilding.
ffi-codegen-drift:
name: FFI Codegen Drift
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Install Rust toolchain
run: rustup show
- name: Cache cargo registry
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock', 'Cargo.toml', 'crates/**/Cargo.toml', 'src/Cargo.toml') }}
restore-keys: ${{ runner.os }}-cargo-
- name: Regenerate FFI command wrappers via build.rs
run: cargo build --locked -p agent-desktop-ffi --features stub-adapter
- name: Assert no drift in commands/generated.rs
run: |
if ! git diff --exit-code crates/ffi/src/commands/generated.rs; then
echo ""
echo "FAIL: crates/ffi/src/commands/generated.rs is out of sync with build.rs templates."
echo " Run 'cargo build -p agent-desktop-ffi' locally and commit the updated file."
exit 1
fi
echo "OK: commands/generated.rs matches build.rs output"

26
.github/workflows/native-e2e.yml vendored Normal file
View file

@ -0,0 +1,26 @@
name: Native desktop E2E
on:
workflow_dispatch:
concurrency:
group: agent-desktop-native-e2e
cancel-in-progress: false
permissions: {}
jobs:
native-e2e:
name: Native macOS E2E
runs-on: [self-hosted, macOS, agent-desktop-e2e]
timeout-minutes: 45
permissions:
contents: read
env:
AGENT_DESKTOP_NATIVE_E2E_RUNNER: "1"
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
- name: Install pinned Rust toolchain
run: rustup show
- name: Build and run the exclusive native suite
run: scripts/run-native-e2e-ci.sh

79
.github/workflows/release.yml vendored
View file

@ -156,15 +156,19 @@ jobs:
restore-keys: ${{ runner.os }}-${{ matrix.target }}-cargo-
- name: Build release binary
run: cargo build --locked --release --target ${{ matrix.target }}
run: |
cargo build --locked --release -p agent-desktop --target ${{ matrix.target }}
cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper --target ${{ matrix.target }}
- name: Check binary size
run: |
SIZE=$(stat -f%z target/${{ matrix.target }}/release/agent-desktop)
HELPER_SIZE=$(stat -f%z target/${{ matrix.target }}/release/agent-desktop-macos-helper)
LIMIT=$((15 * 1024 * 1024))
echo "Binary size: $(du -sh target/${{ matrix.target }}/release/agent-desktop | cut -f1)"
if [ "$SIZE" -gt "$LIMIT" ]; then
echo "FAIL: binary exceeds 15MB limit (${SIZE} bytes)"
echo "Helper size: $(du -sh target/${{ matrix.target }}/release/agent-desktop-macos-helper | cut -f1)"
if [ "$SIZE" -gt "$LIMIT" ] || [ "$HELPER_SIZE" -gt "$LIMIT" ]; then
echo "FAIL: shipped executable exceeds 15MB limit (binary=${SIZE}, helper=${HELPER_SIZE} bytes)"
exit 1
fi
@ -173,7 +177,16 @@ jobs:
VERSION: ${{ needs.release-please.outputs.version }}
run: |
TARBALL="agent-desktop-v${VERSION}-${{ matrix.target }}.tar.gz"
tar -czf "${TARBALL}" -C target/${{ matrix.target }}/release agent-desktop
BINARY="target/${{ matrix.target }}/release/agent-desktop"
HELPER="target/${{ matrix.target }}/release/agent-desktop-macos-helper"
BINARY_SHA=$(shasum -a 256 "$BINARY" | awk '{print $1}')
HELPER_SHA=$(shasum -a 256 "$HELPER" | awk '{print $1}')
tar -czf "${TARBALL}" -C "$(dirname "$BINARY")" agent-desktop agent-desktop-macos-helper
if [ "$BINARY_SHA" != "$(shasum -a 256 "$BINARY" | awk '{print $1}')" ] || \
[ "$HELPER_SHA" != "$(shasum -a 256 "$HELPER" | awk '{print $1}')" ]; then
echo "Release executable changed while packaging" >&2
exit 1
fi
shasum -a 256 "${TARBALL}" > "${TARBALL}.sha256"
- name: Upload artifact
@ -245,7 +258,7 @@ jobs:
shell: bash
run: |
# Co-occurrence of the release profile token and the stub flag on the same
# line is the discriminator. Legitimate stub uses in this file (codegen regen,
# line is the discriminator. Legitimate stub uses in this file (release gates,
# passthrough test) omit the release-ffi profile, so they do not trigger.
PROFILE_TOKEN="release-ffi"
STUB_TOKEN="stub-adapter"
@ -260,6 +273,10 @@ jobs:
- name: Build FFI cdylib (release-ffi profile)
run: cargo build --locked --profile release-ffi -p agent-desktop-ffi --target ${{ matrix.target }}
- name: Build macOS helper for FFI consumers
if: runner.os == 'macOS'
run: cargo build --locked --release -p agent-desktop-macos --bin agent-desktop-macos-helper --target ${{ matrix.target }}
# build.rs bakes install_name=@rpath/... ; a regression here silently
# breaks Swift/SPM consumers, so verify before shipping.
- name: Verify macOS install_name
@ -282,6 +299,10 @@ jobs:
STAGE="agent-desktop-ffi-v${VERSION}-${{ matrix.target }}"
mkdir -p "$STAGE/lib" "$STAGE/include"
cp "target/${{ matrix.target }}/release-ffi/${{ matrix.lib_name }}" "$STAGE/lib/"
if [ "$RUNNER_OS" = "macOS" ]; then
cp "target/${{ matrix.target }}/release/agent-desktop-macos-helper" "$STAGE/lib/"
chmod +x "$STAGE/lib/agent-desktop-macos-helper"
fi
cp crates/ffi/include/agent_desktop.h "$STAGE/include/"
cp LICENSE "$STAGE/"
cat > "$STAGE/README.md" <<EOF
@ -297,6 +318,13 @@ jobs:
consumers need \`-rpath\` pointing at the dylib's directory; \`dlopen\` /
ctypes callers resolve by path and need nothing.
EOF
if [ "$RUNNER_OS" = "macOS" ]; then
# shellcheck disable=SC2016
echo '- `lib/agent-desktop-macos-helper` — required helper, colocated for dylib-image discovery' >> "$STAGE/README.md"
fi
if [ "$RUNNER_OS" != "Windows" ]; then
(cd "$STAGE" && shasum -a 256 lib/* include/* LICENSE > SHA256SUMS)
fi
echo "STAGE=$STAGE" >> "$GITHUB_ENV"
- name: Create archive (Unix)
@ -304,6 +332,10 @@ jobs:
shell: bash
run: |
tar -czf "${STAGE}.tar.gz" "${STAGE}"
(cd "$STAGE" && shasum -a 256 -c SHA256SUMS)
if [ "$RUNNER_OS" = "macOS" ]; then
tar -tzf "${STAGE}.tar.gz" | grep -Fx "${STAGE}/lib/agent-desktop-macos-helper"
fi
shasum -a 256 "${STAGE}.tar.gz" > "${STAGE}.tar.gz.sha256"
- name: Create archive (Windows)
@ -325,11 +357,8 @@ jobs:
agent-desktop-ffi-v*.zip.sha256
retention-days: 1
# Re-runs the three FFI integrity gates at release time so a hotfix or a PR
# that bypassed ci.yml cannot ship a dylib with a stale committed header or
# stale generated.rs. Gate 1 (header drift) must run on macos-latest because
# the committed header is macOS-generated and cbindgen may omit macOS-gated
# symbols on Linux. Gates 2 and 3 are consolidated here for a single job.
# Re-runs FFI integrity gates at release time. Header verification runs on
# macOS because cbindgen may omit macOS-gated symbols on Linux.
ffi-release-gates:
name: FFI Release Gates
needs: release-please
@ -359,9 +388,6 @@ jobs:
- name: Install pinned cbindgen 0.29.4
run: cargo install cbindgen --version 0.29.4 --locked
# Gate 1 — Header drift: committed crates/ffi/include/agent_desktop.h must
# match what cbindgen 0.29.4 generates from the current source. Exits
# non-zero when there is any diff.
- name: Header drift check (cbindgen --verify)
run: >
cbindgen crates/ffi
@ -369,32 +395,11 @@ jobs:
--output crates/ffi/include/agent_desktop.h
--verify
# Gate 2 — Codegen drift: build.rs must produce the exact same
# crates/ffi/src/commands/generated.rs that is committed.
- name: Regenerate FFI command wrappers via build.rs
run: cargo build --locked -p agent-desktop-ffi --features stub-adapter
- name: Codegen drift check (generated.rs)
run: |
if ! git diff --exit-code crates/ffi/src/commands/generated.rs; then
echo ""
echo "FAIL: crates/ffi/src/commands/generated.rs is out of sync with build.rs templates."
echo " Run 'cargo build -p agent-desktop-ffi' locally and commit the updated file."
exit 1
fi
echo "OK: commands/generated.rs matches build.rs output"
# Gate 3 — Stub passthrough: command-backed entrypoints must return
# structured PLATFORM_NOT_SUPPORTED envelopes via the stub adapter.
- name: Stub passthrough test
run: >
cargo test --locked -p agent-desktop-ffi
scripts/cargo-test-isolated-home.sh test --locked -p agent-desktop-ffi
--features stub-adapter --test c_abi_passthrough
# Gate 4 — Panic-boundary proof: mirrors ffi-panic-guard in ci.yml so a
# tag-only flip to panic=abort cannot slip past release gates. Two steps:
# (a) assert the Cargo.toml profile setting; (b) run the panic_spike example
# which exits 0 if catch_unwind caught the panic, SIGABRTs if panic=abort.
- name: Assert release-ffi profile keeps panic=unwind
run: |
if ! grep -A5 '\[profile\.release-ffi\]' Cargo.toml | grep -q 'panic.*=.*"unwind"'; then
@ -404,8 +409,8 @@ jobs:
fi
echo "OK: release-ffi profile retains panic=\"unwind\""
- name: Prove catch_unwind survives the release-ffi profile
run: cargo run --locked --profile release-ffi --example panic_spike -p agent-desktop-ffi
- name: Prove the shipped cdylib catches an exported-boundary panic
run: bash crates/ffi/tests/run_cdylib_panic_probe.sh
publish-github:
name: Publish to GitHub Release

8
.gitignore vendored
View file

@ -49,6 +49,10 @@ lcov.info
tarpaulin-report.html
coverage/
# Generated review and benchmark evidence
/benchmarks/locator-resolution/results/
/regression_findings.json
# Binary artifacts
*.dylib
*.so
@ -75,6 +79,8 @@ skills-lock.json
docs/*
!docs/architecture.excalidraw
!docs/architecture.png
!docs/faq.md
!docs/json-output.md
!docs/phases.md
!docs/solutions/
!docs/solutions/**
@ -85,3 +91,5 @@ rust_out
# trace export artifacts must never land in the repo tree
trace-*.html
__pycache__/
*.py[cod]

55
CLAUDE.md
View file

@ -16,6 +16,7 @@ cargo fmt --all -- --check # Format check
cargo fmt --all # Auto-format
cargo tree -p agent-desktop-core # Verify no platform crate leaks (CI enforces)
bash tests/e2e/run.sh # E2E: real binary vs fixture app, verify by observation (needs --release + AX permission)
bash scripts/perf-baseline-compare.sh # Perf A/B vs merge-base -> report.html (add --apps "Slack,Google Chrome" for Electron coverage)
```
Run the binary: `./target/release/agent-desktop snapshot --app Finder -i`
@ -160,21 +161,24 @@ Batch is not a second dispatcher. `src/batch/mod.rs` deserializes JSON entries i
### Additive Phase Model
- **Phase 1:** Foundation + macOS MVP (56 commands, core engine, macOS adapter)
- **Phase 2:** Windows + Linux adapters, 10+ new commands — core untouched
- **Phase 3:** MCP server mode via `--mcp` flag — wraps existing commands
- **Phase 4:** Daemon, sessions, enterprise quality gates
Numbering follows `docs/phases.md` (the source of truth for phase scope):
Phases 2–4 add adapters, transports, and production readiness work. Nothing in core is rebuilt.
- **Phase 1 / 1.5 / 1.6 (completed):** Foundation + macOS MVP, FFI cdylib distribution, and the Playwright-grade foundation contract (capability supertraits, auto-wait, occlusion gate, live locator, ProcessState, envelope 2.1)
- **Phase 2:** Windows adapter — delivered as sub-phases 2.0–2.15, beginning with a raw-script platform exploration phase, each a <=2,000-LOC PR into the `feat/windows-adapter` integration branch
- **Phase 3:** Linux adapter — same sub-phase template onto `feat/linux-adapter`
- **Phase 4:** MCP server mode via `--mcp` flag — wraps existing commands
- **Phase 5:** Daemon, sessions, enterprise quality gates
Later phases add adapters, transports, and production readiness work. Nothing in core is rebuilt; platform work implements into the settled contract.
## Coding Standards
### File Rules
- **400 LOC hard limit per file.** If approaching 400, split by responsibility. No exceptions. _Exception: files bearing an `@generated` marker that are produced by `build.rs` codegen and validated by a CI drift gate are exempt — the limit applies to the hand-written templates and the build script itself, not the generated output. Do not hand-edit these files; fix the generator instead._
- **400 LOC hard limit per hand-written file.** If approaching 400, split by responsibility. The committed C header at `crates/ffi/include/agent_desktop.h` is generated by cbindgen and verified by the `ffi-header-drift` CI job; change the Rust ABI declarations or cbindgen configuration and regenerate it rather than hand-editing generated declarations.
- **No inline comments.** Code must be self-documenting through naming. Only Rust doc-comments (`///`) on public items when the name alone is insufficient.
- **One struct/enum per file** for domain types. `node.rs` defines `AccessibilityNode`. `action.rs` defines `Action`.
- **One command per file.** Each CLI command lives in its own file under `commands/`. Filename matches the command name. _This rule scopes to hand-written command files; a single `@generated` wrapper file that consolidates multiple command entrypoints is not a violation._
- **One command per file.** Each CLI command lives in its own file under `commands/`. Filename matches the command name.
- **No God objects.** No struct with more than 7 fields. No function with more than 5 parameters. Use builder patterns or config structs.
- **Explicit pub boundaries.** Only `lib.rs` re-exports public items. Internal modules use `pub(crate)`. No wildcard re-exports.
@ -192,7 +196,7 @@ Phases 2–4 add adapters, transports, and production readiness work. Nothing in
PERM_DENIED, ELEMENT_NOT_FOUND, APP_NOT_FOUND, ACTION_FAILED,
ACTION_NOT_SUPPORTED, STALE_REF, AMBIGUOUS_TARGET, WINDOW_NOT_FOUND,
PLATFORM_NOT_SUPPORTED, TIMEOUT, INVALID_ARGS, NOTIFICATION_NOT_FOUND,
SNAPSHOT_NOT_FOUND, POLICY_DENIED, INTERNAL
SNAPSHOT_NOT_FOUND, POLICY_DENIED, APP_UNRESPONSIVE, INTERNAL
```
### Exit Codes
@ -213,7 +217,7 @@ SNAPSHOT_NOT_FOUND, POLICY_DENIED, INTERNAL
| Functions | `snake_case`, verb-first | `build_tree()`, `allocate_refs()` |
| Constants | `SCREAMING_SNAKE_CASE` | `MAX_TREE_DEPTH`, `DEFAULT_TIMEOUT_MS` |
| CLI flags | kebab-case | `--max-depth`, `--include-bounds` |
| Ref IDs | `@e{n}` sequential | `@e1`, `@e2`, `@e14` |
| Ref IDs | `@<snapshot_id>:e{n}` qualified output | `@s8f3k2p9:e1`, `@s8f3k2p9:e14` |
### Platform Crate Folder Structure
@ -278,7 +282,7 @@ Every command produces a response envelope:
```json
{
"version": "2.0",
"version": "2.1",
"ok": true,
"command": "snapshot",
"data": {
@ -294,18 +298,27 @@ Error responses:
```json
{
"version": "2.0",
"version": "2.1",
"ok": false,
"command": "click",
"error": {
"code": "STALE_REF",
"message": "Element could not be resolved from the requested snapshot",
"suggestion": "Run 'snapshot' to refresh, then retry with updated ref"
"suggestion": "Run 'snapshot' to refresh, then retry with updated ref",
"recovery": {
"strategy": "refresh_snapshot_then_retry_original",
"retryable": true,
"requires_fresh_snapshot": true
},
"disposition": {
"delivery": "not_delivered",
"retry": "safe"
}
}
}
```
The `error` object may also carry an optional `details` object (e.g. the actionability report on an actionability failure, candidate summaries on `AMBIGUOUS_TARGET`, or the last observed state on a `wait` `TIMEOUT`).
The `error` object may also carry optional `details` and `recovery` objects. Version 2.1 removed the 2.0 `retry_command` string without a compatibility alias; consumers must use `recovery.strategy` only when `disposition.retry` is `safe`.
### Serialization Rules
@ -316,7 +329,7 @@ The `error` object may also carry an optional `details` object (e.g. the actiona
## Ref System
- Refs allocated in depth-first document order: `@e1`, `@e2`, etc.
- Refs are allocated in depth-first document order and emitted as snapshot-qualified IDs such as `@s8f3k2p9:e1`. Legacy bare IDs such as `@e1` remain valid input only with an explicit `--snapshot s8f3k2p9`.
- An element receives a ref when it is **addressable for an action**: its role is interactive (`button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `switch`, `colorwell`, `menubutton`, `incrementor`, `dockitem`), **or** it advertises an available action regardless of role. Container roles such as `scrollarea` (Scroll) and `disclosure` (Expand/Collapse/Click) are not interactive by role but are genuinely actionable, so they are ref-able — `scroll` / `expand` / `collapse` need a ref to target them
- A bare `SetFocus` affordance does not qualify on its own (focusability is not a primary action), so inert focusable containers stay ref-less
- Static text and non-actionable groups/containers do NOT get refs (they remain in tree for context)
@ -327,14 +340,14 @@ The `error` object may also carry an optional `details` object (e.g. the actiona
- Progressive traversal: `--skeleton` clamps depth to 3, annotates truncated containers with `children_count`. Named/described containers at boundary receive refs as drill-down targets
- Drill-down: `--root @ref` starts from a previously-discovered ref with scoped invalidation (only that ref's subtree refs are replaced on re-drill)
- RefMap size check: write-side guard prevents >1MB refmap files
- **Sessions:** `session start` creates a manifest-gated session under `~/.agent-desktop/sessions/<id>/`, sets `current_session`, and (by default) enables automatic trace segments. Bare `--session <id>` without a manifest scopes only the snapshot namespace — no surprise trace files
- **Trace:** manifest `trace: on` writes per-process JSONL segments under `<session>/trace/<pid>-<procTs>.jsonl`; `--trace <path>` overrides to one file; activation resolves `--session` > `AGENT_DESKTOP_SESSION` > `current_session`
- **Sessions:** `session start` creates and returns a manifest-gated session under `~/.agent-desktop/sessions/<id>/` and enables automatic trace segments by default. It does not activate that session for later processes. Pass the returned ID through `--session` or `AGENT_DESKTOP_SESSION`. Bare `--session <id>` without a manifest scopes only the snapshot namespace — no surprise trace files.
- **Trace:** manifest `trace: on` writes per-process JSONL segments under `<session>/trace/<pid>-<procTs>.jsonl`; `--trace <path>` overrides to one file; activation resolves `--session` > `AGENT_DESKTOP_SESSION` > no session. Snapshot lookup is confined to that selected namespace and never searches other sessions.
## PlatformAdapter Trait
Core defines `PlatformAdapter`; platform crates implement it. Methods default to
`not_supported()`, so an adapter only implements what it supports. Read the
current signatures in `crates/core/src/adapter.rs` — notably strict resolution
current signatures in `crates/core/src/adapter/mod.rs` and its `actions.rs`, `input.rs`, `observation.rs`, and `system.rs` capability fragments — notably strict resolution
(`resolve_element_strict*` → STALE_REF on 0, AMBIGUOUS_TARGET on 2+), live reads
for the actionability preflight (`get_live_*`), and `is_protected_process`
(keeps platform-specific process names out of core).
@ -369,10 +382,10 @@ for the actionability preflight (`get_live_*`), and `is_protected_process`
## Commands
56 commands spanning App/Window, Observation, Interaction, Scroll, Keyboard,
58 commands spanning App/Window, Observation, Interaction, Scroll, Keyboard,
Mouse, Notifications (macOS), Clipboard, Wait, System (including `session`), and
Batch. The full surface and per-command reference live in `skills/agent-desktop/`.
All 56 are implemented on macOS (Phase 1); Windows/Linux (Phase 2/3) target the
All 58 are implemented on macOS (Phase 1); Windows/Linux (Phase 2/3) target the
same surface. Adding a command: see the Extensibility Pattern above.
## Non-Goals
@ -388,3 +401,7 @@ same surface. Adding a command: see the Extensibility Pattern above.
- PRD v2.0: `docs/agent_desktop_prd_v2.pdf`
- Architecture Brainstorm: `docs/brainstorms/2026-02-19-architecture-validation-brainstorm.md`
- Phase 1 Plan: `docs/plans/2026-02-19-feat-agent-desktop-phase1-foundation-plan.md`
## Definition of Done: Performance Baseline
Every substantive change ends with a performance baseline check before merge: run `bash scripts/perf-baseline-compare.sh` (optionally `--apps "Slack,Google Chrome"` for dense Electron/Chromium coverage) and review the generated `report.html` against the merge-base. Latency deltas must be intentional and explainable — never discovered by users.

18
CONCEPTS.md
View file

@ -11,7 +11,7 @@ A structured representation of an application's user interface exposed by the op
An observation of an accessibility tree at a point in time, persisted with the element refs allocated from that observation.
### Snapshot ID
A compact identifier for one persisted snapshot. Explicit snapshot IDs are direct handles; callers do not need the original session when they pass the ID.
A compact identifier for one persisted snapshot. Lookup is confined to the selected session namespace, so an ID created in a session is not a cross-session handle.
### Surface
A scoped UI layer that can be observed separately from the whole window, such as an open menu, sheet, popover, alert, or focused area.
@ -24,7 +24,7 @@ A snapshot operation that starts from an existing ref to observe that element's
### Ref
A short element identifier assigned by agent-desktop to an actionable or drillable node in a snapshot.
Refs are deterministic inside one snapshot but are not stable across UI changes. Callers either pass the snapshot ID that produced the ref or intentionally use the session's latest snapshot pointer.
Refs are deterministic inside one snapshot but are not stable across UI changes. Snapshot and find output qualify each ref with its snapshot ID. Legacy bare refs require the producing snapshot ID as a separate argument.
### RefMap
The persisted mapping from refs to the identity evidence needed to re-identify elements later.
@ -47,9 +47,9 @@ Strict ref resolution rejects missing, stale, and ambiguous matches instead of g
### Session
An on-disk container under `~/.agent-desktop/sessions/<id>/` that owns snapshot refmaps, an optional trace directory, and a `session.json` manifest.
`session start` writes the manifest (`trace: on` unless `--no-trace`), pre-creates `trace/` when tracing is on, and sets `~/.agent-desktop/current_session`. Activating a session (via pointer, `AGENT_DESKTOP_SESSION`, or `--session`) relocates the latest-snapshot namespace as well as the trace sink. Bare `--session <id>` without a manifest remains snapshot-namespace-only for backward compatibility.
`session start` writes the manifest (`trace: on` unless `--no-trace`) and pre-creates `trace/` when tracing is on. It returns the new ID but does not activate it for later processes. Explicit `--session` takes precedence over `AGENT_DESKTOP_SESSION`; with neither, commands use the global, non-session namespace. Bare `--session <id>` without a manifest remains snapshot-namespace-only for backward compatibility.
Use sessions when callers intentionally omit `--snapshot` and want a shared latest observation — typically after `session start` for a coordinated run. Explicit snapshot IDs remain the deterministic path for pinned actions and can be resolved without also passing the session.
Use sessions when callers want a coordinated snapshot namespace and trace sink. Every lookup is confined to its selected namespace, so a snapshot created under a session requires that same `--session` or `AGENT_DESKTOP_SESSION` scope later. Qualified refs remain the deterministic path for pinned actions inside that namespace.
### Session Manifest
The `session.json` file describing one session: id, optional name, created/ended timestamps, and `trace: on|off`.
@ -84,7 +84,9 @@ The platform-neutral set of supported action names that core uses to compare com
Each adapter maps native primitives into this shared vocabulary before core evaluates actionability. New commands should extend the central vocabulary first, then reuse it from actionability, ref allocation, predicates, FFI tests, and platform adapters.
### Interaction Policy
The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input fallbacks. Ref commands expose exactly two modes: **headless** (the default — accessibility-only, no cursor, fails closed when the AX path is unavailable) and **headed** (opt-in via the global `--headed` flag — permits focus stealing and cursor movement so the action chain's physical fallbacks can complete). The AX path is always tried first, so headed never regresses headless-capable elements. The `headed` upgrade applies uniformly to every ref command; each command still declares its own headless base policy (most are pure-AX; `type` uses a focus-fallback base because typing requires focus but never moves the cursor).
The side-effect contract attached to an action request, controlling whether the command may steal focus, move the cursor, or use physical input. Ref commands expose exactly two modes: **headless** (the default — accessibility-only, no cursor, fails closed when the semantic path is unavailable) and **headed** (opt-in via the global `--headed` flag — authorizes the action's declared focus and cursor preconditions).
Core owns those preconditions through `HeadedRequirement`: `FocusedWindow` for keyboard or focus-sensitive work, and `FocusedWindowAndCursor` for pointer delivery. For ref actions, core focuses the exact source window before dispatch and resolves a verified target point for pointer work; the platform adapter owns the OS-specific focus primitive and delivery mechanism. Raw coordinate input has no ref identity, so it never infers or focuses a window. On macOS, headed `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; double/triple-click, hover, and drag are physical-only; expand/collapse and the remaining semantic actions stay semantic after the core focus precondition.
### Headless Ref Action
A ref-based action that uses semantic accessibility operations without implicit focus stealing, cursor movement, synthetic keyboard input, or pasteboard use. This is the default mode.
@ -92,7 +94,7 @@ A ref-based action that uses semantic accessibility operations without implicit
Headless ref actions may still fail when the native accessibility API cannot perform the requested semantic operation; they fail closed with structured actionability or policy errors rather than silently substituting physical input. The broader **headed** policy must be selected explicitly with `--headed`.
### Action Chain
The ordered ladder of strategies a ref action walks to perform one intent — semantic accessibility actions first, then settable attributes, then policy-gated physical input — with each step verified against the element's observed state before it counts as success.
The ordered ladder of strategies a ref action walks to perform one intent, with each step gated by policy and its delivery evidence recorded. The order is action-specific: natural input may put a headed physical step first, while semantic state changes use accessibility actions or settable attributes only.
The chain pins one execution deadline at its start (distinct from the Resolver Deadline, which budgets re-identification) and every step observes it. Expiry while a step may have partially mutated the element surfaces as a structured timeout carrying the observed state, never as a plain step failure — the caller must be able to tell "nothing happened" from "something may have happened".
@ -105,11 +107,11 @@ The remaining time budget carried through strict ref resolution so every native
### Coordinate Fallback
An explicit opt-in path that uses screen coordinates or physical input when semantic accessibility operations cannot perform the requested action.
Physical input lands on the topmost window at the target point, so the fallback first ensures the target element's own window is frontmost — the app being frontmost is not sufficient when the element lives in a background window of that app.
Ref-targeted physical input lands on the topmost window at the resolved point, so core first ensures the target element's exact window is frontmost — the app being frontmost is not sufficient when the element lives in a background window of that app. Raw `--xy` input carries no window identity and therefore moves/clicks at the requested coordinates without focusing any application.
### FFI Ref-Action Parity
The requirement that language bindings using refs follow the same strict resolution, actionability, and interaction-policy semantics as CLI ref commands.
## Relationships
A session owns one latest-snapshot pointer, an optional manifest-gated trace directory, and persisted snapshot refmaps. A snapshot persists a ref map and can be selected directly by snapshot ID. A ref resolves through strict ref resolution into live native evidence, then actionability decides whether a headless ref action can safely dispatch, and the action chain executes that dispatch under its own deadline with the interaction policy gating its physical steps. FFI ref-action parity keeps that same relationship true for language bindings.
A session owns one latest-snapshot pointer, an optional manifest-gated trace directory, and persisted snapshot refmaps. A snapshot persists a ref map and can be selected by ID within that same namespace. A ref resolves through strict ref resolution into live native evidence, then actionability decides whether the action can safely dispatch. In headed mode, core applies the action's focus/cursor requirement before the platform adapter executes the action-specific chain under its own deadline. FFI ref-action parity keeps that same relationship true for language bindings.

2
Cargo.lock generated
View file

@ -39,6 +39,7 @@ dependencies = [
"smallvec",
"thiserror",
"tracing",
"windows-sys",
]
[[package]]
@ -74,6 +75,7 @@ dependencies = [
"core-foundation",
"core-foundation-sys",
"core-graphics",
"libc",
"rustc-hash",
"serde",
"serde_json",

6
Cargo.toml
View file

@ -1,10 +1,11 @@
[workspace]
members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "crates/ffi", "src"]
members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "crates/ffi", "src"]
default-members = ["crates/core", "crates/macos", "crates/windows", "crates/linux", "src"]
resolver = "2"
[workspace.package]
edition = "2024"
rust-version = "1.85"
rust-version = "1.89"
license = "Apache-2.0"
version = "0.4.7" # x-release-please-version
@ -20,6 +21,7 @@ rustc-hash = "2.1"
libc = "0.2"
smallvec = { version = "1.13", features = ["serde", "union"] }
agent-desktop-core = { path = "crates/core" }
windows-sys = { version = "0.61.2" }
[workspace.lints.rust]
unsafe_op_in_unsafe_fn = "warn"

272
README.md
View file

@ -41,9 +41,9 @@
- **Native Rust CLI**: Fast, single binary, no runtime dependencies
- **C-ABI cdylib** (`libagent_desktop_ffi`): Load once from Python / Swift / Go / Ruby / Node / C instead of forking the CLI per call
- **56 commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, plus a bundled `skills` doc loader
- **58 command names, 54 operational commands**: Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, plus a bundled `skills` doc loader. The four held-input names are reserved for a stateful daemon and fail closed in the stateless CLI.
- **Progressive skeleton traversal**: 78–96% token reduction on dense apps via shallow overview + targeted drill-down
- **Snapshot & refs**: AI-optimized workflow using compact snapshot IDs and deterministic element references (`@e1`, `@e2`)
- **Snapshot & refs**: AI-optimized workflow using compact snapshot IDs and qualified element references (`@s8f3k2p9:e1`, `@s8f3k2p9:e2`)
- **Headless-by-default interactions**: Ref actions use accessibility APIs and block silent focus, cursor, keyboard, or pasteboard side effects
- **Structured JSON output**: Machine-readable responses with error codes and recovery hints
- **Works with any app**: Finder, Safari, System Settings, Xcode, Slack — anything with an accessibility tree
@ -71,14 +71,14 @@ cargo build --release
cp target/release/agent-desktop /usr/local/bin/
```
Requires Rust 1.85+ and macOS 13.0+.
Requires Rust 1.89+ and macOS 13.0+.
### Permissions
macOS requires Accessibility permission. Screenshots also require Screen Recording permission. Grant them in **System Settings > Privacy & Security** by adding the app that launches agent-desktop, or:
macOS requires Accessibility permission. Screenshots also require Screen Recording permission, and the Notification Center opener requires Automation permission for System Events. Plain permission checks never prompt. Request missing permissions in a bounded isolated helper with:
```bash
agent-desktop permissions --request # trigger platform permission request path
agent-desktop permissions --request # request missing permissions in an isolated helper
```
Permission fields are explicit objects, for example:
@ -87,10 +87,12 @@ Permission fields are explicit objects, for example:
{
"accessibility": { "state": "granted" },
"screen_recording": { "state": "denied", "suggestion": "Grant Screen Recording permission" },
"automation": { "state": "not_required" }
"automation": { "state": "unknown" }
}
```
Automation reports `granted`, `denied`, or `unknown`; `unknown` means macOS would need to prompt or System Events could not be probed without prompting.
## Language bindings (FFI)
Every GitHub Release ships a prebuilt C-ABI cdylib (`libagent_desktop_ffi`) for macOS, Linux, and Windows alongside the CLI tarballs. `dlopen` it and call the functions declared in `agent_desktop.h` for in-process calls instead of fork-exec per command.
@ -98,9 +100,9 @@ Every GitHub Release ships a prebuilt C-ABI cdylib (`libagent_desktop_ffi`) for
```python
import ctypes
lib = ctypes.CDLL("./lib/libagent_desktop_ffi.dylib")
lib.ad_init(1) # verify ABI major (AD_ABI_VERSION_MAJOR) before any call
lib.ad_init(3) # verify ABI major (AD_ABI_VERSION_MAJOR) before any call
adapter = lib.ad_adapter_create()
# observe -> act: ad_snapshot -> parse an @e ref -> ad_execute_by_ref ...
# observe -> act: ad_snapshot -> parse a qualified ref -> ad_execute_by_ref ...
lib.ad_adapter_destroy(adapter)
```
@ -142,9 +144,10 @@ Agent loop: snapshot → decide → act → snapshot → decide → act → ...
### Trace viewer (read back a session)
```bash
agent-desktop session start --screenshots # opt-in replay artifacts (PNG + refmap copies)
agent-desktop snapshot --app Finder -i # work normally under the active session
agent-desktop click @e5
session_id=$(agent-desktop session start --screenshots | jq -r '.data.session_id')
export AGENT_DESKTOP_SESSION="$session_id"
agent-desktop snapshot --app Finder -i # work inside the explicit session scope
agent-desktop click @s8f3k2p9:e5
agent-desktop trace show --limit 500 # bounded JSON timeline for agents
agent-desktop trace export --out run.html # single-file HTML viewer (works from file://)
```
@ -153,19 +156,20 @@ agent-desktop trace export --out run.html # single-file HTML viewer (works fr
### Shared sessions for multi-agent workflows
Run `session start` once per agent run to create a trace-enabled session (manifest `trace: on` by default) and set the active pointer. Subsequent commands in that run get automatic JSONL segments under `~/.agent-desktop/sessions/<id>/trace/` and share the session's latest-snapshot namespace — no `--trace` on every call.
Run `session start` once per agent run to create a trace-enabled session (manifest `trace: on` by default), then pass the returned ID with global `--session <id>` or `AGENT_DESKTOP_SESSION=<id>`. Commands in that explicit scope get automatic JSONL segments under `~/.agent-desktop/sessions/<id>/trace/` and share the session's latest-snapshot namespace — no `--trace` on every call.
For concurrent **independent** agents, set `AGENT_DESKTOP_SESSION=<id>` per process instead of relying on the global pointer. When multiple agents share one session id, each agent should act on the `snapshot_id` from its own `snapshot` call; implicit latest is a single-agent convenience.
For concurrent **independent** agents, set `AGENT_DESKTOP_SESSION=<id>` per process. When multiple agents share one session ID, each agent should act on the qualified refs from its own `snapshot` call rather than assuming the namespace's latest snapshot is unchanged.
Bare `--session <id>` without a manifest (no `session start`) still scopes the snapshot namespace only and writes no trace files. Explicit `--snapshot <id>` resolves cross-session.
Bare `--session <id>` without a manifest (no `session start`) still scopes the snapshot namespace only and writes no trace files. Snapshot IDs resolve only inside the selected session namespace; they never trigger a cross-session search.
```bash
agent-desktop session start --name release-fix
agent-desktop snapshot --app Xcode -i --compact # uses active session + tracing
agent-desktop wait --element @e9 --predicate actionable --timeout 5000
agent-desktop click @e9
agent-desktop click @e9 --snapshot s2 # pin to a specific observation
agent-desktop session end
agent-desktop session start --name release-fix # note data.session_id
export AGENT_DESKTOP_SESSION=<session_id>
agent-desktop snapshot --app Xcode -i --compact # uses selected session + tracing
agent-desktop wait --element @s8f3k2p9:e9 --predicate actionable --timeout 5000
agent-desktop click @s8f3k2p9:e9
agent-desktop click @e9 --snapshot s2 # legacy bare ref, explicitly pinned
agent-desktop session end "$AGENT_DESKTOP_SESSION"
agent-desktop session gc
```
@ -188,25 +192,26 @@ agent-desktop list-surfaces --app Notes # list menus, sheets, popovers,
### Interaction
```bash
agent-desktop click @e3 # semantic AX-first click
agent-desktop double-click @e3 # AXOpen; physical double-click uses --headed mouse-click --count 2
agent-desktop triple-click @e3 # POLICY_DENIED if physical input is disabled
agent-desktop right-click @e3 # open verified context menu
agent-desktop type @e5 "hello world" # insert text into element
agent-desktop set-value @e5 "new value" # set value directly via AX
agent-desktop clear @e5 # clear element value
agent-desktop focus @e5 # set keyboard focus
agent-desktop select @e9 "Option B" # select verified dropdown/list option
agent-desktop toggle @e12 # flip checkbox or switch
agent-desktop check @e12 # idempotent check
agent-desktop uncheck @e12 # idempotent uncheck
agent-desktop expand @e15 # expand disclosure/tree item
agent-desktop collapse @e15 # collapse disclosure/tree item
agent-desktop scroll @e1 --direction down --amount 3 # scroll (AX-first)
agent-desktop scroll-to @e20 # scroll element into view
agent-desktop click @s8f3k2p9:e3 # strict headless AX click
agent-desktop --headed click @s8f3k2p9:e3 # physical click, focus/cursor allowed
agent-desktop --headed double-click @s8f3k2p9:e3 # physical double-click
agent-desktop --headed triple-click @s8f3k2p9:e3 # physical triple-click
agent-desktop right-click @s8f3k2p9:e3 # open context menu; inspect effect before retrying
agent-desktop type @s8f3k2p9:e5 "hello world" # insert text into element
agent-desktop set-value @s8f3k2p9:e5 "new value" # set value directly via AX
agent-desktop clear @s8f3k2p9:e5 # clear element value
agent-desktop focus @s8f3k2p9:e5 # set keyboard focus
agent-desktop select @s8f3k2p9:e9 "Option B" # select verified dropdown/list option
agent-desktop toggle @s8f3k2p9:e12 # flip checkbox or switch
agent-desktop check @s8f3k2p9:e12 # idempotent check
agent-desktop uncheck @s8f3k2p9:e12 # idempotent uncheck
agent-desktop expand @s8f3k2p9:e15 # expand disclosure/tree item
agent-desktop collapse @s8f3k2p9:e15 # collapse disclosure/tree item
agent-desktop scroll @s8f3k2p9:e1 --direction down --amount 3 # strict headless AX scroll
agent-desktop scroll-to @s8f3k2p9:e20 # scroll element into view
```
> **(macOS, Phase 1)** Pure cursor gestures have no accessibility equivalent, so `triple-click`, `hover`, and `drag` are always physical; `double-click` is headless via `AXOpen` and only needs `--headed` for gesture-only targets. Windows (UIA) and Linux (AT-SPI) adapters may expose different capabilities. See `skills/agent-desktop/references/commands-interaction.md`.
> **(macOS, Phase 1)** Default ref actions are strict headless semantic operations. In headed mode, core focuses the exact ref window before dispatch; pointer commands additionally require a verified target point, while the adapter owns physical delivery. `click`, `right-click`, `type`, `clear`, and `scroll` are physical-first; double/triple-click, hover, and drag are physical-only; expand/collapse and other semantic commands remain semantic. Raw coordinates never imply a target window and therefore never steal focus. See `skills/agent-desktop/references/commands-interaction.md`.
### Keyboard
@ -214,22 +219,22 @@ agent-desktop scroll-to @e20 # scroll element into view
agent-desktop press cmd+s # key combo
agent-desktop press cmd+shift+z # multi-modifier
agent-desktop press escape # single key
agent-desktop key-down shift # hold key
agent-desktop key-up shift # release key
```
`key-down` and `key-up` are reserved command names and return `ACTION_NOT_SUPPORTED` until a stateful daemon can own the held-key lifetime.
### Mouse
```bash
agent-desktop --headed hover @e3 # move cursor to element
agent-desktop --headed hover @s8f3k2p9:e3 # move cursor to element
agent-desktop --headed hover --xy 500,300 # move cursor to coordinates
agent-desktop --headed drag --from @e3 --to @e8 # drag between elements
agent-desktop --headed drag --from @s8f3k2p9:e3 --to @s8f3k2p9:e8 # drag between elements
agent-desktop --headed drag --from-xy 100,200 --to-xy 400,200 # drag between coordinates
agent-desktop --headed mouse-click --xy 500,300 # click at coordinates
agent-desktop --headed mouse-down --xy 500,300 # press at coordinates
agent-desktop --headed mouse-up --xy 500,300 # release at coordinates
```
`mouse-down` and `mouse-up` are likewise reserved; use the atomic `mouse-click` or `drag` commands.
### App & Window Management
```bash
@ -240,26 +245,31 @@ agent-desktop close-app Safari --force # force quit (SIGTERM, then SIGKILL if
agent-desktop list-apps # list running GUI apps
agent-desktop list-windows # list visible windows
agent-desktop list-windows --app Finder # windows for specific app
agent-desktop focus-window w-4521 # bring window to front
agent-desktop resize-window w-4521 800 600 # resize
agent-desktop move-window w-4521 100 100 # move
agent-desktop minimize w-4521 # minimize
agent-desktop maximize w-4521 # maximize
agent-desktop restore w-4521 # restore
agent-desktop focus-window --window-id w-4521 # bring exact window to front
agent-desktop resize-window --window-id w-4521 --width 800 --height 600
agent-desktop move-window --window-id w-4521 --x 100 --y 100
agent-desktop minimize --window-id w-4521
agent-desktop maximize --window-id w-4521
agent-desktop restore --window-id w-4521
```
### Notifications *(macOS only)*
```bash
agent-desktop list-notifications # list all notifications
agent-desktop list-notifications --app "Slack" # filter by app
agent-desktop list-notifications --text "deploy" --limit 5 # filter by text
agent-desktop dismiss-notification 1 # dismiss by index
agent-desktop dismiss-all-notifications # dismiss all
agent-desktop dismiss-all-notifications --app "Slack" # dismiss all from app
agent-desktop notification-action 1 --action "Reply" # click action button
agent-desktop --headed list-notifications # open Notification Center if needed, then list
agent-desktop --headed list-notifications --app "Slack" # filter by app
agent-desktop --headed list-notifications --text "deploy" --limit 5 # filter by text
agent-desktop --headed dismiss-notification 1 --expected-app "Slack" --expected-title "Deploy complete"
agent-desktop --headed dismiss-all-notifications # dismiss all
agent-desktop --headed dismiss-all-notifications --app "Slack" # dismiss all from app
agent-desktop --headed notification-action 1 "Reply" --expected-app "Slack" --expected-title "Deploy complete"
```
Single-notification mutations require an app or title fingerprint from the
same listing. Every mutation requires `--headed` because it opens and focuses
Notification Center. Headless listing can only observe an already-open center;
headed listing may open it and restore the prior frontmost app afterward.
### Clipboard
```bash
@ -272,9 +282,9 @@ agent-desktop clipboard-clear # clear clipboard
```bash
agent-desktop wait 500 # sleep 500ms
agent-desktop wait --element @e3 --timeout 5000 # wait for element
agent-desktop wait --element @e3 --predicate actionable # wait until safe to act
agent-desktop wait --element @e5 --predicate value --value ready
agent-desktop wait --element @s8f3k2p9:e3 --timeout 5000 # wait for element
agent-desktop wait --element @s8f3k2p9:e3 --predicate actionable # wait until safe to act
agent-desktop wait --element @s8f3k2p9:e5 --predicate value --value ready
agent-desktop wait --window "Save" --timeout 10000 # wait for window
agent-desktop wait --text "Loading complete" --app Safari # wait for text
agent-desktop wait --text "Done" --count 1 --app Xcode # wait for exact match count
@ -300,13 +310,13 @@ agent-desktop --session run-a batch '[
### System
```bash
agent-desktop session start [--name LABEL] [--no-trace] # trace-enabled run (sets active pointer)
agent-desktop session start [--name LABEL] [--no-trace] # create session; pass returned ID explicitly
agent-desktop session end [id]
agent-desktop session list
agent-desktop session gc [--older-than SECS] [--ended]
agent-desktop status # platform, permissions, session_id, tracing, latest snapshot
agent-desktop permissions # check accessibility/screen-recording/automation
agent-desktop permissions --request # invoke platform request path
agent-desktop permissions --request # request in the bounded isolated helper
agent-desktop version # version string
agent-desktop skills get desktop --full # bundled agent guidance
```
@ -332,55 +342,11 @@ agent-desktop snapshot [OPTIONS]
## JSON Output
Every command returns structured JSON:
```json
{
"version": "2.0",
"ok": true,
"command": "click",
"data": { "action": "click" }
}
```
Errors include machine-readable codes and recovery hints:
```json
{
"version": "2.0",
"ok": false,
"command": "click",
"error": {
"code": "STALE_REF",
"message": "Element at @e7 no longer matches the last snapshot",
"suggestion": "Run 'snapshot' to refresh refs, then retry"
}
}
```
### Error Codes
| Code | Meaning |
|------|---------|
| `PERM_DENIED` | Accessibility permission not granted |
| `ELEMENT_NOT_FOUND` | No element matched the ref or query |
| `APP_NOT_FOUND` | Application not running or no windows |
| `STALE_REF` | Ref could not be re-identified in the live UI |
| `AMBIGUOUS_TARGET` | Ref recovery matched multiple plausible targets |
| `SNAPSHOT_NOT_FOUND` | Snapshot ID is missing or expired |
| `POLICY_DENIED` | Physical/headed path blocked by policy |
| `ACTION_FAILED` | The OS rejected the action |
| `PLATFORM_NOT_SUPPORTED` | Adapter method not implemented on this platform |
| `TIMEOUT` | Wait condition expired |
| `INVALID_ARGS` | Invalid argument values |
### Exit Codes
`0` success, `1` structured error (JSON on stdout), `2` argument parse error.
See the [versioned JSON envelope, error-code, and exit-code contract](docs/json-output.md).
## Ref System
`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are scoped to a compact `snapshot_id` such as `s8f3k2p9`. Commands can omit `--snapshot` to use the active session's latest snapshot pointer, but passing the ID is more deterministic in multi-step flows and does not require also passing `--session`.
`snapshot` assigns local positions in depth-first order and emits qualified refs such as `@s8f3k2p9:e1`, `@s8f3k2p9:e2`, and `@s8f3k2p9:e3`. A qualified ref embeds the exact snapshot ID and needs no separate `--snapshot`. Legacy bare refs such as `@e3` remain accepted only with an explicit `--snapshot s8f3k2p9`. Snapshot lookup stays inside the selected session namespace.
Interactive roles that receive refs: `button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`.
@ -388,14 +354,14 @@ Static elements (labels, groups, containers) appear in the tree for context but
Reliability contract:
- `session start` creates a manifest-gated session with automatic trace segments and relocates the latest-snapshot namespace. Activation resolves `--session` > `AGENT_DESKTOP_SESSION` > `~/.agent-desktop/current_session` (pointer written only by `session start`).
- `session start` creates and returns a manifest-gated session with automatic trace segments. It does not activate later processes. Activation resolves explicit `--session` first, then `AGENT_DESKTOP_SESSION`; otherwise the command uses the global, non-session namespace.
- Bare `--session <id>` without a manifest scopes snapshots only — no surprise trace files for existing callers.
- Explicit `--snapshot <id>` resolves the saved snapshot directly, including across sessions.
- Snapshot lookup is confined to the selected namespace. A session-owned snapshot requires the same explicit `--session` or `AGENT_DESKTOP_SESSION` scope.
- Ref actions re-identify targets at action time: a moved unique target can proceed, while missing or changed stable identity returns `STALE_REF`.
- Mutable value text is not treated as stable identity, so text fields and timers can keep resolving when the saved window, path, role, and bounds evidence still identify the same element.
- Multiple plausible targets return `AMBIGUOUS_TARGET` instead of choosing arbitrarily.
- Actions run an actionability preflight before dispatch: visibility, stability, enabled state, supported action, policy, and editability.
- `wait --element @e3 --predicate actionable` polls until the target can be acted on.
- `wait --element @s8f3k2p9:e3 --predicate actionable` polls until the target can be acted on.
- With an active trace-enabled session, JSONL segments land under `sessions/<id>/trace/<pid>-*.jsonl` automatically. `--trace <path>` overrides to one file; `--trace-strict` fails on setup and pre-action writes (post-action traces are best-effort).
Stale ref recovery:
@ -427,91 +393,7 @@ cargo clippy --all-targets -- -D warnings # lint (must pass with zero warnings)
## FAQ
### What is agent-desktop?
agent-desktop is a native desktop automation CLI for AI agents. It lets agents observe and control desktop apps through OS accessibility trees, using structured JSON instead of screenshots, pixel matching, or browser-only automation.
### Does agent-desktop require screenshots or pixel matching?
No. The core workflow reads native accessibility trees and assigns refs to interactive elements. Screenshots are available as a separate command, but agents do not need screenshots or pixel matching to click buttons, type into fields, inspect menus, or navigate app windows.
### How does agent-desktop work?
| Component | Function |
|-----------|----------|
| **Native Rust CLI** | Fast, single binary, no runtime dependencies |
| **C-ABI cdylib** | Load once from Python, Swift, Go, Ruby, Node, or C instead of forking |
| **56 Commands** | Observation, interaction, keyboard, mouse, notifications, clipboard, window management, session lifecycle, trace read/export, and bundled `skills` docs |
| **Snapshot & Refs** | Compact snapshot IDs and deterministic element refs like `@e1`, `@e2` |
| **Structured JSON** | Machine-readable responses with error codes and recovery hints |
### What makes agent-desktop useful for AI agents?
| Feature | Benefit |
|---------|---------|
| **Progressive Skeleton Traversal** | 78–96% token reduction on dense apps |
| **Headless-by-Default Actions** | Ref actions use accessibility APIs and block unintended physical side effects |
| **Snapshot Refs** | Agents act on stable refs within a snapshot instead of guessing coordinates |
| **Recovery Hints** | Errors include machine-readable codes and suggestions for the next agent step |
| **Cross-Language FFI** | Python, Swift, Go, Ruby, Node, C, and C++ hosts can call the native library directly |
### Which platforms are supported?
| Feature | macOS | Windows | Linux |
|---------|:-----:|:-------:|:-----:|
| Accessibility tree | **Yes** | Planned | Planned |
| Click/type/keyboard | **Yes** | Planned | Planned |
| Mouse input | **Yes** | Planned | Planned |
| Screenshot | **Yes** | Planned | Planned |
| Clipboard | **Yes** | Planned | Planned |
| App/window management | **Yes** | Planned | Planned |
| Notifications | **Yes** | Planned | Planned |
### How do I install agent-desktop?
Install the CLI from npm:
```bash
npm install -g agent-desktop
agent-desktop snapshot --app Safari
```
Build the FFI library from source:
```bash
cargo build --release
# Outputs: libagent_desktop_ffi.dylib/.so/.dll
```
### What is the ref system?
`snapshot` assigns refs to interactive elements in depth-first order: `@e1`, `@e2`, `@e3`, etc. Refs are scoped to a compact `snapshot_id` such as `s8f3k2p9`. Commands can omit `--snapshot` to use the active session's latest snapshot pointer, but explicit snapshot IDs are the deterministic path and do not require also passing `--session`.
Interactive roles that receive refs:
`button`, `textfield`, `checkbox`, `link`, `menuitem`, `tab`, `slider`, `combobox`, `treeitem`, `cell`, `radiobutton`, `incrementor`, `menubutton`, `switch`, `colorwell`, `dockitem`.
Stale ref recovery:
```text
snapshot -> act -> STALE_REF? -> snapshot again -> retry
```
### Is agent-desktop free and open source?
Yes. agent-desktop is Apache-2.0 licensed for personal and commercial use.
### Where can I get help?
| Resource | Link |
|----------|------|
| **Repository** | [github.com/lahfir/agent-desktop](https://github.com/lahfir/agent-desktop) |
| **ClawHub Skill** | [clawhub.ai/lahfir/agent-desktop](https://clawhub.ai/lahfir/agent-desktop) |
| **skills.sh Listing** | [skills.sh/lahfir/agent-desktop/agent-desktop](https://skills.sh/lahfir/agent-desktop/agent-desktop) |
| **npm Package** | [npmjs.com/package/agent-desktop](https://www.npmjs.com/package/agent-desktop) |
| **CI Status** | [GitHub Actions](https://github.com/lahfir/agent-desktop/actions/workflows/ci.yml?query=branch%3Amain) |
| **Releases** | [GitHub Releases](https://github.com/lahfir/agent-desktop/releases) |
| **Issues** | [GitHub Issues](https://github.com/lahfir/agent-desktop/issues) |
See the [complete FAQ](docs/faq.md) for architecture, platform support, installation, refs, licensing, and support links.
## License

46
benchmarks/locator-resolution/README.md Normal file
View file

@ -0,0 +1,46 @@
# Locator resolution benchmark
This deterministic harness compares the legacy full-snapshot locator path with
the handle-free observed-tree evaluator over synthetic Chromium/Electron accessibility
trees. It exercises deep anonymous wrapper chains, duplicate role/name
candidates, moving bounds, simultaneous `AXIdentifier` and `AXDOMIdentifier`
values, containment predicates, and large trees.
Run it without network access:
```bash
rtk cargo run -p agent-desktop-core --release --example locator_benchmark \
> /private/tmp/agent-desktop-locator-synthetic.json
```
The JSON report includes 31-run p50/p95 wall-clock latency, candidate nodes,
predicate work, requested synthetic attribute values, cardinality, and
correctness. `live_find_selected_refs` uses the same selected-match-only
materialization contract as the default CLI `find` path. `live_arena_direct` is
reported separately as a lower-overhead direct-target metric, and
`live_count_no_refmap` verifies that count-only requests perform no action or
settable evidence reads. The harness does not claim absolute native AX calls:
role-dependent action and settable probes make that model inaccurate. Native
Slack/Electron IPC measurements require Accessibility permission and belong in
the privileged macOS integration suite.
This synthetic benchmark JSON is consumed by the repo-wide performance report:
run `bash scripts/perf-baseline-compare.sh` (optionally with `--apps "Slack,Google
Chrome"`) to produce `report.html`, covering the HEAD-vs-main fixture A/B,
optional real-app read-only probes, and this synthetic benchmark.
For privileged Electron/Chromium runs, the macOS adapter first asks whether the
application root exposes `AXManualAccessibility` as settable, reads its current
value, and sets it at most once when supported. The adapter then waits within
the request's existing absolute deadline until the attribute reports ready,
recording attempted/succeeded/ready activation in locator stats. Unsupported
native applications are left unchanged. This follows
[Electron's official third-party accessibility guidance](https://github.com/electron/electron/blob/main/docs/tutorial/accessibility.md),
which documents setting `AXManualAccessibility` from native assistive software
to expose Chromium's accessibility tree before automatic assistive-technology
detection has enabled it. The
[current Electron macOS implementation](https://github.com/electron/electron/blob/main/shell/browser/mac/electron_application.mm)
advertises the attribute as settable, reports `true` only in complete mode, and
handles `AXEnhancedUserInterface` separately; that source behavior is why the
benchmark methodology requires a post-set readiness read instead of treating a
successful setter as immediate readiness.

10
crates/core/Cargo.toml
View file

@ -15,5 +15,15 @@ base64.workspace = true
libc.workspace = true
smallvec.workspace = true
[target.'cfg(windows)'.dependencies]
windows-sys = { workspace = true, features = [
"Win32_Foundation",
"Win32_Security",
"Win32_Security_Authorization",
"Win32_Storage_FileSystem",
"Win32_System_SystemServices",
"Win32_System_Threading",
] }
[lints]
workspace = true

283
crates/core/examples/locator_benchmark.rs Normal file
View file

@ -0,0 +1,283 @@
#[path = "locator_benchmark/adapter.rs"]
mod adapter;
#[path = "locator_benchmark/fixture.rs"]
mod fixture;
#[path = "locator_benchmark/fixture_builder.rs"]
mod fixture_builder;
#[path = "locator_benchmark/fixture_node.rs"]
mod fixture_node;
#[path = "locator_benchmark/legacy.rs"]
mod legacy;
#[path = "locator_benchmark/live.rs"]
mod live;
#[path = "locator_benchmark/scenario.rs"]
mod scenario;
#[path = "locator_benchmark/scenarios.rs"]
mod scenarios;
use crate::{
legacy::run_legacy,
live::{run_live_count, run_live_direct, run_live_find},
scenario::Scenario,
};
use serde_json::{Value, json};
use std::{error::Error, hint::black_box};
const WARMUP_RUNS: usize = 5;
const MEASURED_RUNS: usize = 31;
fn main() -> Result<(), Box<dyn Error>> {
let reports = scenarios::all()
.iter()
.map(benchmark_scenario)
.collect::<Result<Vec<_>, _>>()?;
let report = json!({
"schema_version": "1.3",
"benchmark": "locator-resolution-electron-synthetic",
"methodology": {
"warmup_runs": WARMUP_RUNS,
"measured_runs": MEASURED_RUNS,
"timing": "wall clock around fixture-to-result resolution; fixture generation excluded",
"legacy_path": "full AccessibilityNode snapshot, ref allocation, recursive matcher",
"live_direct_path": "handle-free observed tree, strict direct target selection without ref-map materialization, memoized evaluator",
"live_find_path": "CLI-compatible default find selection with selected-match-only ref materialization from the same observed tree",
"live_count_path": "CLI-compatible count selection without ref evidence or ref-map materialization",
"read_accounting": "per-path requested AX attribute slots plus action, child-label, promotion, and settable probe classes from locator stats",
"native_ipc": "not measured; native action and settable probe counts vary by role",
"scope": "deterministic synthetic Chromium/Electron accessibility fixtures; CPU and requested synthetic attributes only",
},
"scenarios": reports,
});
println!("{}", serde_json::to_string_pretty(&report)?);
Ok(())
}
fn benchmark_scenario(scenario: &Scenario) -> Result<Value, Box<dyn Error>> {
for run in 0..WARMUP_RUNS {
let fixture = scenario.frame(run);
black_box(run_legacy(fixture, &scenario.query)?);
black_box(run_live_direct(fixture, &scenario.query)?);
black_box(run_live_find(fixture, &scenario.query)?);
black_box(run_live_count(fixture, &scenario.query)?);
}
let mut legacy_ns = Vec::with_capacity(MEASURED_RUNS);
let mut direct_ns = Vec::with_capacity(MEASURED_RUNS);
let mut find_ns = Vec::with_capacity(MEASURED_RUNS);
let mut count_ns = Vec::with_capacity(MEASURED_RUNS);
let mut legacy_counts = Vec::with_capacity(MEASURED_RUNS);
let mut direct_counts = Vec::with_capacity(MEASURED_RUNS);
let mut find_counts = Vec::with_capacity(MEASURED_RUNS);
let mut count_counts = Vec::with_capacity(MEASURED_RUNS);
let mut legacy_scanned = 0_u64;
let mut direct_scanned = 0_u64;
let mut find_scanned = 0_u64;
let mut legacy_predicate_visits = 0_u64;
let mut direct_predicate_cells = 0_u64;
let mut find_predicate_cells = 0_u64;
let mut direct_dom_matches = 0_u64;
let mut find_dom_matches = 0_u64;
let mut find_ref_count = 0_usize;
let mut count_action_reads = 0_u64;
let mut direct_action_reads = 0_u64;
let mut find_action_reads = 0_u64;
let mut direct_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
let mut find_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
let mut count_reads = (0_u64, 0_u64, 0_u64, 0_u64, 0_u64, 0_u64);
let mut find_refs_reresolvable = true;
for run in 0..MEASURED_RUNS {
let fixture = scenario.frame(run);
let (legacy, direct, find, count) = match run % 4 {
0 => (
run_legacy(fixture, &scenario.query)?,
run_live_direct(fixture, &scenario.query)?,
run_live_find(fixture, &scenario.query)?,
run_live_count(fixture, &scenario.query)?,
),
1 => {
let direct = run_live_direct(fixture, &scenario.query)?;
let find = run_live_find(fixture, &scenario.query)?;
let count = run_live_count(fixture, &scenario.query)?;
let legacy = run_legacy(fixture, &scenario.query)?;
(legacy, direct, find, count)
}
2 => {
let find = run_live_find(fixture, &scenario.query)?;
let count = run_live_count(fixture, &scenario.query)?;
let legacy = run_legacy(fixture, &scenario.query)?;
let direct = run_live_direct(fixture, &scenario.query)?;
(legacy, direct, find, count)
}
_ => {
let count = run_live_count(fixture, &scenario.query)?;
let legacy = run_legacy(fixture, &scenario.query)?;
let direct = run_live_direct(fixture, &scenario.query)?;
let find = run_live_find(fixture, &scenario.query)?;
(legacy, direct, find, count)
}
};
legacy_ns.push(legacy.0);
direct_ns.push(direct.elapsed_ns);
find_ns.push(find.elapsed_ns);
count_ns.push(count.elapsed_ns);
legacy_counts.push(legacy.1);
direct_counts.push(direct.correctness.matches);
find_counts.push(find.correctness.matches);
count_counts.push(count.correctness.matches);
legacy_scanned = legacy.2;
legacy_predicate_visits = legacy.3;
direct_scanned = direct.visited;
direct_predicate_cells = direct.memo_cells;
direct_dom_matches = direct.dom_matches;
find_scanned = find.visited;
find_predicate_cells = find.memo_cells;
find_dom_matches = find.dom_matches;
find_ref_count = find.ref_count;
count_action_reads = count.action_reads;
direct_action_reads = direct.action_reads;
find_action_reads = find.action_reads;
direct_reads = read_counts(&direct);
find_reads = read_counts(&find);
count_reads = read_counts(&count);
find_refs_reresolvable &= find.correctness.selected_refs_reresolvable;
}
let expected = scenario.expected_matches;
let legacy_correct = legacy_counts.iter().all(|count| *count == expected);
let direct_correct = direct_counts.iter().all(|count| *count == expected);
let find_correct = find_counts.iter().all(|count| *count == expected);
let count_correct = count_counts.iter().all(|count| *count == expected);
let legacy_p50 = percentile(&mut legacy_ns, 50);
let legacy_p95 = percentile(&mut legacy_ns, 95);
let direct_p50 = percentile(&mut direct_ns, 50);
let direct_p95 = percentile(&mut direct_ns, 95);
let find_p50 = percentile(&mut find_ns, 50);
let find_p95 = percentile(&mut find_ns, 95);
let count_p50 = percentile(&mut count_ns, 50);
let count_p95 = percentile(&mut count_ns, 95);
let fixture = scenario.frame(0);
let legacy_attributes_requested = fixture.nodes.len() as u64 * 22;
Ok(json!({
"name": scenario.name,
"fixture": {
"frames": scenario.frames.len(),
"nodes": fixture.nodes.len(),
"roots": fixture.roots.len(),
"moving_bounds": scenario.moving_bounds_verified(),
"legacy_locator_result_retained_handles": fixture.nodes.len(),
"observed_tree_retained_handles": 0,
},
"expectation": {
"matches": expected,
"cardinality": cardinality(expected),
},
"legacy_snapshot": {
"p50_us": ns_to_us(legacy_p50),
"p95_us": ns_to_us(legacy_p95),
"observed_matches": legacy_counts[0],
"correct_all_runs": legacy_correct,
"candidate_nodes_scanned": legacy_scanned,
"predicate_node_visits": legacy_predicate_visits,
"attributes_requested": legacy_attributes_requested,
"attribute_batches": fixture.nodes.len(),
"action_reads": fixture.nodes.len(),
},
"live_arena_direct": {
"p50_us": ns_to_us(direct_p50),
"p95_us": ns_to_us(direct_p95),
"observed_matches": direct_counts[0],
"correct_all_runs": direct_correct,
"candidate_nodes_scanned": direct_scanned,
"memo_cells_evaluated": direct_predicate_cells,
"native_id_dom_matches": direct_dom_matches,
"attributes_requested": direct_reads.0,
"attribute_batches": direct_reads.1,
"child_label_reads": direct_reads.2,
"promotion_reads": direct_reads.3,
"settable_reads": direct_reads.4,
"action_reads": direct_action_reads,
"peak_handles_owned": direct_reads.5,
},
"live_find_selected_refs": {
"p50_us": ns_to_us(find_p50),
"p95_us": ns_to_us(find_p95),
"observed_matches": find_counts[0],
"correct_all_runs": find_correct,
"candidate_nodes_scanned": find_scanned,
"memo_cells_evaluated": find_predicate_cells,
"native_id_dom_matches": find_dom_matches,
"ref_count": find_ref_count,
"selected_refs_reresolvable": find_refs_reresolvable,
"attributes_requested": find_reads.0,
"attribute_batches": find_reads.1,
"child_label_reads": find_reads.2,
"promotion_reads": find_reads.3,
"settable_reads": find_reads.4,
"action_reads": find_action_reads,
"peak_handles_owned": find_reads.5,
},
"live_count_no_refmap": {
"p50_us": ns_to_us(count_p50),
"p95_us": ns_to_us(count_p95),
"observed_matches": count_counts[0],
"correct_all_runs": count_correct,
"action_reads": count_action_reads,
"ref_count": 0,
"attributes_requested": count_reads.0,
"attribute_batches": count_reads.1,
"child_label_reads": count_reads.2,
"promotion_reads": count_reads.3,
"settable_reads": count_reads.4,
"peak_handles_owned": count_reads.5,
},
"comparison": {
"p50_find_speedup": ratio(legacy_p50, find_p50),
"p95_find_speedup": ratio(legacy_p95, find_p95),
"p50_direct_target_speedup": ratio(legacy_p50, direct_p50),
"p95_direct_target_speedup": ratio(legacy_p95, direct_p95),
"p50_count_speedup": ratio(legacy_p50, count_p50),
"p95_count_speedup": ratio(legacy_p95, count_p95),
"find_correctness_delta": i8::from(find_correct) - i8::from(legacy_correct),
"direct_target_correctness_delta": i8::from(direct_correct) - i8::from(legacy_correct),
"count_correctness_delta": i8::from(count_correct) - i8::from(legacy_correct),
},
}))
}
fn read_counts(run: &live::LiveRun) -> (u64, u64, u64, u64, u64, u64) {
(
run.attributes_requested,
run.attribute_batches,
run.child_label_reads,
run.promotion_reads,
run.settable_reads,
run.peak_handles_owned,
)
}
fn percentile(values: &mut [u128], percentile: usize) -> u128 {
values.sort_unstable();
let index = (values.len() * percentile).div_ceil(100).saturating_sub(1);
values[index]
}
fn ns_to_us(nanoseconds: u128) -> f64 {
nanoseconds as f64 / 1_000.0
}
fn ratio(before: u128, after: u128) -> f64 {
if after == 0 {
return 0.0;
}
before as f64 / after as f64
}
fn cardinality(matches: usize) -> &'static str {
match matches {
0 => "zero",
1 => "one",
_ => "many",
}
}

173
crates/core/examples/locator_benchmark/adapter.rs Normal file
View file

@ -0,0 +1,173 @@
use crate::{
fixture::Fixture,
fixture_builder::{live_target_tree, live_tree},
};
use agent_desktop_core::{
ActionOps, AdapterError, Deadline, ElementIdentifier, ErrorCode, IdentifierKind, InputOps,
NativeHandle, ObservationOps, ObservationRequest, ObservationRoot, ObservationSource,
ObservedTree, RefEntry, SystemOps, WindowFilter, WindowInfo,
};
pub(crate) struct FixtureAdapter<'a> {
pub fixture: &'a Fixture,
}
impl ActionOps for FixtureAdapter<'_> {}
impl InputOps for FixtureAdapter<'_> {}
impl SystemOps for FixtureAdapter<'_> {}
impl ObservationOps for FixtureAdapter<'_> {
fn list_windows(
&self,
_filter: &WindowFilter,
_deadline: Deadline,
) -> Result<Vec<WindowInfo>, AdapterError> {
Ok(vec![self.fixture.window.clone()])
}
fn observe_tree(
&self,
root: ObservationRoot<'_>,
request: &ObservationRequest,
) -> Result<ObservedTree, AdapterError> {
match &root {
ObservationRoot::Window(_) => {
live_tree(self.fixture, request.evidence_for_raw_depth(0))
}
ObservationRoot::Element { entry, .. } => {
let index = fixture_index_for_path(self.fixture, entry.scope.path.as_slice())?;
live_target_tree(
self.fixture,
index,
ObservationSource::from_root(&root),
request,
)
}
}
}
fn resolve_locator_anchor(
&self,
entry: &RefEntry,
deadline: Deadline,
) -> Result<NativeHandle, AdapterError> {
if deadline.is_expired() {
return Err(AdapterError::timeout(
"benchmark locator anchor deadline expired",
));
}
verify_source(self.fixture, entry)?;
let index = fixture_index_for_path(self.fixture, entry.scope.path.as_slice())?;
let node = self
.fixture
.nodes
.get(index as usize)
.ok_or_else(|| stale_anchor("benchmark locator path is invalid"))?;
if node.role != entry.identity.role {
return Err(stale_anchor("benchmark locator role changed"));
}
let identity_matches = entry
.identity
.native_id
.as_ref()
.is_some_and(|identifier| identifier_matches(node, identifier));
let bounds_match = entry.geometry.bounds_hash.is_some()
&& entry.geometry.bounds_hash == node.bounds.bounds_hash();
if !identity_matches && !bounds_match {
return Err(stale_anchor("benchmark locator anchor changed"));
}
Ok(NativeHandle::null())
}
}
fn verify_source(fixture: &Fixture, entry: &RefEntry) -> Result<(), AdapterError> {
let window = &fixture.window;
let process_matches = entry.process.pid == window.pid
&& entry.process.process_instance == window.process_instance;
let window_matches = entry.source.source_window_id.as_deref() == Some(window.id.as_str())
&& entry.source.source_app.as_deref() == Some(window.app.as_str());
if process_matches && window_matches {
Ok(())
} else {
Err(stale_anchor("benchmark locator source changed"))
}
}
fn identifier_matches(
node: &crate::fixture_node::FixtureNode,
identifier: &ElementIdentifier,
) -> bool {
let value = identifier.value.as_str();
match identifier.kind {
IdentifierKind::AxIdentifier => node.identifiers.0.as_deref() == Some(value),
IdentifierKind::AxDomIdentifier => node.identifiers.1.as_deref() == Some(value),
IdentifierKind::Unknown => {
node.identifiers.0.as_deref() == Some(value)
|| node.identifiers.1.as_deref() == Some(value)
}
IdentifierKind::AutomationId
| IdentifierKind::RuntimeId
| IdentifierKind::AtspiObjectPath => false,
}
}
fn stale_anchor(message: &str) -> AdapterError {
AdapterError::new(ErrorCode::StaleRef, message)
}
fn fixture_index_for_path(fixture: &Fixture, path: &[usize]) -> Result<u32, AdapterError> {
let mut index = *fixture
.roots
.first()
.ok_or_else(|| AdapterError::internal("benchmark fixture has no root"))?;
for child_order in path {
let node = fixture
.nodes
.get(index as usize)
.ok_or_else(|| AdapterError::internal("benchmark fixture path is invalid"))?;
index = *node
.children
.get(*child_order)
.ok_or_else(|| AdapterError::internal("benchmark fixture path is invalid"))?;
}
Ok(index)
}
#[cfg(test)]
mod tests {
#[test]
fn selected_find_hydrates_every_benchmark_frame() {
for scenario in crate::scenarios::all() {
for fixture in &scenario.frames {
let result = crate::live::run_live_find(fixture, &scenario.query)
.expect("selected benchmark find");
assert_eq!(
result.correctness.matches, scenario.expected_matches,
"{}",
scenario.name
);
assert!(
result.correctness.selected_refs_reresolvable,
"{}",
scenario.name
);
}
}
}
#[test]
fn selected_find_uses_the_exact_path_anchor() {
let scenario = crate::scenarios::all()
.into_iter()
.find(|scenario| scenario.name == "duplicate_button_role_and_name")
.expect("benchmark scenario");
let result = crate::live::run_live_find(scenario.frame(0), &scenario.query)
.expect("selected benchmark find");
assert_eq!(result.correctness.matches, scenario.expected_matches);
assert!(result.correctness.selected_refs_reresolvable);
assert_eq!(result.ref_count, 50);
}
}

9
crates/core/examples/locator_benchmark/fixture.rs Normal file
View file

@ -0,0 +1,9 @@
use crate::fixture_node::FixtureNode;
use agent_desktop_core::WindowInfo;
#[derive(Clone)]
pub(crate) struct Fixture {
pub nodes: Vec<FixtureNode>,
pub roots: Vec<u32>,
pub window: WindowInfo,
}

318
crates/core/examples/locator_benchmark/fixture_builder.rs Normal file
View file

@ -0,0 +1,318 @@
use crate::{fixture::Fixture, fixture_node::FixtureNode};
use agent_desktop_core::{
AdapterError, ElementIdentifier, EvidenceRequirements, IdentifierEvidence, IdentifierKind,
LocatorEvidence, LocatorField, LocatorIdentifierStats, LocatorReadCounts, LocatorReadStats,
LocatorRefEvidence, LocatorSemanticReadStats, LocatorStats, LocatorTraversalStats,
ObservationRequest, ObservationSource, ObservedSubtree, ObservedTree,
};
pub(crate) fn live_tree(
fixture: &Fixture,
requirements: EvidenceRequirements,
) -> Result<ObservedTree, AdapterError> {
live_tree_from_roots(
fixture,
&fixture.roots,
ObservationSource::Window(fixture.window.clone()),
requirements,
)
}
pub(crate) fn live_tree_from_roots(
fixture: &Fixture,
root_indices: &[u32],
source: ObservationSource,
requirements: EvidenceRequirements,
) -> Result<ObservedTree, AdapterError> {
let roots = root_indices
.iter()
.map(|root| live_node(fixture, *root, requirements))
.collect::<Result<Vec<_>, _>>()?;
let observed_indices = reachable_indices(fixture, root_indices)?;
ObservedTree::from_roots(
roots,
source,
stats_for_indices(fixture, &observed_indices, requirements),
true,
)
}
pub(crate) fn live_target_tree(
fixture: &Fixture,
index: u32,
source: ObservationSource,
request: &ObservationRequest,
) -> Result<ObservedTree, AdapterError> {
let mut reads = Vec::new();
let root = live_target_node(fixture, index, request, 0, &mut reads)?;
ObservedTree::from_roots(vec![root], source, stats_for_reads(fixture, &reads), true)
}
fn live_target_node(
fixture: &Fixture,
index: u32,
request: &ObservationRequest,
raw_depth: u8,
reads: &mut Vec<(usize, EvidenceRequirements)>,
) -> Result<ObservedSubtree, AdapterError> {
let node = fixture
.nodes
.get(index as usize)
.ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?;
let requirements = request.evidence_for_raw_depth(raw_depth);
reads.push((index as usize, requirements));
let at_boundary = raw_depth >= request.max_logical_depth || raw_depth >= request.max_raw_depth;
let children = if at_boundary {
Vec::new()
} else {
node.children
.iter()
.map(|child| {
live_target_node(fixture, *child, request, raw_depth.saturating_add(1), reads)
})
.collect::<Result<Vec<_>, _>>()?
};
let children_count = at_boundary
.then(|| u32::try_from(node.children.len()).ok())
.flatten()
.filter(|count| *count > 0);
Ok(ObservedSubtree::new(
node_evidence(node, requirements),
children,
true,
children_count,
))
}
fn live_node(
fixture: &Fixture,
index: u32,
requirements: EvidenceRequirements,
) -> Result<ObservedSubtree, AdapterError> {
let node = fixture
.nodes
.get(index as usize)
.ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?;
let children = node
.children
.iter()
.map(|child| live_node(fixture, *child, requirements))
.collect::<Result<Vec<_>, _>>()?;
Ok(ObservedSubtree::new(
node_evidence(node, requirements),
children,
true,
None,
))
}
fn node_evidence(node: &FixtureNode, requirements: EvidenceRequirements) -> LocatorEvidence {
LocatorEvidence {
role: LocatorField::Known(node.role.clone()),
name: requested_field(requirements.name, node.name.clone()),
description: requested_field(requirements.description, None),
value: requested_field(requirements.value, None),
identifiers: if requirements.identifiers {
IdentifierEvidence::typed(
[
node.identifiers.0.clone().map(|value| ElementIdentifier {
kind: IdentifierKind::AxIdentifier,
value,
}),
node.identifiers.1.clone().map(|value| ElementIdentifier {
kind: IdentifierKind::AxDomIdentifier,
value,
}),
]
.into_iter()
.flatten(),
Some(0),
true,
)
} else {
IdentifierEvidence::unknown()
},
states: if requirements.states {
LocatorField::Known(Vec::new())
} else {
LocatorField::Unknown
},
ref_evidence: LocatorRefEvidence {
bounds: if requirements.ref_evidence.bounds {
LocatorField::Known(node.bounds)
} else {
LocatorField::Unknown
},
available_actions: if requirements.ref_evidence.actions {
LocatorField::Known(actions(node))
} else {
LocatorField::Unknown
},
},
}
}
fn stats_for_indices(
fixture: &Fixture,
indices: &[usize],
requirements: EvidenceRequirements,
) -> LocatorStats {
let reads = indices
.iter()
.map(|index| (*index, requirements))
.collect::<Vec<_>>();
stats_for_reads(fixture, &reads)
}
fn stats_for_reads(fixture: &Fixture, reads: &[(usize, EvidenceRequirements)]) -> LocatorStats {
let count = reads.len() as u64;
let indices = reads.iter().map(|(index, _)| *index).collect::<Vec<_>>();
LocatorStats {
traversal: LocatorTraversalStats {
nodes_visited: count,
peak_handles_owned: 0,
max_raw_depth: 0,
max_logical_depth: 0,
web_wrapper_nodes: indices
.iter()
.filter_map(|index| fixture.nodes.get(*index))
.filter(|node| node.role == "group" && node.name.is_none())
.count() as u64,
..LocatorTraversalStats::default()
},
reads: LocatorReadStats {
counts: LocatorReadCounts {
attribute_batches: count,
attributes_requested: reads
.iter()
.map(|(_, requirements)| u64::from(attribute_count(*requirements)))
.sum(),
child_reads: count,
action_reads: reads
.iter()
.filter(|(_, requirements)| requirements.ref_evidence.actions)
.count() as u64,
..LocatorReadCounts::default()
},
..LocatorReadStats::default()
},
semantic_reads: LocatorSemanticReadStats {
settable_reads: reads
.iter()
.filter_map(|(index, requirements)| {
fixture
.nodes
.get(*index)
.map(|node| modeled_settable_read(node, *requirements))
})
.sum(),
..LocatorSemanticReadStats::default()
},
identifiers: identifier_stats(fixture, &indices),
..LocatorStats::default()
}
}
fn requested_field<T>(requested: bool, value: Option<T>) -> LocatorField<T> {
if requested {
value.map_or(LocatorField::Absent, LocatorField::Known)
} else {
LocatorField::Unknown
}
}
fn attribute_count(requirements: EvidenceRequirements) -> u32 {
let mut mask = 1_u32;
if requirements.name || requirements.description {
for index in [1, 2, 3, 15, 16, 17] {
mask |= 1_u32 << index;
}
}
if requirements.value {
mask |= 1_u32 << 3;
}
if requirements.identifiers {
mask |= (1_u32 << 13) | (1_u32 << 14);
}
if requirements.states {
for index in 3..=12 {
mask |= 1_u32 << index;
}
mask |= (1_u32 << 18) | (1_u32 << 19);
}
if requirements.ref_evidence.bounds {
for index in 18..=19 {
mask |= 1_u32 << index;
}
}
if requirements.ref_evidence.actions {
for index in 20..=21 {
mask |= 1_u32 << index;
}
}
mask.count_ones()
}
fn modeled_settable_read(node: &FixtureNode, requirements: EvidenceRequirements) -> u64 {
let state_read = u64::from(requirements.states && node.role == "textfield");
let action_reads = if requirements.ref_evidence.actions {
match node.role.as_str() {
"textfield" => 2,
"button" => 1,
_ => 0,
}
} else {
0
};
state_read + action_reads
}
fn reachable_indices(fixture: &Fixture, roots: &[u32]) -> Result<Vec<usize>, AdapterError> {
fn visit(fixture: &Fixture, index: u32, indices: &mut Vec<usize>) -> Result<(), AdapterError> {
let node = fixture
.nodes
.get(index as usize)
.ok_or_else(|| AdapterError::internal("benchmark fixture index is out of bounds"))?;
indices.push(index as usize);
for child in &node.children {
visit(fixture, *child, indices)?;
}
Ok(())
}
let mut indices = Vec::new();
for root in roots {
visit(fixture, *root, &mut indices)?;
}
Ok(indices)
}
fn actions(node: &FixtureNode) -> Vec<String> {
match node.role.as_str() {
"button" | "textfield" => vec!["Click".to_string()],
_ => Vec::new(),
}
}
fn identifier_stats(fixture: &Fixture, indices: &[usize]) -> LocatorIdentifierStats {
LocatorIdentifierStats {
values_observed: indices
.iter()
.filter_map(|index| fixture.nodes.get(*index))
.map(|node| {
u64::from(node.identifiers.0.is_some()) + u64::from(node.identifiers.1.is_some())
})
.sum(),
nodes_with_identifiers: indices
.iter()
.filter_map(|index| fixture.nodes.get(*index))
.filter(|node| node.identifiers.0.is_some() || node.identifiers.1.is_some())
.count() as u64,
nodes_with_multiple_identifiers: indices
.iter()
.filter_map(|index| fixture.nodes.get(*index))
.filter(|node| node.identifiers.0.is_some() && node.identifiers.1.is_some())
.count() as u64,
..LocatorIdentifierStats::default()
}
}

28
crates/core/examples/locator_benchmark/fixture_node.rs Normal file
View file

@ -0,0 +1,28 @@
use agent_desktop_core::Rect;
#[derive(Clone)]
pub(crate) struct FixtureNode {
pub role: String,
pub name: Option<String>,
pub identifiers: (Option<String>, Option<String>),
pub bounds: Rect,
pub children: Vec<u32>,
}
impl FixtureNode {
pub fn new(
role: &str,
name: Option<&str>,
identifiers: (Option<String>, Option<String>),
bounds: Rect,
children: Vec<u32>,
) -> Self {
Self {
role: role.to_string(),
name: name.map(str::to_string),
identifiers,
bounds,
children,
}
}
}

140
crates/core/examples/locator_benchmark/legacy.rs Normal file
View file

@ -0,0 +1,140 @@
use crate::{adapter::FixtureAdapter, fixture::Fixture};
use agent_desktop_core::{
AccessibilityNode, AppError, Deadline, LocatorQuery, accessibility_node_matches, snapshot,
};
use std::{hint::black_box, time::Instant};
pub(crate) fn run_legacy(
fixture: &Fixture,
query: &LocatorQuery,
) -> Result<(u128, usize, u64, u64), AppError> {
let adapter = FixtureAdapter { fixture };
let started = Instant::now();
let snapshot = snapshot::build(
&adapter,
&agent_desktop_core::TreeOptions::default(),
Some(&fixture.window.app),
None,
Deadline::after(5_000)?,
)?;
let mut scanned = 0_u64;
let matches = count_matches(&snapshot.tree, query, &mut scanned);
black_box((&snapshot.refmap, matches));
let elapsed = started.elapsed().as_nanos();
let mut predicate_visits = 0_u64;
instrument_predicate_visits(&snapshot.tree, query, &mut predicate_visits);
Ok((elapsed, matches, scanned, predicate_visits))
}
fn count_matches(node: &AccessibilityNode, query: &LocatorQuery, scanned: &mut u64) -> usize {
*scanned += 1;
usize::from(accessibility_node_matches(node, query))
+ node
.children
.iter()
.map(|child| count_matches(child, query, scanned))
.sum::<usize>()
}
fn instrument_predicate_visits(node: &AccessibilityNode, query: &LocatorQuery, visits: &mut u64) {
let _ = instrumented_match(node, query, visits);
for child in &node.children {
instrument_predicate_visits(child, query, visits);
}
}
fn instrumented_match(node: &AccessibilityNode, query: &LocatorQuery, visits: &mut u64) -> bool {
*visits += 1;
if !identity_matches(node, query) {
return false;
}
if let Some(expected) = query.has_text.as_deref() {
if !node_text_matches(node, expected) && !descendant_text_matches(node, expected, visits) {
return false;
}
}
if let Some(has) = query.containment.has.as_deref() {
if !descendant_query_matches(node, has, visits) {
return false;
}
}
if let Some(has_not) = query.containment.has_not.as_deref() {
if descendant_query_matches(node, has_not, visits) {
return false;
}
}
true
}
fn identity_matches(node: &AccessibilityNode, query: &LocatorQuery) -> bool {
query
.identity
.role
.as_deref()
.is_none_or(|expected| normalized(&node.role) == expected)
&& field_matches(
query.identity.name.as_deref(),
node.identity.name.as_deref(),
query.exact,
)
&& query.identity.native_id.as_deref().is_none_or(|expected| {
node.identity
.native_id
.as_ref()
.is_some_and(|actual| actual.value == expected)
})
}
fn field_matches(expected: Option<&str>, actual: Option<&str>, exact: bool) -> bool {
let Some(expected) = expected else {
return true;
};
let Some(actual) = actual else {
return false;
};
let actual = normalized(actual);
if exact {
actual == expected
} else {
actual.contains(expected)
}
}
fn descendant_text_matches(node: &AccessibilityNode, expected: &str, visits: &mut u64) -> bool {
for child in &node.children {
*visits += 1;
if node_text_matches(child, expected) || descendant_text_matches(child, expected, visits) {
return true;
}
}
false
}
fn descendant_query_matches(
node: &AccessibilityNode,
query: &LocatorQuery,
visits: &mut u64,
) -> bool {
node.children.iter().any(|child| {
instrumented_match(child, query, visits) || descendant_query_matches(child, query, visits)
})
}
fn node_text_matches(node: &AccessibilityNode, expected: &str) -> bool {
[
node.identity.name.as_deref(),
node.identity.description.as_deref(),
node.identity.value.as_deref(),
]
.into_iter()
.flatten()
.any(|value| normalized(value).contains(expected))
}
fn normalized(value: &str) -> String {
value
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_lowercase()
}

148
crates/core/examples/locator_benchmark/live.rs Normal file
View file

@ -0,0 +1,148 @@
use crate::{adapter::FixtureAdapter, fixture::Fixture};
use agent_desktop_core::{
AppError, Deadline, LocatorMaterialization, LocatorQuery, LocatorResolution,
LocatorResolveRequest, LocatorSelection, ObservationRoot, resolve_query,
};
use std::{hint::black_box, time::Instant};
pub(crate) struct LiveCorrectness {
pub(crate) matches: usize,
pub(crate) selected_refs_reresolvable: bool,
}
pub(crate) struct LiveRun {
pub(crate) elapsed_ns: u128,
pub(crate) correctness: LiveCorrectness,
pub(crate) visited: u64,
pub(crate) memo_cells: u64,
pub(crate) dom_matches: u64,
pub(crate) ref_count: usize,
pub(crate) action_reads: u64,
pub(crate) attributes_requested: u64,
pub(crate) attribute_batches: u64,
pub(crate) child_label_reads: u64,
pub(crate) promotion_reads: u64,
pub(crate) settable_reads: u64,
pub(crate) peak_handles_owned: u64,
}
pub(crate) fn run_live_direct(
fixture: &Fixture,
query: &LocatorQuery,
) -> Result<LiveRun, AppError> {
run_live(
fixture,
query,
LocatorSelection::Strict,
LocatorMaterialization::None,
)
}
pub(crate) fn run_live_find(fixture: &Fixture, query: &LocatorQuery) -> Result<LiveRun, AppError> {
run_live(
fixture,
query,
LocatorSelection::All { limit: Some(50) },
LocatorMaterialization::SelectedMatches,
)
}
pub(crate) fn run_live_count(fixture: &Fixture, query: &LocatorQuery) -> Result<LiveRun, AppError> {
run_live(
fixture,
query,
LocatorSelection::Count,
LocatorMaterialization::None,
)
}
fn run_live(
fixture: &Fixture,
query: &LocatorQuery,
selection: LocatorSelection,
materialization: LocatorMaterialization,
) -> Result<LiveRun, AppError> {
let request = LocatorResolveRequest {
selection,
deadline: Deadline::after(5_000)?,
max_raw_depth: 50,
materialization,
};
let adapter = FixtureAdapter { fixture };
let started = Instant::now();
let resolution = resolve_query(
&adapter,
query,
ObservationRoot::Window(&fixture.window),
&request,
)?;
black_box((&resolution.refmap, &resolution.matches));
let elapsed = started.elapsed().as_nanos();
let ref_count = resolution.refmap.as_ref().map_or(0, |refmap| refmap.len());
let selected_refs_reresolvable = selected_refs_reresolvable(fixture, &resolution);
Ok(LiveRun {
elapsed_ns: elapsed,
correctness: LiveCorrectness {
matches: resolution.meta.total_matches as usize,
selected_refs_reresolvable,
},
visited: resolution.stats.traversal.nodes_visited,
memo_cells: resolution.stats.evaluation.memo_cells_evaluated,
dom_matches: resolution.stats.identifiers.fallback_matches,
ref_count,
action_reads: resolution.stats.reads.counts.action_reads,
attributes_requested: resolution.stats.reads.counts.attributes_requested,
attribute_batches: resolution.stats.reads.counts.attribute_batches,
child_label_reads: resolution.stats.semantic_reads.child_label_reads,
promotion_reads: resolution.stats.semantic_reads.promotion_reads,
settable_reads: resolution.stats.semantic_reads.settable_reads,
peak_handles_owned: resolution.stats.traversal.peak_handles_owned,
})
}
fn selected_refs_reresolvable(fixture: &Fixture, resolution: &LocatorResolution) -> bool {
let Some(refmap) = resolution.refmap.as_ref() else {
return true;
};
resolution
.matches
.iter()
.filter_map(|selected| selected.data.ref_id.as_deref())
.all(|ref_id| {
let Some(entry) = refmap.get(ref_id) else {
return false;
};
fixture.nodes.iter().any(|node| {
entry.geometry.bounds_hash == node.bounds.bounds_hash()
&& node.role == entry.identity.role
&& entry.identity.native_id.as_ref().map_or_else(
|| entry.identity.name == node.name,
|expected| {
node.identifiers.0.as_deref() == Some(expected.value.as_str())
|| node.identifiers.1.as_deref() == Some(expected.value.as_str())
},
)
})
})
}
#[cfg(test)]
mod tests {
use agent_desktop_core::Rect;
#[test]
fn duplicate_names_require_the_selected_nodes_bounds() {
let selected = Rect {
x: 10.0,
y: 20.0,
width: 30.0,
height: 40.0,
};
let sibling = Rect {
x: 50.0,
..selected
};
assert_ne!(selected.bounds_hash(), sibling.bounds_hash());
}
}

45
crates/core/examples/locator_benchmark/scenario.rs Normal file
View file

@ -0,0 +1,45 @@
use crate::fixture::Fixture;
use agent_desktop_core::{LocatorQuery, Rect};
use std::collections::BTreeSet;
pub(crate) struct Scenario {
pub name: &'static str,
pub frames: Vec<Fixture>,
pub query: LocatorQuery,
pub expected_matches: usize,
}
impl Scenario {
pub fn frame(&self, run: usize) -> &Fixture {
&self.frames[run % self.frames.len()]
}
pub fn moving_bounds_verified(&self) -> bool {
let bounds = self
.frames
.iter()
.flat_map(target_bounds)
.map(rect_key)
.collect::<BTreeSet<_>>();
bounds.len() > 1
}
}
fn target_bounds(fixture: &Fixture) -> impl Iterator<Item = Rect> + '_ {
fixture.nodes.iter().filter_map(|node| {
node.identifiers
.1
.as_deref()
.is_some_and(|value| value == "composer-send")
.then_some(node.bounds)
})
}
fn rect_key(bounds: Rect) -> (i64, i64, i64, i64) {
(
(bounds.x * 100.0) as i64,
(bounds.y * 100.0) as i64,
(bounds.width * 100.0) as i64,
(bounds.height * 100.0) as i64,
)
}

239
crates/core/examples/locator_benchmark/scenarios.rs Normal file
View file

@ -0,0 +1,239 @@
use crate::{fixture::Fixture, fixture_node::FixtureNode, scenario::Scenario};
use agent_desktop_core::{ContainmentPredicate, IdentityPredicate, LocatorQuery, Rect, WindowInfo};
const MOVING_FRAMES: usize = 11;
pub(crate) fn all() -> Vec<Scenario> {
vec![
deep_anonymous_containment(),
duplicate_role_and_name(),
electron_dual_identifier_moving_bounds(),
large_electron_channel_tree(),
]
}
fn deep_anonymous_containment() -> Scenario {
let fixture = chain_fixture(64, 48);
Scenario {
name: "deep_anonymous_has_text",
frames: vec![fixture],
query: LocatorQuery {
identity: IdentityPredicate {
role: Some("group".into()),
name: Some("layer".into()),
..IdentityPredicate::default()
},
has_text: Some("needle".into()),
..LocatorQuery::default()
},
expected_matches: 64 * 24,
}
}
fn duplicate_role_and_name() -> Scenario {
let fixture = button_fixture(0, false);
Scenario {
name: "duplicate_button_role_and_name",
frames: vec![fixture],
query: LocatorQuery {
identity: IdentityPredicate {
role: Some("button".into()),
name: Some("send".into()),
..IdentityPredicate::default()
},
exact: true,
..LocatorQuery::default()
},
expected_matches: 512,
}
}
fn electron_dual_identifier_moving_bounds() -> Scenario {
let frames = (0..MOVING_FRAMES)
.map(|frame| button_fixture(frame, true))
.collect();
Scenario {
name: "electron_dual_identifier_moving_bounds",
frames,
query: LocatorQuery {
identity: IdentityPredicate {
role: Some("button".into()),
native_id: Some("composer-send".into()),
..IdentityPredicate::default()
},
..LocatorQuery::default()
},
expected_matches: 1,
}
}
fn large_electron_channel_tree() -> Scenario {
let fixture = channel_fixture(640, 8);
Scenario {
name: "large_nested_channel_has_unread",
frames: vec![fixture],
query: LocatorQuery {
identity: IdentityPredicate {
role: Some("group".into()),
name: Some("channel".into()),
..IdentityPredicate::default()
},
containment: ContainmentPredicate {
has: Some(Box::new(LocatorQuery {
identity: IdentityPredicate {
name: Some("unread".into()),
..IdentityPredicate::default()
},
exact: true,
..LocatorQuery::default()
})),
has_not: None,
},
exact: true,
..LocatorQuery::default()
},
expected_matches: 10,
}
}
fn chain_fixture(chains: usize, depth: usize) -> Fixture {
let mut nodes = vec![node("group", Some("Electron Root"), 0.0, Vec::new())];
for chain in 0..chains {
let root = push(
&mut nodes,
node("group", Some("Layer"), chain as f64, Vec::new()),
);
nodes[0].children.push(root);
let mut parent = root;
for level in 1..depth {
let name = (level % 2 == 0).then_some("Layer");
let child = push(
&mut nodes,
node("group", name, (chain * depth + level) as f64, Vec::new()),
);
nodes[parent as usize].children.push(child);
parent = child;
}
let needle = push(
&mut nodes,
node("statictext", Some("Needle"), chain as f64, Vec::new()),
);
nodes[parent as usize].children.push(needle);
}
Fixture {
nodes,
roots: vec![0],
window: window(),
}
}
fn button_fixture(frame: usize, dual_identifier: bool) -> Fixture {
let mut nodes = vec![node("group", Some("Composer"), 0.0, Vec::new())];
let target_position = frame * 37 % 512;
for position in 0..512 {
let logical = if position == target_position {
0
} else {
position + 1
};
let identifiers = if position == target_position && dual_identifier {
(
Some("AXNode-8842".to_string()),
Some("composer-send".to_string()),
)
} else {
(
Some(format!("AXNode-{logical}")),
Some(format!("send-{logical}")),
)
};
let bounds = Rect {
x: (position % 32) as f64 * 18.0 + frame as f64 * 0.75,
y: (position / 32) as f64 * 22.0 + frame as f64 * 1.25,
width: 16.0,
height: 20.0,
};
let child = push(
&mut nodes,
FixtureNode::new("button", Some("Send"), identifiers, bounds, Vec::new()),
);
nodes[0].children.push(child);
}
Fixture {
nodes,
roots: vec![0],
window: window(),
}
}
fn channel_fixture(channels: usize, wrapper_depth: usize) -> Fixture {
let mut nodes = vec![node("group", Some("Slack"), 0.0, Vec::new())];
for channel in 0..channels {
let row = push(
&mut nodes,
node("group", Some("Channel"), channel as f64, Vec::new()),
);
nodes[0].children.push(row);
let mut parent = row;
for wrapper in 0..wrapper_depth {
let child = push(
&mut nodes,
node(
"group",
None,
(channel * wrapper_depth + wrapper) as f64,
Vec::new(),
),
);
nodes[parent as usize].children.push(child);
parent = child;
}
let label = (channel % 64 == 0).then_some("Unread");
let text = push(
&mut nodes,
node("statictext", label, channel as f64, Vec::new()),
);
nodes[parent as usize].children.push(text);
}
Fixture {
nodes,
roots: vec![0],
window: window(),
}
}
fn node(role: &str, name: Option<&str>, offset: f64, children: Vec<u32>) -> FixtureNode {
FixtureNode::new(
role,
name,
(None, None),
Rect {
x: offset % 1200.0,
y: offset % 800.0,
width: 100.0,
height: 24.0,
},
children,
)
}
fn push(nodes: &mut Vec<FixtureNode>, node: FixtureNode) -> u32 {
let index = nodes.len() as u32;
nodes.push(node);
index
}
fn window() -> WindowInfo {
WindowInfo {
id: "w-electron-benchmark".into(),
title: "Electron Benchmark".into(),
app: "SyntheticElectron".into(),
pid: agent_desktop_core::ProcessId::new(4242),
process_instance: Some("benchmark-4242".into()),
bounds: None,
state: agent_desktop_core::WindowState {
is_focused: true,
..Default::default()
},
}
}

39
crates/core/src/accname.rs Normal file
View file

@ -0,0 +1,39 @@
use crate::name_evidence::NameEvidence;
pub fn compute_name(evidence: &NameEvidence) -> Option<String> {
[
evidence.explicit_label.as_deref(),
evidence.labelled_by_text.as_deref(),
evidence.native_title.as_deref(),
evidence.static_value.as_deref(),
evidence.child_label.as_deref(),
evidence.placeholder.as_deref(),
evidence.description.as_deref(),
]
.into_iter()
.find_map(non_blank)
.map(str::to_string)
}
pub fn compute_description(evidence: &NameEvidence) -> Option<String> {
let description = non_blank(evidence.description.as_deref())?;
[
evidence.explicit_label.as_deref(),
evidence.labelled_by_text.as_deref(),
evidence.native_title.as_deref(),
evidence.static_value.as_deref(),
evidence.child_label.as_deref(),
evidence.placeholder.as_deref(),
]
.into_iter()
.any(|candidate| non_blank(candidate).is_some())
.then(|| description.to_string())
}
fn non_blank(value: Option<&str>) -> Option<&str> {
value.filter(|value| !value.trim().is_empty())
}
#[cfg(test)]
#[path = "accname_tests.rs"]
mod tests;

68
crates/core/src/accname_tests.rs Normal file
View file

@ -0,0 +1,68 @@
use super::*;
fn evidence() -> NameEvidence {
NameEvidence {
explicit_label: Some("explicit".into()),
labelled_by_text: Some("labelled".into()),
native_title: Some("title".into()),
static_value: Some("value".into()),
child_label: Some("child".into()),
placeholder: Some("placeholder".into()),
description: Some("description".into()),
}
}
#[test]
fn name_precedence_is_platform_neutral_and_complete() {
let mut value = evidence();
assert_eq!(compute_name(&value).as_deref(), Some("explicit"));
value.explicit_label = None;
assert_eq!(compute_name(&value).as_deref(), Some("labelled"));
value.labelled_by_text = None;
assert_eq!(compute_name(&value).as_deref(), Some("title"));
value.native_title = None;
assert_eq!(compute_name(&value).as_deref(), Some("value"));
value.static_value = None;
assert_eq!(compute_name(&value).as_deref(), Some("child"));
value.child_label = None;
assert_eq!(compute_name(&value).as_deref(), Some("placeholder"));
value.placeholder = None;
assert_eq!(compute_name(&value).as_deref(), Some("description"));
}
#[test]
fn blank_evidence_is_ignored_without_rewriting_content() {
let value = NameEvidence {
explicit_label: Some(" \t ".into()),
native_title: Some(" Save ".into()),
..NameEvidence::default()
};
assert_eq!(compute_name(&value).as_deref(), Some(" Save "));
}
#[test]
fn description_is_not_duplicated_when_it_supplies_the_name() {
let value = NameEvidence {
description: Some("Only description".into()),
..NameEvidence::default()
};
assert_eq!(compute_name(&value).as_deref(), Some("Only description"));
assert_eq!(compute_description(&value), None);
}
#[test]
fn description_remains_separate_when_an_earlier_name_exists() {
let value = NameEvidence {
placeholder: Some("Search".into()),
description: Some("Filters all messages".into()),
..NameEvidence::default()
};
assert_eq!(compute_name(&value).as_deref(), Some("Search"));
assert_eq!(
compute_description(&value).as_deref(),
Some("Filters all messages")
);
}

140
crates/core/src/action.rs
View file

@ -1,5 +1,7 @@
use serde::{Deserialize, Serialize};
use crate::{Direction, DragParams, KeyCombo};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Action {
Click,
@ -26,6 +28,32 @@ pub enum Action {
}
impl Action {
pub fn headed_requirement(&self) -> crate::HeadedRequirement {
match self {
Self::Click
| Self::DoubleClick
| Self::RightClick
| Self::TripleClick
| Self::Scroll(_, _)
| Self::Hover
| Self::Drag(_) => crate::HeadedRequirement::FocusedWindowAndCursor,
Self::SetValue(_)
| Self::SetFocus
| Self::Expand
| Self::Collapse
| Self::Select(_)
| Self::Toggle
| Self::Check
| Self::Uncheck
| Self::ScrollTo
| Self::PressKey(_)
| Self::KeyDown(_)
| Self::KeyUp(_)
| Self::TypeText(_)
| Self::Clear => crate::HeadedRequirement::FocusedWindow,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::Click => "click",
@ -56,17 +84,46 @@ impl Action {
matches!(self, Self::Hover | Self::Drag(_))
}
pub fn requires_hit_test(&self) -> bool {
matches!(
self,
Self::Click
| Self::DoubleClick
| Self::RightClick
| Self::TripleClick
| Self::Hover
| Self::Drag(_)
)
}
pub fn requires_scroll_into_view(&self) -> bool {
matches!(
self,
Self::Click
| Self::DoubleClick
| Self::RightClick
| Self::TripleClick
| Self::SetValue(_)
| Self::Expand
| Self::Collapse
| Self::Select(_)
| Self::Toggle
| Self::Check
| Self::Uncheck
| Self::TypeText(_)
| Self::Clear
| Self::Hover
| Self::Drag(_)
)
}
pub fn may_use_focus_fallback(&self) -> bool {
matches!(self, Self::TypeText(_) | Self::PressKey(_))
}
/// Returns the minimum `InteractionPolicy` the CLI uses for this action.
/// `TypeText` and `PressKey` require focus to land in the right field, so
/// their base is `focus_fallback`. Everything else is pure-AX and uses
/// `headless`. FFI callers join this base with their caller-supplied policy
/// so they can only elevate, never downgrade below CLI parity.
/// Returns the command's minimum interaction policy.
pub fn base_interaction_policy(&self) -> crate::interaction_policy::InteractionPolicy {
if self.may_use_focus_fallback() {
if matches!(self, Self::PressKey(_)) {
crate::interaction_policy::InteractionPolicy::focus_fallback()
} else {
crate::interaction_policy::InteractionPolicy::headless()
@ -74,77 +131,6 @@ impl Action {
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum Direction {
Up,
Down,
Left,
Right,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Point {
pub x: f64,
pub y: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MouseButton {
Left,
Right,
Middle,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DragParams {
pub from: Point,
pub to: Point,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
/// Time to hold over the destination before releasing. Some platforms require
/// a minimum dwell before the drop registers; `None` uses the adapter default.
#[serde(skip_serializing_if = "Option::is_none")]
pub drop_delay_ms: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MouseEventKind {
Move,
Down,
Up,
Click { count: u32 },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MouseEvent {
pub kind: MouseEventKind,
pub point: Point,
pub button: MouseButton,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum WindowOp {
Resize { width: f64, height: f64 },
Move { x: f64, y: f64 },
Minimize,
Maximize,
Restore,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KeyCombo {
pub key: String,
pub modifiers: Vec<Modifier>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum Modifier {
Cmd,
Ctrl,
Alt,
Shift,
}
#[cfg(test)]
#[path = "action_tests.rs"]
mod tests;

106
crates/core/src/action_request.rs
View file

@ -1,3 +1,4 @@
use crate::Point;
use crate::action::Action;
use crate::interaction_policy::InteractionPolicy;
use serde::{Deserialize, Serialize};
@ -6,13 +7,30 @@ use serde::{Deserialize, Serialize};
pub struct ActionRequest {
pub action: Action,
pub policy: InteractionPolicy,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_ms: Option<u64>,
#[serde(skip)]
pub(crate) verified_point: Option<Point>,
#[serde(skip)]
pub(crate) expected_process: Option<crate::ProcessIdentity>,
}
impl ActionRequest {
pub fn headed_requirement(&self) -> crate::HeadedRequirement {
if self.policy.is_headed() {
self.action.headed_requirement()
} else {
crate::HeadedRequirement::None
}
}
pub fn headless(action: Action) -> Self {
Self {
action,
policy: InteractionPolicy::headless(),
timeout_ms: None,
verified_point: None,
expected_process: None,
}
}
@ -20,6 +38,9 @@ impl ActionRequest {
Self {
action,
policy: InteractionPolicy::focus_fallback(),
timeout_ms: None,
verified_point: None,
expected_process: None,
}
}
@ -27,14 +48,40 @@ impl ActionRequest {
Self {
action,
policy: InteractionPolicy::headed(),
timeout_ms: None,
verified_point: None,
expected_process: None,
}
}
pub fn with_timeout_ms(mut self, timeout_ms: Option<u64>) -> Self {
self.timeout_ms = timeout_ms;
self
}
pub(crate) fn with_verified_point(mut self, point: Option<Point>) -> Self {
self.verified_point = point;
self
}
pub fn with_expected_process(mut self, process: crate::ProcessIdentity) -> Self {
self.expected_process = Some(process);
self
}
pub fn verified_point(&self) -> Option<&Point> {
self.verified_point.as_ref()
}
pub fn expected_process(&self) -> Option<&crate::ProcessIdentity> {
self.expected_process.as_ref()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::action::{Action, Direction};
use crate::{Action, Direction};
#[test]
fn default_policy_is_headless() {
@ -49,10 +96,67 @@ mod tests {
assert_eq!(request.policy, InteractionPolicy::headless());
}
#[test]
fn headed_requirement_is_inert_until_the_caller_selects_headed_mode() {
assert_eq!(
ActionRequest::headless(Action::Click).headed_requirement(),
crate::HeadedRequirement::None
);
assert_eq!(
ActionRequest::headed(Action::Click).headed_requirement(),
crate::HeadedRequirement::FocusedWindowAndCursor
);
assert_eq!(
ActionRequest::headed(Action::TypeText("text".into())).headed_requirement(),
crate::HeadedRequirement::FocusedWindow
);
}
#[test]
fn focus_fallback_policy_never_moves_cursor() {
let request = ActionRequest::focus_fallback(Action::Scroll(Direction::Down, 1));
assert!(request.policy.allow_focus_steal);
assert!(!request.policy.allow_cursor_move);
}
/// Regression coverage: `ActionRequest.timeout_ms` must stay
/// `#[serde(default)]` so a legacy payload recorded before `timeout_ms`
/// existed (or any FFI/batch caller that omits the key) still
/// deserializes instead of erroring out.
#[test]
fn action_request_json_without_timeout_ms_key_deserializes_to_none() {
let request: ActionRequest = serde_json::from_value(serde_json::json!({
"action": "Click",
"policy": {
"allow_focus_steal": false,
"allow_cursor_move": false,
},
}))
.unwrap();
assert_eq!(request.timeout_ms, None);
}
#[test]
fn positive_timeout_round_trips_through_the_wire_contract() {
let request = ActionRequest::headless(Action::Click).with_timeout_ms(Some(5_000));
let json = serde_json::to_value(&request).unwrap();
assert_eq!(json["timeout_ms"], 5_000);
let decoded: ActionRequest = serde_json::from_value(json).unwrap();
assert_eq!(decoded.timeout_ms, Some(5_000));
}
#[test]
fn execution_identity_is_runtime_only() {
let request = ActionRequest::headless(Action::Click)
.with_expected_process(crate::ProcessIdentity::new(42, "generation-1"));
assert_eq!(
request.expected_process(),
Some(&crate::ProcessIdentity::new(42, "generation-1"))
);
let json = serde_json::to_value(request).unwrap();
assert!(json.get("expected_process").is_none());
}
}

121
crates/core/src/action_result.rs
View file

@ -1,5 +1,8 @@
use crate::{action_step::ActionStep, element_state::ElementState};
use serde::{Deserialize, Serialize};
use crate::{
Action, AdapterError, DeliverySemantics, ErrorCode, action_step::ActionStep,
action_step_outcome::ActionStepOutcome, element_state::ElementState,
};
use serde::{Deserialize, Deserializer, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ActionResult {
@ -8,14 +11,56 @@ pub struct ActionResult {
pub post_state: Option<ElementState>,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub steps: Vec<ActionStep>,
#[serde(skip_serializing_if = "Option::is_none")]
pub details: Option<serde_json::Value>,
#[serde(
default = "default_action_disposition",
deserialize_with = "deserialize_action_disposition"
)]
disposition: DeliverySemantics,
}
impl ActionResult {
pub fn new(action: impl Into<String>) -> Self {
pub fn from_execution(
action: &Action,
steps: Vec<ActionStep>,
post_state: Option<ElementState>,
) -> Result<Self, AdapterError> {
let label = action.name();
let (delivered, verified) = delivery_summary(&steps);
if !delivered {
return Ok(Self::satisfied_without_delivery(label).with_steps(steps));
}
if let Some(state) = post_state.as_ref() {
verify_post_state(action, state)?;
}
let mut result = Self::delivered_unverified(label).with_steps(steps);
if verified {
result = result.with_verified_delivery();
}
if let Some(state) = post_state {
result = result.with_state(state);
}
Ok(result)
}
pub fn satisfied_without_delivery(action: impl Into<String>) -> Self {
Self {
action: action.into(),
post_state: None,
steps: Vec::new(),
details: None,
disposition: DeliverySemantics::not_delivered(),
}
}
pub fn delivered_unverified(action: impl Into<String>) -> Self {
Self {
action: action.into(),
post_state: None,
steps: Vec::new(),
details: None,
disposition: default_action_disposition(),
}
}
@ -28,4 +73,74 @@ impl ActionResult {
self.steps = steps;
self
}
pub fn with_details(mut self, details: serde_json::Value) -> Self {
self.details = Some(details);
self
}
pub fn with_verified_delivery(mut self) -> Self {
if self.disposition == DeliverySemantics::delivered_unverified() {
self.disposition = DeliverySemantics::delivered_verified();
}
self
}
pub const fn disposition(&self) -> DeliverySemantics {
self.disposition
}
}
fn delivery_summary(steps: &[ActionStep]) -> (bool, bool) {
let mut delivered = false;
let mut verification = None;
for step in steps {
if matches!(step.outcome, ActionStepOutcome::Succeeded) {
delivered = true;
if let Some(verified) = step.verified() {
verification = Some(verification.unwrap_or(true) && verified);
}
}
}
(delivered, verification.unwrap_or(false))
}
fn verify_post_state(action: &Action, state: &ElementState) -> Result<(), AdapterError> {
if matches!(action, Action::Clear)
&& state
.value
.as_deref()
.is_some_and(|value| !value.is_empty())
{
return Err(AdapterError::new(
ErrorCode::ActionFailed,
"Clear reported success but element value is still non-empty",
)
.with_suggestion("Retry 'clear', or use 'press cmd+a' then 'press delete'.")
.with_disposition(DeliverySemantics::delivered_unverified()));
}
Ok(())
}
const fn default_action_disposition() -> DeliverySemantics {
DeliverySemantics::delivered_unverified()
}
fn deserialize_action_disposition<'de, D>(deserializer: D) -> Result<DeliverySemantics, D::Error>
where
D: Deserializer<'de>,
{
let disposition = DeliverySemantics::deserialize(deserializer)?;
match disposition {
DeliverySemantics::NotDelivered
| DeliverySemantics::DeliveredUnverified
| DeliverySemantics::DeliveredVerified => Ok(disposition),
_ => Err(serde::de::Error::custom(
"successful action result must represent satisfied or delivered work",
)),
}
}
#[cfg(test)]
#[path = "action_result_tests.rs"]
mod tests;

190
crates/core/src/action_result_tests.rs Normal file
View file

@ -0,0 +1,190 @@
use super::*;
fn state(value: Option<&str>) -> ElementState {
ElementState {
role: "textfield".into(),
states: vec![],
value: value.map(str::to_string),
enabled: None,
hidden: None,
offscreen: None,
}
}
fn execution_result(verifications: &[Option<bool>]) -> ActionResult {
let steps = verifications
.iter()
.map(|verified| {
let step = ActionStep::succeeded("Step");
match verified {
Some(value) => step.with_verified(*value),
None => step,
}
})
.collect();
ActionResult::from_execution(&Action::Click, steps, None).expect("execution result")
}
#[test]
fn delivered_unverified_constructor_is_explicit() {
assert_eq!(
ActionResult::delivered_unverified("click").disposition(),
DeliverySemantics::delivered_unverified()
);
}
#[test]
fn satisfied_without_delivery_is_successful_and_safe_to_repeat() {
let result = ActionResult::satisfied_without_delivery("check").with_steps(vec![
ActionStep::skipped("AlreadyInState").with_verified(true),
]);
assert_eq!(result.disposition(), DeliverySemantics::not_delivered());
let json = serde_json::to_value(result).unwrap();
assert_eq!(json["disposition"]["delivery"], "not_delivered");
assert_eq!(json["disposition"]["retry"], "safe");
}
#[test]
fn verification_cannot_turn_a_no_op_into_claimed_delivery() {
let result = ActionResult::satisfied_without_delivery("check").with_verified_delivery();
assert_eq!(result.disposition(), DeliverySemantics::not_delivered());
}
#[test]
fn verified_action_uses_verified_delivery() {
assert_eq!(
ActionResult::delivered_unverified("click")
.with_verified_delivery()
.disposition(),
DeliverySemantics::delivered_verified()
);
}
#[test]
fn legacy_json_defaults_to_delivered_unverified() {
let result: ActionResult = serde_json::from_value(serde_json::json!({
"action": "click"
}))
.expect("legacy action result");
assert_eq!(
result.disposition(),
DeliverySemantics::delivered_unverified()
);
}
#[test]
fn successful_action_rejects_unknown_and_uncertain_dispositions() {
for delivery in ["unknown", "delivery_uncertain"] {
let retry = if delivery == "unknown" {
"unknown"
} else {
"unsafe"
};
let value = serde_json::json!({
"action": "click",
"disposition": {
"delivery": delivery,
"retry": retry
}
});
assert!(serde_json::from_value::<ActionResult>(value).is_err());
}
}
#[test]
fn successful_action_deserializes_satisfied_without_delivery() {
let value = serde_json::json!({
"action": "check",
"disposition": {
"delivery": "not_delivered",
"retry": "safe"
}
});
let result: ActionResult = serde_json::from_value(value).unwrap();
assert_eq!(result.disposition(), DeliverySemantics::not_delivered());
}
#[test]
fn execution_without_succeeded_steps_is_satisfied_without_delivery() {
let steps = vec![ActionStep::skipped("AlreadyInState").with_verified(true)];
let result = ActionResult::from_execution(&Action::Check, steps, Some(state(Some("1"))))
.expect("no-op result");
assert_eq!(result.disposition(), DeliverySemantics::not_delivered());
assert_eq!(result.steps.len(), 1);
assert!(result.post_state.is_none());
}
#[test]
fn execution_derives_disposition_from_succeeded_step_evidence() {
for (verifications, expected) in [
(vec![Some(true)], DeliverySemantics::delivered_verified()),
(vec![Some(false)], DeliverySemantics::delivered_unverified()),
(
vec![Some(true), None],
DeliverySemantics::delivered_verified(),
),
(vec![None], DeliverySemantics::delivered_unverified()),
(
vec![Some(true), Some(false)],
DeliverySemantics::delivered_unverified(),
),
] {
assert_eq!(execution_result(&verifications).disposition(), expected);
}
}
#[test]
fn execution_attaches_post_state_without_changing_serialization() {
let steps = vec![ActionStep::succeeded("AXValue").with_verified(false)];
let post_state = state(Some("done"));
let actual = ActionResult::from_execution(
&Action::SetValue("done".into()),
steps.clone(),
Some(post_state.clone()),
)
.expect("result");
let expected = ActionResult::delivered_unverified("set-value")
.with_steps(steps)
.with_state(post_state);
assert_eq!(
serde_json::to_value(actual).unwrap(),
serde_json::to_value(expected).unwrap()
);
}
#[test]
fn clear_postcondition_preserves_exact_error_contract() {
for post_state in [state(Some("")), state(None)] {
ActionResult::from_execution(
&Action::Clear,
vec![ActionStep::succeeded("AXValue").with_verified(true)],
Some(post_state),
)
.expect("clear result");
}
let error = ActionResult::from_execution(
&Action::Clear,
vec![ActionStep::succeeded("AXValue").with_verified(true)],
Some(state(Some("still here"))),
)
.expect_err("non-empty clear must fail");
assert_eq!(error.code, ErrorCode::ActionFailed);
assert_eq!(
error.message,
"Clear reported success but element value is still non-empty"
);
assert_eq!(
error.suggestion.as_deref(),
Some("Retry 'clear', or use 'press cmd+a' then 'press delete'.")
);
assert_eq!(error.disposition, DeliverySemantics::delivered_unverified());
}

33
crates/core/src/action_step.rs
View file

@ -1,10 +1,15 @@
use crate::action_step_outcome::ActionStepOutcome;
use crate::step_mechanism::StepMechanism;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ActionStep {
label: String,
pub outcome: ActionStepOutcome,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub mechanism: Option<StepMechanism>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub verified: Option<bool>,
}
impl ActionStep {
@ -12,6 +17,8 @@ impl ActionStep {
Self {
label: label.to_string(),
outcome: ActionStepOutcome::Attempted,
mechanism: None,
verified: None,
}
}
@ -19,6 +26,8 @@ impl ActionStep {
Self {
label: label.to_string(),
outcome: ActionStepOutcome::Skipped,
mechanism: None,
verified: None,
}
}
@ -26,10 +35,34 @@ impl ActionStep {
Self {
label: label.to_string(),
outcome: ActionStepOutcome::Succeeded,
mechanism: None,
verified: None,
}
}
pub fn label(&self) -> &str {
&self.label
}
pub fn mechanism(&self) -> Option<StepMechanism> {
self.mechanism
}
pub fn verified(&self) -> Option<bool> {
self.verified
}
pub fn with_mechanism(mut self, mechanism: StepMechanism) -> Self {
self.mechanism = Some(mechanism);
self
}
pub fn with_verified(mut self, verified: bool) -> Self {
self.verified = Some(verified);
self
}
}
#[cfg(test)]
#[path = "action_step_tests.rs"]
mod tests;

48
crates/core/src/action_step_tests.rs Normal file
View file

@ -0,0 +1,48 @@
use super::ActionStep;
use crate::step_mechanism::StepMechanism;
use crate::trace_sanitize::sanitize_trace_value;
use serde_json::json;
#[test]
fn legacy_action_step_json_round_trips_without_new_fields() {
let legacy = r#"{"label":"AXPress","outcome":"succeeded"}"#;
let step: ActionStep = serde_json::from_str(legacy).unwrap();
assert_eq!(step.label(), "AXPress");
assert!(step.mechanism().is_none());
assert!(step.verified().is_none());
let round_trip = serde_json::to_value(&step).unwrap();
let legacy_value: serde_json::Value = serde_json::from_str(legacy).unwrap();
assert_eq!(round_trip, legacy_value);
}
#[test]
fn action_step_serializes_mechanism_and_verified() {
let step = ActionStep::succeeded("verified_press")
.with_mechanism(StepMechanism::SemanticApi)
.with_verified(true);
let value = serde_json::to_value(&step).unwrap();
assert_eq!(value["mechanism"], "semantic_api");
assert_eq!(value["verified"], true);
}
#[test]
fn action_step_omits_absent_mechanism_and_verified() {
let step = ActionStep::skipped("AXConfirm");
let value = serde_json::to_value(&step).unwrap();
assert!(value.get("mechanism").is_none());
assert!(value.get("verified").is_none());
}
#[test]
fn trace_preserves_mechanism_and_verified() {
let value = sanitize_trace_value(json!({
"steps": [{
"label": "AXPress",
"outcome": "succeeded",
"mechanism": "semantic_api",
"verified": true
}]
}));
assert_eq!(value["steps"][0]["mechanism"], "semantic_api");
assert_eq!(value["steps"][0]["verified"], true);
}

216
crates/core/src/action_tests.rs
View file

@ -1,5 +1,8 @@
use super::*;
use crate::interaction_policy::InteractionPolicy;
use super::Action;
use crate::{
Direction, DragParams, KeyCombo, Modifier, MouseButton, MouseEvent, MouseEventKind, Point,
interaction_policy::InteractionPolicy,
};
fn dummy_key() -> KeyCombo {
KeyCombo {
@ -26,7 +29,7 @@ fn action_names_do_not_include_payloads() {
(
Action::PressKey(KeyCombo {
key: "A".into(),
modifiers: vec![Modifier::Cmd],
modifiers: vec![Modifier::Meta],
}),
"press",
),
@ -37,6 +40,47 @@ fn action_names_do_not_include_payloads() {
}
}
#[test]
fn headed_requirements_distinguish_window_and_pointer_preconditions() {
for action in [
Action::Click,
Action::DoubleClick,
Action::RightClick,
Action::TripleClick,
Action::Scroll(Direction::Down, 1),
Action::Hover,
Action::Drag(dummy_drag()),
] {
assert_eq!(
action.headed_requirement(),
crate::HeadedRequirement::FocusedWindowAndCursor,
"{} must require focus plus cursor delivery",
action.name()
);
}
for action in [
Action::SetValue("v".into()),
Action::SetFocus,
Action::Expand,
Action::Collapse,
Action::Select("v".into()),
Action::Toggle,
Action::Check,
Action::Uncheck,
Action::ScrollTo,
Action::PressKey(dummy_key()),
Action::TypeText("v".into()),
Action::Clear,
] {
assert_eq!(
action.headed_requirement(),
crate::HeadedRequirement::FocusedWindow,
"{} must require focus without pointer movement",
action.name()
);
}
}
#[test]
fn pure_ax_actions_base_policy_is_headless() {
let headless = InteractionPolicy::headless();
@ -68,21 +112,21 @@ fn pure_ax_actions_base_policy_is_headless() {
}
#[test]
fn press_key_and_type_text_base_policy_is_focus_fallback() {
let focus = InteractionPolicy::focus_fallback();
fn type_text_is_headless_but_explicit_press_allows_focus() {
let headless = InteractionPolicy::headless();
assert_eq!(
Action::PressKey(KeyCombo {
key: "a".into(),
modifiers: vec![Modifier::Cmd],
modifiers: vec![Modifier::Meta],
})
.base_interaction_policy(),
focus,
"PressKey must request focus_fallback to land in the right field"
InteractionPolicy::focus_fallback(),
"PressKey is an explicit physical-input command"
);
assert_eq!(
Action::TypeText("hello".into()).base_interaction_policy(),
focus,
"TypeText must request focus_fallback"
headless,
"TypeText must not gain implicit focus or keyboard permission"
);
}
@ -123,3 +167,155 @@ fn hover_and_drag_base_policy_is_headless_independent_of_cursor_requirement() {
"Drag.requires_cursor_policy() must still be true"
);
}
#[test]
fn requires_hit_test_covers_ref_targeted_pointer_actions() {
let hit_tested: &[Action] = &[
Action::Click,
Action::DoubleClick,
Action::RightClick,
Action::TripleClick,
Action::Hover,
Action::Drag(dummy_drag()),
];
for action in hit_tested {
assert!(
action.requires_hit_test(),
"{} must require a hit test before dispatch",
action.name()
);
}
let not_hit_tested: &[Action] = &[
Action::SetValue("v".into()),
Action::SetFocus,
Action::Expand,
Action::Collapse,
Action::Select("s".into()),
Action::Toggle,
Action::Check,
Action::Uncheck,
Action::Scroll(Direction::Down, 1),
Action::ScrollTo,
Action::PressKey(dummy_key()),
Action::KeyDown(dummy_key()),
Action::KeyUp(dummy_key()),
Action::TypeText("t".into()),
Action::Clear,
];
for action in not_hit_tested {
assert!(
!action.requires_hit_test(),
"{} must not require a hit test",
action.name()
);
}
}
#[test]
fn requires_scroll_into_view_covers_all_actions() {
let scroll_into_view: &[Action] = &[
Action::Click,
Action::DoubleClick,
Action::RightClick,
Action::TripleClick,
Action::SetValue("v".into()),
Action::Expand,
Action::Collapse,
Action::Select("s".into()),
Action::Toggle,
Action::Check,
Action::Uncheck,
Action::TypeText("t".into()),
Action::Clear,
Action::Hover,
Action::Drag(dummy_drag()),
];
for action in scroll_into_view {
assert!(
action.requires_scroll_into_view(),
"{} must require scroll-into-view before dispatch",
action.name()
);
}
let not_scroll_into_view: &[Action] = &[
Action::Scroll(Direction::Down, 1),
Action::ScrollTo,
Action::SetFocus,
Action::PressKey(dummy_key()),
Action::KeyDown(dummy_key()),
Action::KeyUp(dummy_key()),
];
for action in not_scroll_into_view {
assert!(
!action.requires_scroll_into_view(),
"{} must not require scroll-into-view",
action.name()
);
}
}
/// F10 regression coverage: `MouseEvent.modifiers` must stay `#[serde(default)]`
/// so a legacy payload recorded before modifiers existed (or any FFI/batch
/// caller that omits the key) still deserializes instead of erroring out.
#[test]
fn mouse_event_json_without_modifiers_key_deserializes_to_empty() {
let event: MouseEvent = serde_json::from_value(serde_json::json!({
"kind": { "Click": { "count": 1 } },
"point": { "x": 1.0, "y": 2.0 },
"button": "Left",
}))
.unwrap();
assert!(event.modifiers.is_empty());
}
#[test]
fn mouse_event_json_with_modifiers_round_trips() {
let event = MouseEvent {
kind: MouseEventKind::Click { count: 2 },
point: Point { x: 1.0, y: 2.0 },
button: MouseButton::Left,
modifiers: vec![Modifier::Meta, Modifier::Shift],
};
let json = serde_json::to_value(&event).unwrap();
let round_tripped: MouseEvent = serde_json::from_value(json).unwrap();
assert_eq!(
round_tripped.modifiers,
vec![Modifier::Meta, Modifier::Shift]
);
}
#[test]
fn modifier_meta_serializes_semantically_and_accepts_legacy_cmd() {
assert_eq!(serde_json::to_string(&Modifier::Meta).unwrap(), "\"Meta\"");
assert_eq!(
serde_json::from_str::<Modifier>("\"Cmd\"").unwrap(),
Modifier::Meta
);
}
#[test]
fn wheel_event_round_trips_platform_neutral_line_deltas() {
let event = MouseEvent {
kind: MouseEventKind::Wheel {
delta_x: -2.0,
delta_y: 3.0,
},
point: Point { x: 50.0, y: 60.0 },
button: MouseButton::Left,
modifiers: vec![Modifier::Shift],
};
let json = serde_json::to_value(&event).unwrap();
let decoded: MouseEvent = serde_json::from_value(json).unwrap();
assert!(matches!(
decoded.kind,
MouseEventKind::Wheel {
delta_x: -2.0,
delta_y: 3.0
}
));
}

21
crates/core/src/actionability/check.rs
View file

@ -1,10 +1,19 @@
use super::ActionabilityStatus;
use super::{occluder::Occluder, status::ActionabilityStatus};
use crate::ErrorCode;
use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct ActionabilityCheck {
pub name: &'static str,
pub status: ActionabilityStatus,
#[derive(Debug, Clone, Serialize, PartialEq)]
pub(crate) struct ActionabilityCheck {
pub(crate) check: &'static str,
pub(crate) status: ActionabilityStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
pub(crate) reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) occluder: Option<Occluder>,
#[serde(skip)]
pub(crate) terminal_code: Option<ErrorCode>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) hit_test: Option<super::hit_test_evidence::HitTestEvidence>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) stability: Option<super::stability_evidence::StabilityEvidence>,
}

74
crates/core/src/actionability/check_result.rs Normal file
View file

@ -0,0 +1,74 @@
use super::{check::ActionabilityCheck, occluder::Occluder, status::ActionabilityStatus};
use crate::{ErrorCode, Rect};
pub(super) fn pass(check: &'static str) -> ActionabilityCheck {
ActionabilityCheck {
check,
status: ActionabilityStatus::Pass,
reason: None,
occluder: None,
terminal_code: None,
hit_test: None,
stability: None,
}
}
pub(super) fn fail(check: &'static str, reason: impl Into<String>) -> ActionabilityCheck {
ActionabilityCheck {
check,
status: ActionabilityStatus::Fail,
reason: Some(reason.into()),
occluder: None,
terminal_code: None,
hit_test: None,
stability: None,
}
}
pub(super) fn fail_terminal(
check: &'static str,
reason: impl Into<String>,
code: ErrorCode,
) -> ActionabilityCheck {
ActionabilityCheck {
check,
status: ActionabilityStatus::Fail,
reason: Some(reason.into()),
occluder: None,
terminal_code: Some(code),
hit_test: None,
stability: None,
}
}
pub(super) fn unknown(check: &'static str, reason: impl Into<String>) -> ActionabilityCheck {
ActionabilityCheck {
check,
status: ActionabilityStatus::Unknown,
reason: Some(reason.into()),
occluder: None,
terminal_code: None,
hit_test: None,
stability: None,
}
}
pub(super) fn occluded(
role: Option<String>,
name: Option<String>,
bounds: Option<Rect>,
) -> ActionabilityCheck {
let reason = role.as_deref().map_or_else(
|| "occluded by another element".to_string(),
|role| format!("occluded by {role}"),
);
ActionabilityCheck {
check: "receives_events",
status: ActionabilityStatus::Fail,
reason: Some(reason),
occluder: Some(Occluder { role, name, bounds }),
terminal_code: None,
hit_test: None,
stability: None,
}
}

159
crates/core/src/actionability/evaluate.rs Normal file
View file

@ -0,0 +1,159 @@
use super::{
evidence::ActionabilityEvidence, gates, receives_events::receives_events_check,
report::ActionabilityReport, requirements::ActionabilityRequirements,
stability::StabilityExpectation,
};
use crate::{
AdapterError, ErrorCode,
action_request::ActionRequest,
adapter::{NativeHandle, PlatformAdapter},
};
use serde_json::json;
#[cfg(test)]
use crate::refs::RefEntry;
#[cfg(test)]
pub(crate) fn check(
entry: &RefEntry,
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
use crate::{
adapter::{ActionOps, InputOps, ObservationOps, SystemOps},
hit_test::HitTestResult,
};
struct TestHitAdapter;
impl ObservationOps for TestHitAdapter {
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
Ok(HitTestResult::ReachesTarget)
}
}
impl ActionOps for TestHitAdapter {}
impl InputOps for TestHitAdapter {}
impl SystemOps for TestHitAdapter {}
let evidence = ActionabilityEvidence {
state: crate::ElementState {
role: entry.identity.role.clone(),
states: entry.capabilities.states.clone(),
value: entry.identity.value.clone(),
enabled: Some(!crate::state::has_state(
&entry.capabilities.states,
crate::state::DISABLED,
)),
hidden: Some(crate::state::has_state(
&entry.capabilities.states,
crate::state::HIDDEN,
)),
offscreen: Some(crate::state::has_state(
&entry.capabilities.states,
crate::state::OFFSCREEN,
)),
},
states_complete: true,
bounds: entry.geometry.bounds,
available_actions: entry.capabilities.available_actions.clone(),
};
let handle = NativeHandle::null();
let adapter = TestHitAdapter;
check_with_stability(
StabilityExpectation::permissive(entry.geometry.bounds_hash),
&evidence,
request,
Some((&handle, &adapter)),
crate::Deadline::standard()?,
)
}
pub(super) fn check_with_stability(
stability: StabilityExpectation,
evidence: &ActionabilityEvidence,
request: &ActionRequest,
hit_test: Option<(&NativeHandle, &dyn PlatformAdapter)>,
deadline: crate::Deadline,
) -> Result<ActionabilityReport, AdapterError> {
let requirements = ActionabilityRequirements::for_action(&request.action);
let pointer_delivery =
requirements.pointer_delivery(&request.action, &evidence.available_actions, request.policy);
let mut checks = Vec::new();
let mut verified_point = None;
if requirements.visible {
checks.push(gates::visibility(evidence));
}
if requirements.requires_stability(pointer_delivery) {
checks.push(gates::stability(stability, evidence.bounds));
}
if requirements.enabled {
checks.push(gates::enabled(evidence));
}
if requirements.editable {
checks.push(gates::editable(evidence, &request.action));
}
checks.push(gates::action_supported(evidence, request));
checks.push(gates::policy(request));
if checks
.iter()
.any(|check| !matches!(check.status, super::status::ActionabilityStatus::Pass))
{
return finish(
evidence,
ActionabilityReport::from_checks(checks, verified_point, pointer_delivery),
);
}
if matches!(pointer_delivery, super::PointerDelivery::Physical) {
match hit_test {
Some((handle, adapter)) => {
let (check, point) =
receives_events_check(evidence.bounds, handle, adapter, request, deadline)?;
checks.push(check);
verified_point = point;
}
None => checks.push(super::check_result::unknown(
"receives_events",
"live hit-test context unavailable",
)),
}
}
finish(
evidence,
ActionabilityReport::from_checks(checks, verified_point, pointer_delivery),
)
}
fn finish(
evidence: &ActionabilityEvidence,
report: ActionabilityReport,
) -> Result<ActionabilityReport, AdapterError> {
if report.actionable {
return Ok(report);
}
let code = report.terminal_code().unwrap_or(ErrorCode::ActionFailed);
let suggestion = if code == ErrorCode::ActionFailed {
"Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended."
} else {
"Waiting will not help: this element cannot satisfy the action as targeted. Target an element that advertises the action (check available_actions in a fresh snapshot) or adjust the interaction policy (e.g. pass --headed)."
};
let mut details = json!(report);
if let Some(object) = details.as_object_mut()
&& let Some(bounds) = evidence.bounds
{
if let Some(bounds_hash) = bounds.bounds_hash() {
object.insert("observed_bounds_hash".into(), json!(bounds_hash));
}
}
Err(AdapterError::new(
code,
format!("Target is not actionable: {}", report.failure_reasons()),
)
.with_details(details)
.with_suggestion(suggestion)
.with_disposition(crate::DeliverySemantics::not_delivered()))
}

8
crates/core/src/actionability/evidence.rs Normal file
View file

@ -0,0 +1,8 @@
use crate::{Rect, element_state::ElementState};
pub(super) struct ActionabilityEvidence {
pub(super) state: ElementState,
pub(super) states_complete: bool,
pub(super) bounds: Option<Rect>,
pub(super) available_actions: Vec<String>,
}

193
crates/core/src/actionability/gates.rs Normal file
View file

@ -0,0 +1,193 @@
use super::{
check::ActionabilityCheck,
check_result::{fail, fail_terminal, pass, unknown},
evidence::ActionabilityEvidence,
};
use crate::{ErrorCode, Rect, action::Action, action_request::ActionRequest, capability};
pub(super) fn visibility(evidence: &ActionabilityEvidence) -> ActionabilityCheck {
match evidence.state.hidden {
Some(true) => return fail("visible", "live hidden state is true"),
None if !evidence.states_complete => {
return unknown("visible", "live hidden state unavailable");
}
None if crate::state::has_state(&evidence.state.states, crate::state::HIDDEN) => {
return fail("visible", "canonical hidden state is present");
}
None => {}
Some(false) => {}
}
match evidence.state.offscreen {
Some(true) => return fail("visible", "live offscreen state is true"),
None => return unknown("visible", "live offscreen state unavailable"),
Some(false) => {}
}
let Some(bounds) = evidence.bounds else {
return unknown("visible", "bounds unavailable");
};
if !bounds_are_visible(Some(bounds)) {
return fail("visible", "bounds are zero-sized");
}
pass("visible")
}
pub(super) fn stability(
expectation: super::stability::StabilityExpectation,
bounds: Option<Rect>,
) -> ActionabilityCheck {
let evidence = super::stability_evidence::StabilityEvidence {
samples: expectation.samples,
span_ms: expectation.span_ms,
};
let Some(bounds) = bounds else {
let mut check = unknown("stable", "live bounds unavailable");
check.stability = Some(evidence);
return check;
};
let check = if expectation.strict {
if let Some(expected) = expectation.bounds {
if super::stability_sampler::geometry_matches(expected, bounds) {
pass("stable")
} else {
fail("stable", "bounds changed after the stability window")
}
} else if let Some(expected_hash) = expectation.bounds_hash {
if bounds.bounds_hash() == Some(expected_hash) {
pass("stable")
} else {
fail("stable", "bounds changed since the previous observation")
}
} else {
fail("stable", "waiting for stable live bounds observations")
}
} else {
let Some(expected) = expectation.bounds_hash else {
let mut check = unknown("stable", "snapshot bounds hash unavailable");
check.stability = Some(evidence);
return check;
};
let Some(observed_hash) = bounds.bounds_hash() else {
let mut check = unknown("stable", "live bounds are invalid");
check.stability = Some(evidence);
return check;
};
if observed_hash == expected {
pass("stable")
} else {
unknown("stable", "bounds changed since snapshot")
}
};
let mut check = check;
check.stability = Some(evidence);
check
}
pub(super) fn enabled(evidence: &ActionabilityEvidence) -> ActionabilityCheck {
match evidence.state.enabled {
Some(true) => pass("enabled"),
Some(false) => fail("enabled", "live enabled state is false"),
None => unknown("enabled", "live enabled state unavailable"),
}
}
pub(super) fn action_supported(
evidence: &ActionabilityEvidence,
request: &ActionRequest,
) -> ActionabilityCheck {
if request.action.requires_cursor_policy() {
return pass("supported_action");
}
if matches!(request.action, Action::ScrollTo) {
return pass("supported_action");
}
if matches!(request.action, Action::Click | Action::RightClick) {
if capability::supports_direct_semantic_pointer_delivery(
&request.action,
&evidence.available_actions,
) {
return pass("supported_action");
}
if request.policy.allow_cursor_move && request.policy.allow_focus_steal {
return pass("supported_action");
}
return fail_terminal(
"supported_action",
"direct semantic delivery is unavailable and physical fallback is denied",
ErrorCode::PolicyDenied,
);
}
if matches!(request.action, Action::DoubleClick | Action::TripleClick) {
if request.policy.allow_cursor_move && request.policy.allow_focus_steal {
return pass("supported_action");
}
return fail_terminal(
"supported_action",
"semantic gesture is unavailable and physical fallback is denied",
ErrorCode::PolicyDenied,
);
}
if capability::contains_any(
&evidence.available_actions,
capability::for_action(&request.action),
) {
return pass("supported_action");
}
if request.action.may_use_focus_fallback() && request.policy.allow_focus_steal {
return pass("supported_action");
}
if request.action.may_use_focus_fallback() {
return fail_terminal(
"supported_action",
"semantic action is unavailable and focus fallback is denied",
ErrorCode::PolicyDenied,
);
}
let expected = capability::for_action(&request.action).join(" or ");
fail_terminal(
"supported_action",
format!("{expected} is not available"),
ErrorCode::ActionNotSupported,
)
}
pub(super) fn policy(request: &ActionRequest) -> ActionabilityCheck {
if request.action.requires_cursor_policy() && !request.policy.allow_cursor_move {
return fail_terminal(
"policy",
"action requires cursor movement but policy denies it",
ErrorCode::PolicyDenied,
);
}
pass("policy")
}
pub(super) fn editable(evidence: &ActionabilityEvidence, action: &Action) -> ActionabilityCheck {
if !matches!(
action,
Action::SetValue(_) | Action::TypeText(_) | Action::Clear
) {
return pass("editable");
}
if evidence.state.role == "textfield" || evidence.state.role == "combobox" {
return pass("editable");
}
if capability::contains(&evidence.available_actions, capability::SET_VALUE) {
return pass("editable");
}
fail_terminal(
"editable",
format!("role {} is not editable", evidence.state.role),
ErrorCode::ActionNotSupported,
)
}
#[cfg(test)]
pub(crate) fn states_are_enabled(states: &[String]) -> bool {
!crate::state::has_state(states, crate::state::DISABLED)
}
pub(crate) fn bounds_are_visible(bounds: Option<Rect>) -> bool {
bounds.is_some_and(|bounds| {
bounds.validate().is_ok() && bounds.width > 0.0 && bounds.height > 0.0
})
}

8
crates/core/src/actionability/hit_test_evidence.rs Normal file
View file

@ -0,0 +1,8 @@
use serde::Serialize;
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
pub(crate) struct HitTestEvidence {
pub(crate) attempted: u8,
pub(crate) unknown: u8,
pub(crate) occluded: u8,
}

143
crates/core/src/actionability/live.rs Normal file
View file

@ -0,0 +1,143 @@
use super::{
evaluate::check_with_stability, evidence::ActionabilityEvidence, report::ActionabilityReport,
stability::StabilityExpectation,
};
use crate::{
AdapterError, DeliverySemantics, ErrorCode, IdentityMatch, LocatorField,
action_request::ActionRequest,
adapter::{LiveElement, NativeHandle, PlatformAdapter},
ref_identity::{has_meaningful_identity, identity_match},
refs::RefEntry,
};
/// Groups the read-only coordinates of a live element check: which element,
/// which handle to read it through, which adapter to read with, and the
/// deadline that bounds the read. Kept separate from the `request` (what
/// action to check against) and `stability` (how settled the element must
/// be) parameters, which vary independently per call.
pub(crate) struct LiveCheckTarget<'a> {
pub(crate) entry: &'a RefEntry,
pub(crate) handle: &'a NativeHandle,
pub(crate) adapter: &'a dyn PlatformAdapter,
pub(crate) deadline: crate::Deadline,
}
#[cfg(test)]
pub(crate) fn check_live(
entry: &RefEntry,
handle: &NativeHandle,
adapter: &dyn PlatformAdapter,
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
let deadline = crate::Deadline::standard()?;
check_live_with_stability(
&LiveCheckTarget {
entry,
handle,
adapter,
deadline,
},
request,
StabilityExpectation::permissive(entry.geometry.bounds_hash),
)
}
pub(crate) fn check_live_with_stability(
target: &LiveCheckTarget<'_>,
request: &ActionRequest,
stability: StabilityExpectation,
) -> Result<ActionabilityReport, AdapterError> {
let evidence = observe(
target.entry,
target
.adapter
.get_live_element(target.handle, target.deadline),
)?;
check_with_stability(
stability,
&evidence,
request,
Some((target.handle, target.adapter)),
target.deadline,
)
}
fn observe(
entry: &RefEntry,
live: Result<LiveElement, AdapterError>,
) -> Result<ActionabilityEvidence, AdapterError> {
let live = match live {
Ok(live) => live,
Err(err)
if matches!(
err.code,
ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported
) =>
{
return Err(AdapterError::new(
ErrorCode::ActionNotSupported,
"Live element evidence is required for actionability checks",
)
.with_details(serde_json::json!({ "source_code": err.code.as_str() }))
.with_disposition(DeliverySemantics::not_delivered()));
}
Err(err) => return Err(err),
};
validate_live_identity(entry, &live)?;
Ok(ActionabilityEvidence {
state: live.state,
states_complete: live.states_complete,
bounds: live.bounds,
available_actions: live.available_actions,
})
}
fn validate_live_identity(entry: &RefEntry, live: &LiveElement) -> Result<(), AdapterError> {
if live.state.role == "unknown" || live.state.role != entry.identity.role {
return Err(stale_live_element(
"Resolved element changed role before dispatch",
));
}
if !has_meaningful_identity(entry) {
return Ok(());
}
let value = live
.state
.value
.clone()
.map(LocatorField::Known)
.unwrap_or(LocatorField::Absent);
match identity_match(
entry,
&live.identity.name,
&value,
&live.identity.description,
&live.identity.identifiers,
) {
IdentityMatch::Match => Ok(()),
IdentityMatch::NoMatch => Err(stale_live_element(
"Resolved element changed identity before dispatch",
)),
IdentityMatch::Unknown => Err(AdapterError::app_unresponsive(
entry
.source
.source_app
.as_deref()
.unwrap_or("target application"),
)
.with_details(serde_json::json!({
"kind": "live_identity_evidence",
"complete": false,
"retryable": true,
}))),
}
}
fn stale_live_element(message: &'static str) -> AdapterError {
AdapterError::new(ErrorCode::StaleRef, message)
.with_suggestion(
"Re-run a snapshot to obtain fresh refs, then retry with the new ref (CLI: snapshot; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)",
)
.with_details(serde_json::json!({ "retryable": true }))
.with_disposition(DeliverySemantics::not_delivered())
}

259
crates/core/src/actionability/mod.rs
View file

@ -1,239 +1,36 @@
use crate::capability;
use crate::{
action::Action,
action_request::ActionRequest,
adapter::{NativeHandle, PlatformAdapter},
error::{AdapterError, ErrorCode},
node::Rect,
refs::RefEntry,
};
use serde_json::json;
mod check;
mod check_result;
mod evaluate;
mod evidence;
mod gates;
mod hit_test_evidence;
mod live;
mod occluder;
mod pointer_delivery;
mod receives_events;
mod report;
mod requirements;
mod stability;
mod stability_evidence;
pub(crate) mod stability_sampler;
mod status;
pub use check::ActionabilityCheck;
pub use report::ActionabilityReport;
pub use status::ActionabilityStatus;
#[cfg(test)]
pub fn check(
entry: &RefEntry,
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
check_with_stability(entry.bounds_hash, entry, request)
}
pub fn check_live(
entry: &RefEntry,
handle: &NativeHandle,
adapter: &dyn PlatformAdapter,
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
let mut observed = entry.clone();
let live = match adapter.get_live_element(handle) {
Ok(live) => live,
Err(err)
if matches!(
err.code,
ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported
) =>
{
return check_with_stability(entry.bounds_hash, &observed, request);
}
Err(err) => return Err(err),
};
if live_element_is_stale(&live) {
return Err(AdapterError::new(
ErrorCode::StaleRef,
"Resolved element no longer exposes live accessibility state",
)
.with_suggestion(
"Re-run a snapshot to obtain fresh refs, then retry with the new ref (CLI: snapshot; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)",
));
}
if let Some(state) = live.state {
observed.role = state.role;
observed.states = state.states;
observed.value = state.value.or(observed.value);
}
observed.bounds = live.bounds;
if let Some(actions) = live.available_actions
&& !actions.is_empty()
{
observed.available_actions = actions;
}
check_with_stability(entry.bounds_hash, &observed, request)
}
fn live_element_is_stale(live: &crate::adapter::LiveElement) -> bool {
let role_unknown = live
.state
.as_ref()
.is_none_or(|state| state.role == "unknown");
let actions_empty = live.available_actions.as_ref().is_none_or(Vec::is_empty);
role_unknown && live.bounds.is_none() && actions_empty
}
fn check_with_stability(
expected_bounds_hash: Option<u64>,
entry: &RefEntry,
request: &ActionRequest,
) -> Result<ActionabilityReport, AdapterError> {
let checks = vec![
visibility_check(entry),
stability_check(expected_bounds_hash, entry.bounds),
enabled_check(entry),
action_supported_check(entry, request),
policy_check(request),
editable_check(entry, &request.action),
];
let actionable = checks
.iter()
.all(|check| !matches!(check.status, ActionabilityStatus::Fail));
let report = ActionabilityReport { actionable, checks };
if report.actionable {
return Ok(report);
}
Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Target is not actionable: {}", failure_reasons(&report)),
)
.with_details(json!(report))
.with_suggestion(
"Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended.",
))
}
fn visibility_check(entry: &RefEntry) -> ActionabilityCheck {
let Some(bounds) = entry.bounds else {
return unknown("visible", "bounds unavailable");
};
if !bounds_are_visible(Some(bounds)) {
return fail("visible", "bounds are zero-sized");
}
pass("visible")
}
fn stability_check(expected_bounds_hash: Option<u64>, bounds: Option<Rect>) -> ActionabilityCheck {
let Some(expected) = expected_bounds_hash else {
return unknown("stable", "snapshot bounds hash unavailable");
};
let Some(bounds) = bounds else {
return unknown("stable", "live bounds unavailable");
};
if bounds.bounds_hash() != expected {
return unknown("stable", "bounds changed since snapshot");
}
pass("stable")
}
fn enabled_check(entry: &RefEntry) -> ActionabilityCheck {
if !states_are_enabled(&entry.states) {
return fail("enabled", "entry state contains disabled");
}
pass("enabled")
}
pub fn states_are_enabled(states: &[String]) -> bool {
!states.iter().any(|state| state == "disabled")
}
pub fn bounds_are_visible(bounds: Option<Rect>) -> bool {
bounds.is_some_and(|bounds| bounds.width > 0.0 && bounds.height > 0.0)
}
fn action_supported_check(entry: &RefEntry, request: &ActionRequest) -> ActionabilityCheck {
if request.action.requires_cursor_policy() {
return pass("supported_action");
}
if capability::contains_any(
&entry.available_actions,
capability::for_action(&request.action),
) {
return pass("supported_action");
}
if may_use_fallback(&request.action, request) {
return unknown(
"supported_action",
"semantic action unavailable but fallback policy allows attempt",
);
}
let expected = capability::for_action(&request.action).join(" or ");
fail("supported_action", format!("{expected} is not available"))
}
fn policy_check(request: &ActionRequest) -> ActionabilityCheck {
if request.action.requires_cursor_policy() && !request.policy.allow_cursor_move {
return fail(
"policy",
"action requires cursor movement but policy denies it",
);
}
if request.action.may_use_focus_fallback() && !request.policy.allow_focus_steal {
return fail("policy", "action requires focus but policy denies it");
}
pass("policy")
}
fn editable_check(entry: &RefEntry, action: &Action) -> ActionabilityCheck {
if !matches!(
action,
Action::SetValue(_) | Action::TypeText(_) | Action::Clear
) {
return pass("editable");
}
if entry.role == "textfield" || entry.role == "combobox" {
return pass("editable");
}
if capability::contains(&entry.available_actions, capability::SET_VALUE) {
return pass("editable");
}
fail("editable", format!("role {} is not editable", entry.role))
}
fn failure_reasons(report: &ActionabilityReport) -> String {
report
.checks
.iter()
.filter(|check| matches!(check.status, ActionabilityStatus::Fail))
.map(|check| {
let reason = check.reason.as_deref().unwrap_or("failed");
format!("{} ({reason})", check.name)
})
.collect::<Vec<_>>()
.join(", ")
}
fn may_use_fallback(action: &Action, request: &ActionRequest) -> bool {
action.may_use_focus_fallback() && request.policy.allow_focus_steal
}
fn pass(name: &'static str) -> ActionabilityCheck {
ActionabilityCheck {
name,
status: ActionabilityStatus::Pass,
reason: None,
}
}
fn fail(name: &'static str, reason: impl Into<String>) -> ActionabilityCheck {
ActionabilityCheck {
name,
status: ActionabilityStatus::Fail,
reason: Some(reason.into()),
}
}
fn unknown(name: &'static str, reason: impl Into<String>) -> ActionabilityCheck {
ActionabilityCheck {
name,
status: ActionabilityStatus::Unknown,
reason: Some(reason.into()),
}
}
pub(crate) use check::ActionabilityCheck;
#[cfg(test)]
pub(crate) use evaluate::check;
pub(crate) use gates::bounds_are_visible;
#[cfg(test)]
pub(crate) use gates::states_are_enabled;
#[cfg(test)]
pub(crate) use live::check_live;
pub(crate) use live::{LiveCheckTarget, check_live_with_stability};
pub(crate) use pointer_delivery::PointerDelivery;
pub(crate) use receives_events::require_receives_events;
pub(crate) use requirements::requires_stability;
pub(crate) use stability::StabilityExpectation;
#[cfg(test)]
pub(crate) use status::ActionabilityStatus;
#[cfg(test)]
#[path = "../actionability_tests.rs"]

12
crates/core/src/actionability/occluder.rs Normal file
View file

@ -0,0 +1,12 @@
use crate::Rect;
use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq)]
pub(crate) struct Occluder {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) role: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) bounds: Option<Rect>,
}

6
crates/core/src/actionability/pointer_delivery.rs Normal file
View file

@ -0,0 +1,6 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum PointerDelivery {
NotApplicable,
Semantic,
Physical,
}

114
crates/core/src/actionability/receives_events.rs Normal file
View file

@ -0,0 +1,114 @@
use super::{
PointerDelivery,
check::ActionabilityCheck,
check_result::{occluded, pass, unknown},
hit_test_evidence::HitTestEvidence,
report::ActionabilityReport,
};
use crate::{
AdapterError, ErrorCode, Point,
action_request::ActionRequest,
adapter::{NativeHandle, PlatformAdapter},
hit_test::HitTestResult,
};
use serde_json::json;
pub(super) fn receives_events_check(
bounds: Option<crate::Rect>,
handle: &NativeHandle,
adapter: &dyn PlatformAdapter,
request: &ActionRequest,
deadline: crate::Deadline,
) -> Result<(ActionabilityCheck, Option<Point>), AdapterError> {
if !request.action.requires_hit_test() {
return Ok((pass("receives_events"), None));
}
let Some(bounds) = bounds else {
return Ok((unknown("receives_events", "bounds unavailable"), None));
};
let mut first_occlusion = None;
let mut evidence = HitTestEvidence {
attempted: 0,
unknown: 0,
occluded: 0,
};
for point in candidate_points(bounds) {
evidence.attempted += 1;
match adapter.hit_test(handle, point.clone(), deadline) {
Ok(HitTestResult::ReachesTarget) => {
let mut check = pass("receives_events");
check.hit_test = Some(evidence);
return Ok((check, Some(point)));
}
Ok(HitTestResult::InterceptedBy { role, name, bounds }) => {
evidence.occluded += 1;
first_occlusion.get_or_insert_with(|| occluded(role, name, bounds));
}
Ok(HitTestResult::Unknown) => evidence.unknown += 1,
Err(error) if error.code == ErrorCode::PlatformNotSupported => {
evidence.unknown += 1;
let mut check = unknown("receives_events", "hit testing is not supported");
check.hit_test = Some(evidence);
return Ok((check, None));
}
Err(error) => return Err(error),
}
}
let mut check = if evidence.unknown > 0 && evidence.occluded > 0 {
let mut mixed = first_occlusion.unwrap_or_else(|| {
unknown(
"receives_events",
"hit test evidence mixed unknown and occluded outcomes",
)
});
mixed.status = super::status::ActionabilityStatus::Unknown;
mixed.reason = Some("hit test evidence mixed unknown and occluded outcomes".into());
mixed
} else if evidence.unknown > 0 {
unknown("receives_events", "hit test result inconclusive")
} else {
first_occlusion
.unwrap_or_else(|| unknown("receives_events", "hit test result inconclusive"))
};
check.hit_test = Some(evidence);
Ok((check, None))
}
fn candidate_points(bounds: crate::Rect) -> [Point; 5] {
let point = |x: f64, y: f64| Point {
x: bounds.x + bounds.width * x,
y: bounds.y + bounds.height * y,
};
[
point(0.5, 0.5),
point(0.25, 0.25),
point(0.75, 0.25),
point(0.25, 0.75),
point(0.75, 0.75),
]
}
pub(crate) fn require_receives_events(
handle: &NativeHandle,
point: Point,
adapter: &dyn PlatformAdapter,
deadline: crate::Deadline,
) -> Result<(), AdapterError> {
let check = match adapter.hit_test(handle, point, deadline) {
Ok(HitTestResult::ReachesTarget) => return Ok(()),
Ok(HitTestResult::Unknown) => return Ok(()),
Err(error) if error.code == ErrorCode::PlatformNotSupported => return Ok(()),
Err(error) => return Err(error),
Ok(HitTestResult::InterceptedBy { role, name, bounds }) => occluded(role, name, bounds),
};
let report = ActionabilityReport::from_checks(vec![check], None, PointerDelivery::Physical);
Err(AdapterError::new(
ErrorCode::ActionFailed,
format!("Target is not actionable: {}", report.failure_reasons()),
)
.with_details(json!(report))
.with_suggestion(
"Wait for the target to become actionable, refresh the snapshot, or use an explicit physical/focus command if intended.",
)
.with_disposition(crate::DeliverySemantics::not_delivered()))
}

91
crates/core/src/actionability/report.rs
View file

@ -1,8 +1,89 @@
use super::ActionabilityCheck;
use super::{PointerDelivery, check::ActionabilityCheck, status::ActionabilityStatus};
use crate::{ErrorCode, Point};
use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct ActionabilityReport {
pub actionable: bool,
pub checks: Vec<ActionabilityCheck>,
#[derive(Debug, Clone, Serialize, PartialEq)]
pub(crate) struct ActionabilityReport {
pub(crate) actionable: bool,
pub(crate) checks: Vec<ActionabilityCheck>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) verified_point: Option<Point>,
#[serde(skip)]
pub(crate) pointer_delivery: PointerDelivery,
}
impl ActionabilityReport {
pub(crate) fn from_checks(
checks: Vec<ActionabilityCheck>,
verified_point: Option<Point>,
pointer_delivery: PointerDelivery,
) -> Self {
let actionable = checks.iter().all(|check| !is_blocking(check));
Self {
actionable,
checks,
verified_point,
pointer_delivery,
}
}
pub(crate) fn terminal_code(&self) -> Option<ErrorCode> {
self.checks
.iter()
.find(|check| is_blocking(check))
.and_then(|check| check.terminal_code.clone())
}
pub(crate) fn failure_reasons(&self) -> String {
self.checks
.iter()
.filter(|check| is_blocking(check))
.map(|check| {
let reason = check.reason.as_deref().unwrap_or("failed");
format!("{} ({reason})", check.check)
})
.collect::<Vec<_>>()
.join(", ")
}
}
fn is_blocking(check: &ActionabilityCheck) -> bool {
matches!(check.status, ActionabilityStatus::Fail)
|| matches!(check.status, ActionabilityStatus::Unknown)
&& (check.check != "receives_events"
|| check
.hit_test
.as_ref()
.is_some_and(|evidence| evidence.occluded > 0))
}
#[cfg(test)]
mod tests {
use super::*;
fn failed(check: &'static str, terminal_code: Option<ErrorCode>) -> ActionabilityCheck {
ActionabilityCheck {
check,
status: ActionabilityStatus::Fail,
reason: None,
occluder: None,
terminal_code,
hit_test: None,
stability: None,
}
}
#[test]
fn retryable_failure_is_not_overridden_by_a_later_terminal_check() {
let report = ActionabilityReport::from_checks(
vec![
failed("enabled", None),
failed("supported_action", Some(ErrorCode::PolicyDenied)),
],
None,
PointerDelivery::NotApplicable,
);
assert_eq!(report.terminal_code(), None);
}
}

85
crates/core/src/actionability/requirements.rs Normal file
View file

@ -0,0 +1,85 @@
use super::PointerDelivery;
use crate::action::Action;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ActionabilityRequirements {
pub(crate) visible: bool,
pub(crate) stable: bool,
pub(crate) enabled: bool,
pub(crate) editable: bool,
pub(crate) receives_events: bool,
}
impl ActionabilityRequirements {
pub(crate) fn for_action(action: &Action) -> Self {
match action {
Action::Click | Action::DoubleClick | Action::RightClick | Action::TripleClick => {
Self::new(true, true, true, false, true)
}
Action::Hover | Action::Drag(_) => Self::new(true, true, false, false, true),
Action::SetValue(_) | Action::TypeText(_) | Action::Clear => {
Self::new(true, false, true, true, false)
}
Action::Select(_) => Self::new(true, false, true, false, false),
Action::Expand
| Action::Collapse
| Action::Toggle
| Action::Check
| Action::Uncheck
| Action::Scroll(_, _) => Self::new(true, true, true, false, false),
Action::ScrollTo => Self::new(false, true, false, false, false),
Action::SetFocus | Action::PressKey(_) | Action::KeyDown(_) | Action::KeyUp(_) => {
Self::new(false, false, false, false, false)
}
}
}
pub(crate) fn pointer_delivery(
&self,
action: &Action,
available_actions: &[String],
policy: crate::InteractionPolicy,
) -> PointerDelivery {
if !self.receives_events {
return PointerDelivery::NotApplicable;
}
if policy.is_headed() && action.headed_requirement().requires_cursor() {
PointerDelivery::Physical
} else if crate::capability::supports_direct_semantic_pointer_delivery(
action,
available_actions,
) {
PointerDelivery::Semantic
} else {
PointerDelivery::Physical
}
}
pub(crate) fn requires_stability(&self, pointer_delivery: PointerDelivery) -> bool {
self.stable && !matches!(pointer_delivery, PointerDelivery::Semantic)
}
const fn new(
visible: bool,
stable: bool,
enabled: bool,
editable: bool,
receives_events: bool,
) -> Self {
Self {
visible,
stable,
enabled,
editable,
receives_events,
}
}
}
pub(crate) fn requires_stability(action: &Action) -> bool {
ActionabilityRequirements::for_action(action).stable
}
#[cfg(test)]
#[path = "requirements_tests.rs"]
mod tests;

88
crates/core/src/actionability/requirements_tests.rs Normal file
View file

@ -0,0 +1,88 @@
use super::*;
use crate::{DragParams, KeyCombo, Point};
fn key() -> KeyCombo {
KeyCombo {
key: "a".into(),
modifiers: Vec::new(),
}
}
#[test]
fn pointer_actions_require_playwright_style_positional_checks() {
let click = ActionabilityRequirements::for_action(&Action::Click);
assert!(click.visible && click.stable && click.enabled && click.receives_events);
let hover = ActionabilityRequirements::for_action(&Action::Hover);
assert!(hover.visible && hover.stable && hover.receives_events);
assert!(!hover.enabled);
let drag = ActionabilityRequirements::for_action(&Action::Drag(DragParams {
from: Point { x: 0.0, y: 0.0 },
to: Point { x: 1.0, y: 1.0 },
duration_ms: None,
drop_delay_ms: None,
}));
assert_eq!(drag, hover);
}
#[test]
fn pointer_delivery_is_selected_from_live_semantic_capability_evidence() {
let requirements = ActionabilityRequirements::for_action(&Action::Click);
assert_eq!(
requirements.pointer_delivery(
&Action::Click,
&[crate::capability::CLICK.into()],
crate::InteractionPolicy::headless(),
),
PointerDelivery::Semantic
);
assert_eq!(
requirements.pointer_delivery(&Action::Click, &[], crate::InteractionPolicy::headless(),),
PointerDelivery::Physical
);
assert_eq!(
requirements.pointer_delivery(
&Action::DoubleClick,
&[crate::capability::CLICK.into()],
crate::InteractionPolicy::headless(),
),
PointerDelivery::Physical
);
assert_eq!(
requirements.pointer_delivery(
&Action::Click,
&[crate::capability::CLICK.into()],
crate::InteractionPolicy::headed(),
),
PointerDelivery::Physical
);
assert!(!requirements.requires_stability(PointerDelivery::Semantic));
assert!(requirements.requires_stability(PointerDelivery::Physical));
}
#[test]
fn editing_actions_do_not_require_positional_stability_or_hit_testing() {
let fill = ActionabilityRequirements::for_action(&Action::SetValue("x".into()));
assert!(fill.visible && fill.enabled && fill.editable);
assert!(!fill.stable && !fill.receives_events);
}
#[test]
fn keyboard_and_focus_actions_skip_irrelevant_geometry_checks() {
for action in [
Action::SetFocus,
Action::PressKey(key()),
Action::KeyDown(key()),
Action::KeyUp(key()),
] {
let requirements = ActionabilityRequirements::for_action(&action);
assert!(!requirements.visible);
assert!(!requirements.stable);
assert!(!requirements.enabled);
assert!(!requirements.editable);
assert!(!requirements.receives_events);
}
}

40
crates/core/src/actionability/stability.rs Normal file
View file

@ -0,0 +1,40 @@
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct StabilityExpectation {
pub(crate) bounds_hash: Option<u64>,
pub(crate) bounds: Option<crate::Rect>,
pub(crate) strict: bool,
pub(crate) samples: u32,
pub(crate) span_ms: u64,
}
impl StabilityExpectation {
pub(crate) fn permissive(bounds_hash: Option<u64>) -> Self {
Self {
bounds_hash,
bounds: None,
strict: false,
samples: 1,
span_ms: 0,
}
}
pub(crate) fn strict(bounds: Option<crate::Rect>, samples: u32, span_ms: u64) -> Self {
Self {
bounds_hash: bounds.and_then(|bounds| bounds.bounds_hash()),
bounds,
strict: true,
samples,
span_ms,
}
}
pub(crate) fn strict_hash(bounds_hash: Option<u64>) -> Self {
Self {
bounds_hash,
bounds: None,
strict: true,
samples: 1,
span_ms: 0,
}
}
}

7
crates/core/src/actionability/stability_evidence.rs Normal file
View file

@ -0,0 +1,7 @@
use serde::Serialize;
#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
pub(crate) struct StabilityEvidence {
pub(crate) samples: u32,
pub(crate) span_ms: u64,
}

69
crates/core/src/actionability/stability_sampler.rs Normal file
View file

@ -0,0 +1,69 @@
use crate::Rect;
use std::time::Duration;
pub(crate) const STABILITY_SAMPLE_INTERVAL: Duration = Duration::from_millis(17);
pub(crate) const MIN_STABILITY_SPAN: Duration = Duration::from_millis(34);
const GEOMETRY_TOLERANCE: f64 = 0.5;
pub(crate) struct StabilitySampler {
previous: Option<Rect>,
consecutive: u32,
stable_since: Duration,
}
impl StabilitySampler {
pub(crate) fn new() -> Self {
Self {
previous: None,
consecutive: 0,
stable_since: Duration::ZERO,
}
}
pub(crate) fn observe(&mut self, bounds: Option<Rect>, elapsed: Duration) -> bool {
let Some(bounds) = bounds else {
self.previous = None;
self.consecutive = 0;
self.stable_since = elapsed;
return false;
};
if self
.previous
.is_some_and(|previous| geometry_matches(previous, bounds))
{
self.consecutive += 1;
} else {
self.consecutive = 1;
self.stable_since = elapsed;
}
self.previous = Some(bounds);
self.consecutive >= 3 && elapsed.saturating_sub(self.stable_since) >= MIN_STABILITY_SPAN
}
pub(crate) fn samples(&self) -> u32 {
self.consecutive
}
pub(crate) fn stable_span(&self, elapsed: Duration) -> Duration {
elapsed.saturating_sub(self.stable_since)
}
pub(crate) fn bounds(&self) -> Option<Rect> {
self.previous
}
}
pub(crate) fn geometry_matches(left: Rect, right: Rect) -> bool {
[
(left.x, right.x),
(left.y, right.y),
(left.width, right.width),
(left.height, right.height),
]
.into_iter()
.all(|(left, right)| (left - right).abs() <= GEOMETRY_TOLERANCE)
}
#[cfg(test)]
#[path = "stability_sampler_tests.rs"]
mod tests;

38
crates/core/src/actionability/stability_sampler_tests.rs Normal file
View file

@ -0,0 +1,38 @@
use super::*;
fn rect(x: f64) -> Rect {
Rect {
x,
y: 0.0,
width: 100.0,
height: 20.0,
}
}
#[test]
fn thirty_hertz_change_cannot_pass_on_phase_aligned_equal_samples() {
let mut sampler = StabilitySampler::new();
assert!(!sampler.observe(Some(rect(0.0)), Duration::ZERO));
assert!(!sampler.observe(Some(rect(0.0)), Duration::from_millis(17)));
assert!(!sampler.observe(Some(rect(10.0)), Duration::from_millis(34)));
assert!(!sampler.observe(Some(rect(10.0)), Duration::from_millis(51)));
assert!(sampler.observe(Some(rect(10.0)), Duration::from_millis(68)));
}
#[test]
fn sixty_hertz_motion_resets_the_stability_window() {
let mut sampler = StabilitySampler::new();
for (index, elapsed) in [0, 17, 34, 51].into_iter().enumerate() {
assert!(!sampler.observe(Some(rect(index as f64)), Duration::from_millis(elapsed)));
}
assert!(!sampler.observe(Some(rect(3.0)), Duration::from_millis(68)));
assert!(sampler.observe(Some(rect(3.0)), Duration::from_millis(85)));
}
#[test]
fn subpixel_jitter_within_tolerance_is_stable() {
let mut sampler = StabilitySampler::new();
assert!(!sampler.observe(Some(rect(10.0)), Duration::ZERO));
assert!(!sampler.observe(Some(rect(10.2)), Duration::from_millis(17)));
assert!(sampler.observe(Some(rect(9.8)), Duration::from_millis(34)));
}

2
crates/core/src/actionability/status.rs
View file

@ -2,7 +2,7 @@ use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ActionabilityStatus {
pub(crate) enum ActionabilityStatus {
Pass,
Fail,
Unknown,

63
crates/core/src/actionability_live_failure_tests.rs Normal file
View file

@ -0,0 +1,63 @@
use super::*;
#[test]
fn empty_live_actions_replace_stale_snapshot_capabilities() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.geometry.bounds,
actions: Some(vec![]),
};
let err = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("supported_action"));
}
#[test]
fn unsupported_live_reads_fail_closed() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&UnsupportedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::ActionNotSupported);
assert!(err.message.contains("Live element evidence"));
}
#[test]
fn empty_live_element_fails_as_stale_before_dispatch() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&DeadLiveElementAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::StaleRef);
assert!(err.message.contains("changed role"));
}
#[test]
fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&LiveReadErrorAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::PermDenied);
}

311
crates/core/src/actionability_live_tests.rs
View file

@ -1,11 +1,12 @@
use super::*;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::{
AdapterError, ErrorCode, Rect,
action::Action,
action_request::ActionRequest,
adapter::{LiveElement, NativeHandle, PlatformAdapter, SnapshotSurface},
adapter::{LiveElement, NativeHandle, SnapshotSurface},
capability,
element_state::ElementState,
node::Rect,
refs::RefEntry,
};
@ -15,87 +16,204 @@ struct LiveAdapter {
actions: Option<Vec<String>>,
}
impl PlatformAdapter for LiveAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
impl ObservationOps for LiveAdapter {
fn get_live_element(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
identity: crate::adapter::live_identity("OK"),
state: self.state.clone().unwrap_or_else(|| ElementState {
role: "button".into(),
states: Vec::new(),
value: None,
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
}),
states_complete: true,
bounds: self.bounds,
available_actions: self.actions.clone().unwrap_or_default(),
})
}
fn get_live_state(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<ElementState>, AdapterError> {
Ok(self.state.clone())
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
Ok(self.bounds)
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Vec<String>>, AdapterError> {
Ok(self.actions.clone())
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<crate::hit_test::HitTestResult, AdapterError> {
Ok(crate::hit_test::HitTestResult::ReachesTarget)
}
}
impl ActionOps for LiveAdapter {}
impl InputOps for LiveAdapter {}
impl SystemOps for LiveAdapter {}
struct CombinedLiveAdapter;
impl PlatformAdapter for CombinedLiveAdapter {
fn get_live_element(&self, _handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
impl ObservationOps for CombinedLiveAdapter {
fn get_live_element(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
state: Some(ElementState {
identity: crate::adapter::live_identity("OK"),
state: ElementState {
role: "button".into(),
states: vec![],
value: None,
}),
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
},
states_complete: true,
bounds: Some(Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}),
available_actions: Some(vec![capability::CLICK.into()]),
available_actions: vec![capability::CLICK.into()],
})
}
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
fn get_live_state(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<ElementState>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Vec<String>>, AdapterError> {
panic!("check_live should use get_live_element")
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<crate::hit_test::HitTestResult, AdapterError> {
Ok(crate::hit_test::HitTestResult::ReachesTarget)
}
}
impl ActionOps for CombinedLiveAdapter {}
impl InputOps for CombinedLiveAdapter {}
impl SystemOps for CombinedLiveAdapter {}
struct LiveReadErrorAdapter;
impl PlatformAdapter for LiveReadErrorAdapter {
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
impl ObservationOps for LiveReadErrorAdapter {
fn get_live_element(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<LiveElement, AdapterError> {
Err(AdapterError::permission_denied())
}
fn get_live_state(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<ElementState>, AdapterError> {
Err(AdapterError::permission_denied())
}
}
impl ActionOps for LiveReadErrorAdapter {}
impl InputOps for LiveReadErrorAdapter {}
impl SystemOps for LiveReadErrorAdapter {}
struct UnsupportedLiveAdapter;
impl PlatformAdapter for UnsupportedLiveAdapter {}
impl ObservationOps for UnsupportedLiveAdapter {}
impl ActionOps for UnsupportedLiveAdapter {}
impl InputOps for UnsupportedLiveAdapter {}
impl SystemOps for UnsupportedLiveAdapter {}
struct DeadLiveElementAdapter;
impl PlatformAdapter for DeadLiveElementAdapter {
fn get_live_element(&self, _handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
impl ObservationOps for DeadLiveElementAdapter {
fn get_live_element(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<LiveElement, AdapterError> {
Ok(LiveElement {
state: Some(ElementState {
identity: crate::adapter::live_identity("OK"),
state: ElementState {
role: "unknown".into(),
states: vec![],
value: None,
}),
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
},
states_complete: true,
bounds: None,
available_actions: Some(vec![]),
available_actions: vec![],
})
}
}
impl ActionOps for DeadLiveElementAdapter {}
impl InputOps for DeadLiveElementAdapter {}
impl SystemOps for DeadLiveElementAdapter {}
fn entry() -> RefEntry {
let bounds = Rect {
x: 1.0,
@ -104,36 +222,54 @@ fn entry() -> RefEntry {
height: 20.0,
};
RefEntry {
pid: 1,
role: "button".into(),
name: Some("OK".into()),
value: None,
description: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
source_surface: SnapshotSurface::Window,
root_ref: None,
path_is_absolute: true,
path: smallvec::SmallVec::new(),
process: crate::RefProcess {
pid: crate::ProcessId::new(1),
process_instance: Some("test-instance".into()),
},
identity: crate::RefEntryIdentity {
role: "button".into(),
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
},
geometry: crate::RefGeometry {
bounds: Some(bounds),
bounds_hash: bounds.bounds_hash(),
},
capabilities: crate::RefCapabilities {
states: vec![],
available_actions: vec![capability::CLICK.into()],
},
source: crate::RefSource {
source_app: None,
source_window_id: None,
source_window_title: None,
source_window_bounds_hash: None,
source_surface: SnapshotSurface::Window,
},
scope: crate::RefScope {
root_ref: None,
path_is_absolute: true,
path: smallvec::SmallVec::new(),
},
}
}
#[test]
fn live_actionability_overrides_stale_snapshot_state() {
let mut stale = entry();
stale.states.push("disabled".into());
stale.capabilities.states.push("disabled".into());
let adapter = LiveAdapter {
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
}),
bounds: stale.bounds,
bounds: stale.geometry.bounds,
actions: Some(vec![capability::CLICK.into()]),
};
@ -151,14 +287,14 @@ fn live_actionability_overrides_stale_snapshot_state() {
#[test]
fn live_actionability_uses_combined_live_element_read() {
let mut stale = entry();
stale.states.push("disabled".into());
stale.bounds = Some(Rect {
stale.capabilities.states.push("disabled".into());
stale.geometry.bounds = Some(Rect {
x: 1.0,
y: 1.0,
width: 0.0,
height: 20.0,
});
stale.available_actions = vec![];
stale.capabilities.available_actions = vec![];
let report = check_live(
&stale,
@ -174,10 +310,17 @@ fn live_actionability_uses_combined_live_element_read() {
#[test]
fn live_actionability_uses_actions_gained_after_snapshot() {
let mut stale = entry();
stale.available_actions = vec![];
stale.capabilities.available_actions = vec![];
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
state: Some(ElementState {
role: "button".into(),
states: vec![],
value: None,
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
}),
bounds: stale.geometry.bounds,
actions: Some(vec![capability::CLICK.into()]),
};
@ -197,7 +340,7 @@ fn live_actionability_fails_when_action_disappears_after_snapshot() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
bounds: stale.geometry.bounds,
actions: Some(vec![capability::SET_VALUE.into()]),
};
@ -209,12 +352,12 @@ fn live_actionability_fails_when_action_disappears_after_snapshot() {
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("supported_action"));
}
#[test]
fn live_actionability_allows_identity_resolved_bounds_change() {
fn live_actionability_rejects_unstable_identity_resolved_bounds_change() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
@ -227,79 +370,17 @@ fn live_actionability_allows_identity_resolved_bounds_change() {
actions: Some(vec![capability::CLICK.into()]),
};
let report = check_live(
let err = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
let stable = report
.checks
.iter()
.find(|check| check.name == "stable")
.unwrap();
assert_eq!(stable.status, ActionabilityStatus::Unknown);
}
#[test]
fn empty_live_actions_do_not_erase_snapshot_capabilities() {
let stale = entry();
let adapter = LiveAdapter {
state: None,
bounds: stale.bounds,
actions: Some(vec![]),
};
let report = check_live(
&stale,
&NativeHandle::null(),
&adapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn unsupported_live_reads_fall_back_to_snapshot_entry() {
let report = check_live(
&entry(),
&NativeHandle::null(),
&UnsupportedLiveAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap();
assert!(report.actionable);
}
#[test]
fn empty_live_element_fails_as_stale_before_dispatch() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&DeadLiveElementAdapter,
&ActionRequest::headless(Action::Click),
&ActionRequest::headed(Action::DoubleClick),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::StaleRef);
assert!(err.message.contains("no longer exposes live"));
assert_eq!(err.code, ErrorCode::ActionFailed);
assert!(err.message.contains("stable"));
}
#[test]
fn live_read_errors_are_not_silently_downgraded_to_snapshot_data() {
let err = check_live(
&entry(),
&NativeHandle::null(),
&LiveReadErrorAdapter,
&ActionRequest::headless(Action::Click),
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::PermDenied);
}
#[path = "actionability_live_failure_tests.rs"]
mod failure_tests;

266
crates/core/src/actionability_tests.rs
View file

@ -1,11 +1,7 @@
use super::*;
use crate::{
action::{Action, Direction},
action_request::ActionRequest,
adapter::SnapshotSurface,
capability,
node::Rect,
refs::RefEntry,
Action, Direction, ErrorCode, Rect, action_request::ActionRequest, adapter::SnapshotSurface,
capability, refs::RefEntry,
};
fn entry() -> RefEntry {
@ -16,25 +12,106 @@ fn entry() -> RefEntry {
height: 20.0,
};
RefEntry {
pid: 1,
role: "button".into(),
name: Some("OK".into()),
value: None,
description: None,
states: vec![],
bounds: Some(bounds),
bounds_hash: Some(bounds.bounds_hash()),
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
source_surface: SnapshotSurface::Window,
root_ref: None,
path_is_absolute: true,
path: smallvec::SmallVec::new(),
process: crate::RefProcess {
pid: crate::ProcessId::new(1),
process_instance: Some("test-instance".into()),
},
identity: crate::RefEntryIdentity {
role: "button".into(),
name: Some("OK".into()),
value: None,
description: None,
native_id: None,
},
geometry: crate::RefGeometry {
bounds: Some(bounds),
bounds_hash: bounds.bounds_hash(),
},
capabilities: crate::RefCapabilities {
states: vec![],
available_actions: vec![capability::CLICK.into()],
},
source: crate::RefSource {
source_app: None,
source_window_id: None,
source_window_title: None,
source_window_bounds_hash: None,
source_surface: SnapshotSurface::Window,
},
scope: crate::RefScope {
root_ref: None,
path_is_absolute: true,
path: smallvec::SmallVec::new(),
},
}
}
fn visibility_evidence(
hidden: Option<bool>,
states: Vec<String>,
states_complete: bool,
) -> evidence::ActionabilityEvidence {
evidence::ActionabilityEvidence {
state: crate::ElementState {
role: "button".into(),
states,
value: None,
enabled: Some(true),
hidden,
offscreen: Some(false),
},
states_complete,
bounds: Some(Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}),
available_actions: vec![capability::CLICK.into()],
}
}
#[test]
fn explicit_typed_hidden_state_wins() {
let check = gates::visibility(&visibility_evidence(Some(true), Vec::new(), true));
assert_eq!(check.status, ActionabilityStatus::Fail);
assert_eq!(check.reason.as_deref(), Some("live hidden state is true"));
}
#[test]
fn complete_canonical_hidden_state_fails_when_typed_state_is_absent() {
let check = gates::visibility(&visibility_evidence(
None,
vec![crate::state::HIDDEN.into()],
true,
));
assert_eq!(check.status, ActionabilityStatus::Fail);
assert_eq!(
check.reason.as_deref(),
Some("canonical hidden state is present")
);
}
#[test]
fn complete_canonical_states_without_hidden_allow_visibility_checks_to_continue() {
let check = gates::visibility(&visibility_evidence(None, Vec::new(), true));
assert_eq!(check.status, ActionabilityStatus::Pass);
}
#[test]
fn incomplete_canonical_states_keep_missing_typed_hidden_unknown() {
let check = gates::visibility(&visibility_evidence(None, Vec::new(), false));
assert_eq!(check.status, ActionabilityStatus::Unknown);
assert_eq!(
check.reason.as_deref(),
Some("live hidden state unavailable")
);
}
#[test]
fn click_passes_when_target_is_enabled_visible_and_supported() {
let report = check(&entry(), &ActionRequest::headless(Action::Click)).unwrap();
@ -42,10 +119,27 @@ fn click_passes_when_target_is_enabled_visible_and_supported() {
assert!(report.actionable);
}
#[test]
fn scroll_to_attempts_adapter_delivery_without_a_native_element_capability() {
let mut target = entry();
target.capabilities.available_actions.clear();
let report = check(&target, &ActionRequest::headless(Action::ScrollTo)).unwrap();
assert!(report.actionable);
}
#[test]
fn states_are_enabled_reads_the_canonical_disabled_token() {
assert!(!states_are_enabled(&[crate::state::DISABLED.to_string()]));
assert!(states_are_enabled(&[]));
assert!(states_are_enabled(&[crate::state::FOCUSED.to_string()]));
}
#[test]
fn disabled_entry_fails_before_action_dispatch() {
let mut entry = entry();
entry.states.push("disabled".into());
entry.capabilities.states.push("disabled".into());
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
@ -63,14 +157,68 @@ fn zero_sized_bounds_fail_visibility() {
width: 0.0,
height: 20.0,
};
entry.bounds = Some(bounds);
entry.bounds_hash = Some(bounds.bounds_hash());
entry.geometry.bounds = Some(bounds);
entry.geometry.bounds_hash = bounds.bounds_hash();
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
assert!(err.message.contains("visible"));
}
#[test]
fn hidden_state_fails_visibility_before_action_dispatch() {
let mut entry = entry();
entry.capabilities.states.push(crate::state::HIDDEN.into());
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
assert!(err.message.contains("visible"));
}
#[test]
fn offscreen_state_fails_visibility_before_action_dispatch() {
let mut entry = entry();
entry
.capabilities
.states
.push(crate::state::OFFSCREEN.into());
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
assert!(err.message.contains("visible"));
}
#[test]
fn hidden_entry_fails_visibility_even_when_bounds_are_none() {
let mut entry = entry();
entry.capabilities.states.push(crate::state::HIDDEN.into());
entry.geometry.bounds = None;
entry.geometry.bounds_hash = None;
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert!(err.message.contains("visible"));
assert!(err.message.contains("hidden"));
}
#[test]
fn offscreen_entry_fails_visibility_even_when_bounds_are_none() {
let mut entry = entry();
entry
.capabilities
.states
.push(crate::state::OFFSCREEN.into());
entry.geometry.bounds = None;
entry.geometry.bounds_hash = None;
let err = check(&entry, &ActionRequest::headless(Action::Click)).unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert!(err.message.contains("visible"));
assert!(err.message.contains("offscreen"));
}
#[test]
fn text_input_requires_editable_target() {
let err = check(
@ -79,6 +227,7 @@ fn text_input_requires_editable_target() {
)
.unwrap_err();
assert_eq!(err.code, ErrorCode::ActionNotSupported);
assert!(err.message.contains("editable"));
}
@ -86,14 +235,15 @@ fn text_input_requires_editable_target() {
fn cursor_movement_requires_physical_policy() {
let err = check(&entry(), &ActionRequest::headless(Action::Hover)).unwrap_err();
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("policy"));
}
#[test]
fn headless_type_text_fails_policy_before_dispatch() {
let mut target = entry();
target.role = "textfield".into();
target.available_actions = vec![capability::SET_VALUE.into()];
target.identity.role = "textfield".into();
target.capabilities.available_actions = vec![capability::SET_VALUE.into()];
let err = check(
&target,
@ -101,43 +251,59 @@ fn headless_type_text_fails_policy_before_dispatch() {
)
.unwrap_err();
assert!(err.message.contains("policy"));
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("focus"));
}
#[test]
fn right_click_requires_right_click_capability_before_dispatch() {
fn headless_right_click_denies_physical_fallback() {
let err = check(&entry(), &ActionRequest::headless(Action::RightClick)).unwrap_err();
assert_eq!(err.code, ErrorCode::ActionFailed);
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("supported_action"));
}
#[test]
fn headless_multi_clicks_deny_physical_fallback_before_hit_testing() {
for action in [Action::DoubleClick, Action::TripleClick] {
let err = check(&entry(), &ActionRequest::headless(action)).unwrap_err();
assert_eq!(err.code, ErrorCode::PolicyDenied);
assert!(err.message.contains("supported_action"));
}
}
#[test]
fn headed_click_allows_verified_physical_fallback() {
let mut target = entry();
target.capabilities.available_actions.clear();
assert!(check(&target, &ActionRequest::headed(Action::Click)).is_ok());
}
#[test]
fn headed_multi_clicks_require_and_allow_physical_delivery() {
let mut target = entry();
target.capabilities.available_actions.clear();
for action in [Action::DoubleClick, Action::TripleClick] {
assert!(check(&target, &ActionRequest::headed(action)).is_ok());
}
}
#[test]
fn command_aliases_match_platform_capabilities() {
let click_entry = entry();
assert!(check(&click_entry, &ActionRequest::headless(Action::DoubleClick)).is_ok());
assert!(check(&click_entry, &ActionRequest::headless(Action::TripleClick)).is_ok());
assert!(check(&click_entry, &ActionRequest::headless(Action::Check)).is_ok());
assert!(check(&click_entry, &ActionRequest::headless(Action::Uncheck)).is_ok());
let mut editable = entry();
editable.role = "textfield".into();
editable.available_actions = vec![capability::SET_VALUE.into()];
editable.identity.role = "textfield".into();
editable.capabilities.available_actions = vec![capability::SET_VALUE.into()];
assert!(check(&editable, &ActionRequest::headless(Action::Clear)).is_ok());
let mut scrollable = entry();
scrollable.available_actions = vec![capability::SCROLL.into()];
assert!(
check(
&scrollable,
&ActionRequest::headless(Action::Scroll(Direction::Down, 1))
)
.is_ok()
);
assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_err());
scrollable.available_actions = vec![capability::SCROLL_TO.into()];
scrollable.capabilities.available_actions = vec![capability::SCROLL.into()];
assert!(
check(
&scrollable,
@ -146,4 +312,14 @@ fn command_aliases_match_platform_capabilities() {
.is_ok()
);
assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_ok());
scrollable.capabilities.available_actions = vec![capability::SCROLL_TO.into()];
assert!(
check(
&scrollable,
&ActionRequest::headless(Action::Scroll(Direction::Down, 1))
)
.is_err()
);
assert!(check(&scrollable, &ActionRequest::headless(Action::ScrollTo)).is_ok());
}

397
crates/core/src/adapter.rs
View file

@ -1,397 +0,0 @@
use crate::{
PermissionReport, PermissionState,
action::{DragParams, KeyCombo, MouseEvent, WindowOp},
action_request::ActionRequest,
action_result::ActionResult,
element_state::ElementState,
error::{AdapterError, ErrorCode},
node::{AccessibilityNode, AppInfo, Rect, SurfaceInfo, WindowInfo},
notification::{NotificationFilter, NotificationIdentity, NotificationInfo},
refs::RefEntry,
};
use std::marker::PhantomData;
pub struct WindowFilter {
pub focused_only: bool,
pub app: Option<String>,
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, Default, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum SnapshotSurface {
#[default]
Window,
Focused,
Menu,
Menubar,
Sheet,
Popover,
Alert,
}
impl SnapshotSurface {
pub fn is_window(surface: &Self) -> bool {
matches!(surface, Self::Window)
}
}
#[derive(Clone, Copy)]
pub struct TreeOptions {
pub max_depth: u8,
pub include_bounds: bool,
pub interactive_only: bool,
pub compact: bool,
pub surface: SnapshotSurface,
pub skeleton: bool,
}
impl Default for TreeOptions {
fn default() -> Self {
Self {
max_depth: 10,
include_bounds: false,
interactive_only: false,
compact: false,
surface: SnapshotSurface::Window,
skeleton: false,
}
}
}
impl TreeOptions {
pub(crate) fn with_ref_identity_bounds(mut self) -> Self {
self.include_bounds = true;
self
}
}
#[derive(Debug, Clone, Default)]
pub struct LiveElement {
pub state: Option<ElementState>,
pub bounds: Option<Rect>,
pub available_actions: Option<Vec<String>>,
}
pub(crate) fn optional_live_read<T>(
result: Result<Option<T>, AdapterError>,
) -> Result<Option<T>, AdapterError> {
match result {
Ok(value) => Ok(value),
Err(err) if is_live_read_unsupported(&err) => Ok(None),
Err(err) => Err(err),
}
}
fn is_live_read_unsupported(err: &AdapterError) -> bool {
matches!(
err.code,
ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported
)
}
pub enum ScreenshotTarget {
Screen(usize),
/// Capture the largest visible window owned by this process ID.
Window(i32),
FullScreen,
}
pub struct NativeHandle {
pub(crate) ptr: *const std::ffi::c_void,
_not_send_sync: PhantomData<*const ()>,
}
impl NativeHandle {
/// # Safety
///
/// `ptr` must be a valid platform accessibility handle whose ownership is
/// transferred to the caller. The adapter that creates the handle must
/// document how it is released through [`PlatformAdapter::release_handle`].
pub unsafe fn from_ptr(ptr: *const std::ffi::c_void) -> Self {
Self {
ptr,
_not_send_sync: PhantomData,
}
}
pub fn null() -> Self {
Self {
ptr: std::ptr::null(),
_not_send_sync: PhantomData,
}
}
}
impl NativeHandle {
/// Returns the raw platform pointer. For use by platform adapter crates only.
/// Callers must not retain the pointer beyond the lifetime of this handle.
pub fn as_raw(&self) -> *const std::ffi::c_void {
self.ptr
}
}
pub struct ImageBuffer {
pub data: Vec<u8>,
pub format: ImageFormat,
pub width: u32,
pub height: u32,
}
pub enum ImageFormat {
Png,
Jpg,
}
impl ImageFormat {
pub fn as_str(&self) -> &'static str {
match self {
ImageFormat::Png => "png",
ImageFormat::Jpg => "jpg",
}
}
}
pub trait PlatformAdapter: Send + Sync {
fn list_windows(&self, _filter: &WindowFilter) -> Result<Vec<WindowInfo>, AdapterError> {
Err(AdapterError::not_supported("list_windows"))
}
fn list_apps(&self) -> Result<Vec<AppInfo>, AdapterError> {
Err(AdapterError::not_supported("list_apps"))
}
fn get_tree(
&self,
_win: &WindowInfo,
_opts: &TreeOptions,
) -> Result<AccessibilityNode, AdapterError> {
Err(AdapterError::not_supported("get_tree"))
}
fn execute_action(
&self,
_handle: &NativeHandle,
_request: ActionRequest,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::not_supported("execute_action"))
}
fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> {
Err(AdapterError::not_supported("resolve_element_strict"))
}
/// Resolves an element under a caller deadline. Defaults to delegating
/// to [`PlatformAdapter::resolve_element_strict`], ignoring the timeout,
/// so adapters that implement only the un-timed variant still support
/// `wait --element`. Override to honor the remaining budget.
fn resolve_element_strict_with_timeout(
&self,
entry: &RefEntry,
timeout: std::time::Duration,
) -> Result<NativeHandle, AdapterError> {
tracing::trace!(
?timeout,
"resolve_element_strict_with_timeout: default impl does not enforce the deadline; override to honor it"
);
self.resolve_element_strict(entry)
}
/// Releases a platform handle that an implementation took ownership of during resolve.
/// Adapter methods that receive `&NativeHandle` borrow it only; they must not consume
/// or release it. The default no-op is correct for adapters whose handles are owned
/// or freed elsewhere.
fn release_handle(&self, _handle: &NativeHandle) -> Result<(), AdapterError> {
Ok(())
}
fn permission_report(&self) -> PermissionReport {
PermissionReport {
accessibility: PermissionState::Denied {
suggestion: "Platform adapter not available".into(),
},
screen_recording: PermissionState::Unknown,
automation: PermissionState::NotRequired,
}
}
fn unknown_accessibility_means_unsupported(&self) -> bool {
true
}
fn request_permissions(&self) -> PermissionReport {
self.permission_report()
}
fn focus_window(&self, _win: &WindowInfo) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("focus_window"))
}
/// Brings the application owning `pid` to the foreground. Best-effort guard
/// invoked before physical (cursor/keyboard) input that targets a known
/// element, so synthetic events land on the intended window rather than
/// whatever happens to be frontmost.
fn focus_app(&self, _pid: i32) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("focus_app"))
}
fn launch_app(&self, _id: &str, _timeout_ms: u64) -> Result<WindowInfo, AdapterError> {
Err(AdapterError::not_supported("launch_app"))
}
fn close_app(&self, _id: &str, _force: bool) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("close_app"))
}
/// Reports whether closing `identifier` would terminate a process the OS
/// depends on (window server, login session, shell). The set is
/// inherently platform-specific, so each adapter owns its own list;
/// core only asks. The default denies nothing.
fn is_protected_process(&self, _identifier: &str) -> bool {
false
}
/// Reports whether `combo` is a platform-dangerous keyboard shortcut that
/// should be refused unless the caller explicitly forces it (for example
/// macOS Cmd+Q quit, Ctrl+Cmd+Q lock, Cmd+Alt+Esc force-quit). Which
/// combos are dangerous — and how key names alias to physical keys — is
/// platform-specific, so each adapter owns its own list; core only asks
/// and lets the caller override via `--force`. The default blocks nothing,
/// leaving the decision entirely to the calling agent.
fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool {
false
}
fn screenshot(&self, _target: ScreenshotTarget) -> Result<ImageBuffer, AdapterError> {
Err(AdapterError::not_supported("screenshot"))
}
fn get_clipboard(&self) -> Result<String, AdapterError> {
Err(AdapterError::not_supported("get_clipboard"))
}
fn set_clipboard(&self, _text: &str) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("set_clipboard"))
}
fn focused_window(&self) -> Result<Option<WindowInfo>, AdapterError> {
Err(AdapterError::not_supported("focused_window"))
}
fn get_live_value(&self, _handle: &NativeHandle) -> Result<Option<String>, AdapterError> {
Err(AdapterError::not_supported("get_live_value"))
}
fn get_live_state(&self, _handle: &NativeHandle) -> Result<Option<ElementState>, AdapterError> {
Err(AdapterError::not_supported("get_live_state"))
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
) -> Result<Option<Vec<String>>, AdapterError> {
Err(AdapterError::not_supported("get_live_actions"))
}
fn get_live_element(&self, handle: &NativeHandle) -> Result<LiveElement, AdapterError> {
let live = LiveElement {
state: optional_live_read(self.get_live_state(handle))?,
bounds: optional_live_read(self.get_element_bounds(handle))?,
available_actions: optional_live_read(self.get_live_actions(handle))?,
};
if live.state.is_none() && live.bounds.is_none() && live.available_actions.is_none() {
return Err(AdapterError::not_supported("get_live_element"));
}
Ok(live)
}
fn press_key_for_app(
&self,
_app_name: &str,
_combo: &crate::action::KeyCombo,
) -> Result<crate::action_result::ActionResult, AdapterError> {
Err(AdapterError::not_supported("press_key_for_app"))
}
fn wait_for_menu(&self, _pid: i32, _open: bool, _timeout_ms: u64) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("wait_for_menu"))
}
fn list_surfaces(&self, _pid: i32) -> Result<Vec<SurfaceInfo>, AdapterError> {
Err(AdapterError::not_supported("list_surfaces"))
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
Err(AdapterError::not_supported("get_element_bounds"))
}
fn window_op(&self, _win: &WindowInfo, _op: WindowOp) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("window_op"))
}
fn mouse_event(&self, _event: MouseEvent) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("mouse_event"))
}
fn key_event(&self, _combo: &KeyCombo, _down: bool) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("key_event"))
}
fn drag(&self, _params: DragParams) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("drag"))
}
fn clear_clipboard(&self) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("clear_clipboard"))
}
fn list_notifications(
&self,
_filter: &NotificationFilter,
) -> Result<Vec<NotificationInfo>, AdapterError> {
Err(AdapterError::not_supported("list_notifications"))
}
fn dismiss_notification(
&self,
_index: usize,
_app_filter: Option<&str>,
) -> Result<NotificationInfo, AdapterError> {
Err(AdapterError::not_supported("dismiss_notification"))
}
fn dismiss_all_notifications(
&self,
_app_filter: Option<&str>,
) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> {
Err(AdapterError::not_supported("dismiss_all_notifications"))
}
/// Press a named action button on the notification at `index`.
///
/// `identity` lets the caller pin the targeted notification to an
/// expected app / title fingerprint. Notification Center reindexes
/// entries between listings, so index-only targeting can press the
/// wrong button if a notification arrives or is dismissed between
/// `list_notifications` and this call. When any identity field is
/// `Some`, implementations must return
/// `ErrorCode::NotificationNotFound` if the row at `index` does not
/// match. Passing an empty identity (or `None`) preserves legacy
/// index-only behavior for callers that reconcile themselves.
fn notification_action(
&self,
_index: usize,
_identity: Option<&NotificationIdentity>,
_action_name: &str,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::not_supported("notification_action"))
}
fn get_subtree(
&self,
_handle: &NativeHandle,
_opts: &TreeOptions,
) -> Result<AccessibilityNode, AdapterError> {
Err(AdapterError::not_supported("get_subtree"))
}
}

24
crates/core/src/adapter/actions.rs Normal file
View file

@ -0,0 +1,24 @@
use crate::{
AdapterError, InteractionLease, action_request::ActionRequest, action_result::ActionResult,
native_handle::NativeHandle,
};
pub trait ActionOps: Send + Sync {
fn execute_action(
&self,
_handle: &NativeHandle,
_request: ActionRequest,
_lease: &InteractionLease,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::not_supported("execute_action"))
}
fn scroll_into_view(
&self,
handle: &NativeHandle,
lease: &InteractionLease,
) -> Result<(), AdapterError> {
let _ = (handle, lease);
Err(AdapterError::not_supported("scroll_into_view"))
}
}

59
crates/core/src/adapter/input.rs Normal file
View file

@ -0,0 +1,59 @@
use crate::{
AdapterError, ClipboardContent, ClipboardFormat, Deadline, DragParams, InteractionLease,
KeyCombo, MouseEvent,
};
/// `get_clipboard`/`set_clipboard` were removed pre-1.0 in favor of
/// `get_clipboard_content`/`set_clipboard_content`; the C ABI
/// (`ad_get_clipboard`/`ad_set_clipboard`) is unaffected.
pub trait InputOps: Send + Sync {
fn mouse_event(
&self,
_event: MouseEvent,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("mouse_event"))
}
fn key_event(
&self,
_combo: &KeyCombo,
_down: bool,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("key_event"))
}
fn drag(&self, _params: DragParams, _lease: &InteractionLease) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("drag"))
}
fn clear_clipboard(&self, _lease: &InteractionLease) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("clear_clipboard"))
}
/// Reads the requested clipboard representation. Returns `Ok(None)`
/// when the pasteboard has no data of the requested shape (or, for
/// `Auto`, no data at all) — a normal, non-error outcome distinct from
/// `Err(not_supported)`, which means this platform never implements
/// clipboard reads.
fn get_clipboard_content(
&self,
_format: ClipboardFormat,
_deadline: Deadline,
) -> Result<Option<ClipboardContent>, AdapterError> {
Err(AdapterError::not_supported("get_clipboard_content"))
}
fn set_clipboard_content(
&self,
_content: &ClipboardContent,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("set_clipboard_content"))
}
}
#[cfg(test)]
#[path = "input_tests.rs"]
mod tests;

61
crates/core/src/adapter/input_tests.rs Normal file
View file

@ -0,0 +1,61 @@
use super::*;
use crate::ErrorCode;
struct DefaultOnly;
impl InputOps for DefaultOnly {}
fn lease() -> crate::InteractionLease {
crate::InteractionLease::guarded(crate::Deadline::standard().unwrap(), ()).unwrap()
}
#[test]
fn default_clear_clipboard_is_not_supported() {
let err = DefaultOnly.clear_clipboard(&lease()).unwrap_err();
assert_eq!(err.code, ErrorCode::PlatformNotSupported);
}
#[test]
fn default_get_clipboard_content_is_not_supported() {
let err = DefaultOnly
.get_clipboard_content(ClipboardFormat::Text, crate::Deadline::standard().unwrap())
.unwrap_err();
assert_eq!(err.code, ErrorCode::PlatformNotSupported);
}
#[test]
fn default_set_clipboard_content_is_not_supported() {
let err = DefaultOnly
.set_clipboard_content(&ClipboardContent::Text("x".into()), &lease())
.unwrap_err();
assert_eq!(err.code, ErrorCode::PlatformNotSupported);
}
/// KTD13: `get_clipboard`/`set_clipboard` (string-only) were removed, not
/// wrapped — `get_clipboard_content`/`set_clipboard_content` are the only
/// surface. A regression that re-adds either legacy method would compile
/// (they'd just be unused trait members) so this guard reads the source
/// text directly rather than relying on the type system to catch it.
#[test]
fn legacy_string_clipboard_methods_are_gone() {
let src = include_str!("input.rs");
assert!(
!src.contains("fn get_clipboard("),
"get_clipboard (string-only) must stay removed; use get_clipboard_content"
);
assert!(
!src.contains("fn set_clipboard("),
"set_clipboard (string-only) must stay removed; use set_clipboard_content"
);
assert!(
src.contains("fn clear_clipboard("),
"clear_clipboard must remain (KTD13 keeps it unchanged)"
);
assert!(
src.contains("fn get_clipboard_content("),
"get_clipboard_content must remain the read surface"
);
assert!(
src.contains("fn set_clipboard_content("),
"set_clipboard_content must remain the write surface"
);
}

28
crates/core/src/adapter/mod.rs Normal file
View file

@ -0,0 +1,28 @@
pub(crate) mod actions;
pub(crate) mod input;
pub(crate) mod observation;
pub(crate) mod system;
#[cfg(test)]
mod test_support;
pub(crate) use actions::ActionOps;
pub(crate) use input::InputOps;
pub(crate) use observation::ObservationOps;
pub(crate) use observation::optional_live_read;
pub(crate) use system::SystemOps;
#[cfg(test)]
pub(crate) use test_support::{
complete_live_observation, exact_window_focus, guarded_interaction_lease, live_identity,
observed_tree,
};
pub(crate) use crate::live_element::LiveElement;
pub(crate) use crate::native_handle::NativeHandle;
pub(crate) use crate::screenshot_target::ScreenshotTarget;
pub(crate) use crate::snapshot_surface::SnapshotSurface;
pub(crate) use crate::tree_options::TreeOptions;
pub(crate) use crate::window_filter::WindowFilter;
pub trait PlatformAdapter: ObservationOps + ActionOps + InputOps + SystemOps {}
impl<T: ObservationOps + ActionOps + InputOps + SystemOps> PlatformAdapter for T {}

165
crates/core/src/adapter/observation.rs Normal file
View file

@ -0,0 +1,165 @@
use crate::{
AccessibilityNode, AdapterError, AppInfo, Deadline, ErrorCode, Rect, SurfaceInfo, WindowInfo,
element_state::ElementState,
live_element::LiveElement,
live_locator::{ObservationRequest, ObservationRoot, ObservedTree},
native_handle::NativeHandle,
refs::RefEntry,
tree_options::TreeOptions,
window_filter::WindowFilter,
};
pub(crate) fn optional_live_read<T>(
result: Result<Option<T>, AdapterError>,
) -> Result<Option<T>, AdapterError> {
match result {
Ok(value) => Ok(value),
Err(err) if is_live_read_unsupported(&err) => Ok(None),
Err(err) => Err(err),
}
}
fn is_live_read_unsupported(err: &AdapterError) -> bool {
matches!(
err.code,
ErrorCode::PlatformNotSupported | ErrorCode::ActionNotSupported
)
}
pub trait ObservationOps: Send + Sync {
fn observe_tree(
&self,
_root: ObservationRoot<'_>,
_request: &ObservationRequest,
) -> Result<ObservedTree, AdapterError> {
Err(AdapterError::not_supported("observe_tree"))
}
fn list_windows(
&self,
_filter: &WindowFilter,
_deadline: Deadline,
) -> Result<Vec<WindowInfo>, AdapterError> {
Err(AdapterError::not_supported("list_windows"))
}
fn list_apps(&self, _deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> {
Err(AdapterError::not_supported("list_apps"))
}
fn list_apps_scoped(
&self,
name: &str,
bundle_id: Option<&str>,
deadline: Deadline,
) -> Result<Vec<AppInfo>, AdapterError> {
Ok(self
.list_apps(deadline)?
.into_iter()
.filter(|app| {
app.name.eq_ignore_ascii_case(name)
&& bundle_id.is_none_or(|expected| {
app.bundle_id
.as_deref()
.is_some_and(|actual| actual.eq_ignore_ascii_case(expected))
})
})
.collect())
}
fn get_tree(
&self,
_win: &WindowInfo,
_opts: &TreeOptions,
_deadline: Deadline,
) -> Result<AccessibilityNode, AdapterError> {
Err(AdapterError::not_supported("get_tree"))
}
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: Deadline,
) -> Result<NativeHandle, AdapterError> {
Err(AdapterError::not_supported("resolve_element_strict"))
}
fn resolve_locator_anchor(
&self,
_entry: &RefEntry,
_deadline: Deadline,
) -> Result<NativeHandle, AdapterError> {
Err(AdapterError::not_supported("resolve_locator_anchor"))
}
fn get_subtree(
&self,
_handle: &NativeHandle,
_opts: &TreeOptions,
_deadline: Deadline,
) -> Result<AccessibilityNode, AdapterError> {
Err(AdapterError::not_supported("get_subtree"))
}
fn list_surfaces(
&self,
_process: crate::ProcessIdentity,
_deadline: Deadline,
) -> Result<Vec<SurfaceInfo>, AdapterError> {
Err(AdapterError::not_supported("list_surfaces"))
}
fn get_live_value(
&self,
_handle: &NativeHandle,
_deadline: Deadline,
) -> Result<Option<String>, AdapterError> {
Err(AdapterError::not_supported("get_live_value"))
}
fn get_live_state(
&self,
_handle: &NativeHandle,
_deadline: Deadline,
) -> Result<Option<ElementState>, AdapterError> {
Err(AdapterError::not_supported("get_live_state"))
}
fn get_live_actions(
&self,
_handle: &NativeHandle,
_deadline: Deadline,
) -> Result<Option<Vec<String>>, AdapterError> {
Err(AdapterError::not_supported("get_live_actions"))
}
fn get_live_element(
&self,
_handle: &NativeHandle,
_deadline: Deadline,
) -> Result<LiveElement, AdapterError> {
Err(AdapterError::not_supported("get_live_element"))
}
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: Deadline,
) -> Result<Option<Rect>, AdapterError> {
Err(AdapterError::not_supported("get_element_bounds"))
}
fn hit_test(
&self,
handle: &NativeHandle,
point: crate::Point,
_deadline: Deadline,
) -> Result<crate::hit_test::HitTestResult, AdapterError> {
let _ = (handle, point);
Err(AdapterError::not_supported("hit_test"))
}
}
#[cfg(test)]
#[path = "observation_tests.rs"]
mod tests;

88
crates/core/src/adapter/observation_tests.rs Normal file
View file

@ -0,0 +1,88 @@
use super::*;
use crate::{adapter::SnapshotSurface, refs::RefEntry};
use std::{sync::Mutex, time::Duration};
struct TimedResolver {
observed: Mutex<Option<Duration>>,
}
impl ObservationOps for TimedResolver {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
*self.observed.lock().expect("capture lock") = Some(deadline.remaining());
Ok(NativeHandle::null())
}
}
#[test]
fn untimed_convenience_delegates_to_the_deadline_aware_primitive() {
let resolver = TimedResolver {
observed: Mutex::new(None),
};
resolver
.resolve_element_strict(&entry(), crate::Deadline::standard().unwrap())
.expect("timed resolver should serve the convenience call");
assert!(
resolver
.observed
.lock()
.expect("capture lock")
.is_some_and(|duration| duration <= Duration::from_secs(5))
);
}
#[test]
fn locator_anchor_never_falls_back_to_the_public_strict_resolver() {
let resolver = TimedResolver {
observed: Mutex::new(None),
};
let error = resolver
.resolve_locator_anchor(&entry(), crate::Deadline::standard().unwrap())
.err()
.expect("unsupported exact-path resolution must fail closed");
assert_eq!(error.code, ErrorCode::PlatformNotSupported);
assert_eq!(*resolver.observed.lock().expect("capture lock"), None);
}
fn entry() -> RefEntry {
RefEntry {
process: crate::RefProcess {
pid: crate::ProcessId::new(1),
process_instance: Some("test-instance".into()),
},
identity: crate::RefEntryIdentity {
role: "button".into(),
name: None,
value: None,
description: None,
native_id: None,
},
geometry: crate::RefGeometry {
bounds: None,
bounds_hash: None,
},
capabilities: crate::RefCapabilities {
states: Vec::new(),
available_actions: Vec::new(),
},
source: crate::RefSource {
source_app: None,
source_window_id: None,
source_window_title: None,
source_window_bounds_hash: None,
source_surface: SnapshotSurface::Window,
},
scope: crate::RefScope {
root_ref: None,
path_is_absolute: false,
path: Default::default(),
},
}
}

233
crates/core/src/adapter/system.rs Normal file
View file

@ -0,0 +1,233 @@
use crate::{
AdapterError, AdapterSession, AppInfo, Deadline, DismissAllNotificationsRequest,
DismissNotificationRequest, ImageBuffer, InteractionLease, InteractionPolicy, KeyCombo,
NotificationActionRequest, NotificationFilter, NotificationInfo, PermissionReport,
PermissionState, ProcessIdentity, SessionAffinity, SignalBaseline, SignalFilter, WindowInfo,
WindowOp, action_result::ActionResult, display_info::DisplayInfo,
screenshot_target::ScreenshotTarget,
};
pub trait SystemOps: Send + Sync {
fn acquire_interaction_lease(
&self,
_deadline: Deadline,
) -> Result<InteractionLease, AdapterError> {
Err(AdapterError::not_supported("acquire_interaction_lease"))
}
fn permission_report(&self, _deadline: Deadline) -> Result<PermissionReport, AdapterError> {
Ok(PermissionReport {
accessibility: PermissionState::Denied {
suggestion: "Platform adapter not available".into(),
},
screen_recording: PermissionState::Unknown,
automation: PermissionState::NotRequired,
})
}
fn unknown_accessibility_means_unsupported(&self) -> bool {
true
}
/// Performs one renderer-accessibility activation mutation and returns.
/// Readiness polling belongs to core after the interaction lease is dropped.
fn activate_renderer_accessibility(
&self,
_process: ProcessIdentity,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported(
"activate_renderer_accessibility",
))
}
fn request_permissions(
&self,
lease: &InteractionLease,
) -> Result<PermissionReport, AdapterError> {
self.permission_report(lease.deadline())
}
/// Performs the platform-native focus/raise operation for the exact window.
/// Core decides when focus is required; implementations must return only
/// after that window is confirmed focused, or return an error.
fn focus_window(
&self,
_win: &WindowInfo,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("focus_window"))
}
fn launch_app(
&self,
_id: &str,
_options: &crate::launch_options::LaunchOptions,
_lease: &InteractionLease,
) -> Result<WindowInfo, AdapterError> {
Err(AdapterError::not_supported("launch_app"))
}
fn process_state(
&self,
_process: ProcessIdentity,
_deadline: Deadline,
) -> Result<crate::process_state::ProcessState, AdapterError> {
Err(AdapterError::not_supported("process_state"))
}
fn supported_surfaces(&self) -> Vec<crate::adapter::SnapshotSurface> {
Vec::new()
}
/// Opens adapter-native connection affinity for a persistent host.
///
/// The returned session may retain platform connection state, but never a
/// resolved element handle. Stateless command callers do not need to open a
/// session. Adapters without persistent native state return
/// `PLATFORM_NOT_SUPPORTED`.
fn open_session(
&self,
_affinity: &SessionAffinity,
_deadline: Deadline,
) -> Result<Box<dyn AdapterSession>, AdapterError> {
Err(AdapterError::not_supported("open_session"))
}
/// Captures a point-in-time [`SignalBaseline`] snapshot,
/// narrowed by `filter` when the caller already knows which app it cares
/// about. `deadline` is one absolute budget shared by every native read in
/// the capture; an adapter must return `TIMEOUT` rather than publish an
/// observation completed at or after it. `wait --event` calls this once at
/// wait-start and again on every poll, then diffs the two snapshots with
/// `crate::diff_signals` — the adapter never decides what changed,
/// only what the desktop looks like right now.
fn capture_signal_baseline(
&self,
_filter: &SignalFilter,
_deadline: Deadline,
) -> Result<SignalBaseline, AdapterError> {
Err(AdapterError::not_supported("capture_signal_baseline"))
}
fn close_app(
&self,
_app: &AppInfo,
_force: bool,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("close_app"))
}
/// Reports whether closing `identifier` would terminate a process the OS
/// depends on (window server, login session, shell). The set is
/// inherently platform-specific, so each adapter owns its own list;
/// core only asks. The default denies nothing.
fn is_protected_process(&self, _identifier: &str) -> bool {
false
}
/// Reports whether `combo` is a platform-dangerous keyboard shortcut that
/// should be refused unless the caller explicitly forces it (for example
/// macOS Cmd+Q quit, Ctrl+Cmd+Q lock, Cmd+Alt+Esc force-quit). Which
/// combos are dangerous — and how key names alias to physical keys — is
/// platform-specific, so each adapter owns its own list; core only asks
/// and lets the caller override via `--force`. The default blocks nothing,
/// leaving the decision entirely to the calling agent.
fn is_blocked_combo(&self, _combo: &KeyCombo) -> bool {
false
}
fn screenshot(
&self,
_target: ScreenshotTarget,
_deadline: Deadline,
) -> Result<ImageBuffer, AdapterError> {
Err(AdapterError::not_supported("screenshot"))
}
fn list_displays(&self, _deadline: Deadline) -> Result<Vec<DisplayInfo>, AdapterError> {
Err(AdapterError::not_supported("list_displays"))
}
fn focused_window(&self, _deadline: Deadline) -> Result<Option<WindowInfo>, AdapterError> {
Err(AdapterError::not_supported("focused_window"))
}
fn press_key_for_app(
&self,
_process: ProcessIdentity,
_combo: &crate::KeyCombo,
_policy: crate::InteractionPolicy,
_lease: &InteractionLease,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::not_supported("press_key_for_app"))
}
fn wait_for_menu(
&self,
_process: ProcessIdentity,
_open: bool,
_deadline: Deadline,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("wait_for_menu"))
}
fn window_op(
&self,
_win: &WindowInfo,
_op: WindowOp,
_lease: &InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::not_supported("window_op"))
}
/// Resolves a live window by `WindowInfo.id`, corroborating the match against
/// `pid` and, when present, `title`. Opaque ids must not be parsed as numeric
/// outside the adapter; macOS uses `w-<kCGWindowNumber>`.
fn resolve_window_strict(
&self,
_win: &WindowInfo,
_deadline: Deadline,
) -> Result<WindowInfo, AdapterError> {
Err(AdapterError::not_supported("resolve_window_strict"))
}
fn list_notifications(
&self,
_filter: &NotificationFilter,
_policy: InteractionPolicy,
_deadline: Deadline,
_lease: Option<&InteractionLease>,
) -> Result<Vec<NotificationInfo>, AdapterError> {
Err(AdapterError::not_supported("list_notifications"))
}
fn dismiss_notification(
&self,
_request: DismissNotificationRequest<'_>,
_lease: &InteractionLease,
) -> Result<NotificationInfo, AdapterError> {
Err(AdapterError::not_supported("dismiss_notification"))
}
fn dismiss_all_notifications(
&self,
_request: DismissAllNotificationsRequest<'_>,
_lease: &InteractionLease,
) -> Result<(Vec<NotificationInfo>, Vec<String>), AdapterError> {
Err(AdapterError::not_supported("dismiss_all_notifications"))
}
fn notification_action(
&self,
_request: NotificationActionRequest<'_>,
_lease: &InteractionLease,
) -> Result<ActionResult, AdapterError> {
Err(AdapterError::not_supported("notification_action"))
}
}
#[cfg(test)]
#[path = "system_tests.rs"]
mod tests;

33
crates/core/src/adapter/system_tests.rs Normal file
View file

@ -0,0 +1,33 @@
use super::*;
struct DefaultOnly;
impl SystemOps for DefaultOnly {}
#[test]
fn default_interaction_lease_is_unsupported_in_test_builds_too() {
let error = match DefaultOnly.acquire_interaction_lease(crate::Deadline::standard().unwrap()) {
Ok(_) => panic!("default lease must fail closed"),
Err(error) => error,
};
assert_eq!(error.code, crate::ErrorCode::PlatformNotSupported);
}
#[test]
fn default_surfaces_fail_closed() {
assert!(DefaultOnly.supported_surfaces().is_empty());
}
#[test]
fn default_open_session_is_explicitly_unsupported() {
let affinity = crate::SessionAffinity {
session_id: Some("test-session".into()),
};
let error = match DefaultOnly.open_session(&affinity, crate::Deadline::standard().unwrap()) {
Ok(_) => panic!("default session affinity must fail closed"),
Err(error) => error,
};
assert_eq!(error.code, crate::ErrorCode::PlatformNotSupported);
assert!(error.message.contains("open_session"));
}

168
crates/core/src/adapter/test_support.rs Normal file
View file

@ -0,0 +1,168 @@
macro_rules! complete_live_observation {
($role:expr, $name:expr, [$($action:expr),* $(,)?]) => {
fn get_live_element(
&self,
_handle: &$crate::adapter::NativeHandle,
_deadline: $crate::Deadline,
) -> Result<$crate::LiveElement, $crate::AdapterError> {
Ok($crate::LiveElement {
identity: $crate::LiveIdentity {
name: $crate::LocatorField::Known($name.into()),
description: $crate::LocatorField::Absent,
identifiers: $crate::IdentifierEvidence::absent(),
},
state: $crate::ElementState {
role: $role.into(),
states: Vec::new(),
value: None,
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
},
states_complete: true,
bounds: Some($crate::Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}),
available_actions: vec![$($action.into()),*],
})
}
fn get_live_state(
&self,
_handle: &$crate::adapter::NativeHandle,
_deadline: $crate::Deadline,
) -> Result<Option<$crate::ElementState>, $crate::AdapterError> {
Ok(Some($crate::ElementState {
role: $role.into(),
states: Vec::new(),
value: None,
enabled: Some(true),
hidden: Some(false),
offscreen: Some(false),
}))
}
fn get_element_bounds(
&self,
_handle: &$crate::adapter::NativeHandle,
_deadline: $crate::Deadline,
) -> Result<Option<$crate::Rect>, $crate::AdapterError> {
Ok(Some($crate::Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
}))
}
fn get_live_actions(
&self,
_handle: &$crate::adapter::NativeHandle,
_deadline: $crate::Deadline,
) -> Result<Option<Vec<String>>, $crate::AdapterError> {
Ok(Some(vec![$($action.into()),*]))
}
fn hit_test(
&self,
_handle: &$crate::adapter::NativeHandle,
_point: $crate::Point,
_deadline: $crate::Deadline,
) -> Result<$crate::hit_test::HitTestResult, $crate::AdapterError> {
Ok($crate::hit_test::HitTestResult::ReachesTarget)
}
};
}
pub(crate) use complete_live_observation;
macro_rules! guarded_interaction_lease {
() => {
fn acquire_interaction_lease(
&self,
deadline: $crate::Deadline,
) -> Result<$crate::InteractionLease, $crate::AdapterError> {
$crate::InteractionLease::guarded(deadline, ())
}
};
}
pub(crate) use guarded_interaction_lease;
macro_rules! exact_window_focus {
() => {
fn resolve_window_strict(
&self,
window: &$crate::WindowInfo,
_deadline: $crate::Deadline,
) -> Result<$crate::WindowInfo, $crate::AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
_window: &$crate::WindowInfo,
_lease: &$crate::InteractionLease,
) -> Result<(), $crate::AdapterError> {
Ok(())
}
};
}
pub(crate) use exact_window_focus;
pub(crate) fn live_identity(name: &str) -> crate::LiveIdentity {
crate::LiveIdentity {
name: crate::LocatorField::Known(name.into()),
description: crate::LocatorField::Absent,
identifiers: crate::IdentifierEvidence::absent(),
}
}
pub(crate) fn observed_tree(
root: &crate::live_locator::ObservationRoot<'_>,
node: crate::AccessibilityNode,
) -> Result<crate::live_locator::ObservedTree, crate::AdapterError> {
use crate::live_locator::{
IdentifierEvidence, LocatorEvidence, LocatorField, LocatorRefEvidence, LocatorStats,
ObservationSource, ObservedSubtree, ObservedTree,
};
fn field(value: Option<String>) -> LocatorField<String> {
value
.map(LocatorField::Known)
.unwrap_or(LocatorField::Absent)
}
fn subtree(node: crate::AccessibilityNode) -> ObservedSubtree {
let native_id = node.identity.native_id.clone();
let evidence = LocatorEvidence {
role: LocatorField::Known(node.role),
name: field(node.identity.name),
description: field(node.identity.description),
value: field(node.identity.value),
identifiers: IdentifierEvidence::typed(native_id, Some(0), true),
states: LocatorField::Known(node.presentation.states),
ref_evidence: LocatorRefEvidence {
bounds: node
.presentation
.bounds
.map(LocatorField::Known)
.unwrap_or(LocatorField::Absent),
available_actions: LocatorField::Known(node.presentation.available_actions),
},
};
let children = node.children.into_iter().map(subtree).collect();
ObservedSubtree::new(evidence, children, true, node.children_count)
}
ObservedTree::from_roots(
vec![subtree(node)],
ObservationSource::from_root(root),
LocatorStats::default(),
true,
)
}

270
crates/core/src/adapter_error.rs Normal file
View file

@ -0,0 +1,270 @@
use serde_json::Value;
use thiserror::Error;
use crate::{DeliverySemantics, ErrorCode, InteractionPolicy};
#[derive(Debug, Error, Clone)]
#[error("{message}")]
pub struct AdapterError {
pub code: ErrorCode,
pub message: String,
pub suggestion: Option<String>,
pub platform_detail: Option<String>,
pub details: Option<Value>,
pub disposition: DeliverySemantics,
retryability: crate::retryability::Retryability,
}
impl AdapterError {
pub fn new(code: ErrorCode, message: impl Into<String>) -> Self {
Self {
code,
message: message.into(),
suggestion: None,
platform_detail: None,
details: None,
disposition: DeliverySemantics::unknown(),
retryability: crate::retryability::Retryability::Unspecified,
}
}
pub fn with_suggestion(mut self, suggestion: impl Into<String>) -> Self {
self.suggestion = Some(suggestion.into());
self
}
pub fn with_platform_detail(mut self, detail: impl Into<String>) -> Self {
self.platform_detail = Some(detail.into());
self
}
pub fn with_details(mut self, details: Value) -> Self {
let retryability = crate::retryability::Retryability::from_details(&details);
if retryability != crate::retryability::Retryability::Unspecified {
self.retryability = retryability;
}
self.details = Some(details);
self
}
pub fn with_disposition(mut self, disposition: DeliverySemantics) -> Self {
self.disposition = disposition;
self
}
pub fn is_explicitly_retryable(&self) -> bool {
self.retryability == crate::retryability::Retryability::Retryable
}
pub(crate) fn is_retryable_resolution_failure(&self) -> bool {
matches!(
self.code,
ErrorCode::StaleRef
| ErrorCode::AmbiguousTarget
| ErrorCode::Timeout
| ErrorCode::AppUnresponsive
) && self.is_explicitly_retryable()
}
pub fn permits_retry_by_default(&self) -> bool {
self.retryability != crate::retryability::Retryability::NonRetryable
}
pub fn stale_ref(ref_id: &str) -> Self {
Self::new(
ErrorCode::StaleRef,
format!("{ref_id} not found in current RefMap"),
)
.with_suggestion(
"Re-run a snapshot to obtain fresh refs, then retry with the new ref \
(CLI: snapshot [--skeleton]; FFI: ad_snapshot then ad_execute_by_ref with the returned snapshot_id)",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn ambiguous_target(message: impl Into<String>) -> Self {
Self::new(ErrorCode::AmbiguousTarget, message)
.with_suggestion(
"Re-run a snapshot to refresh refs, then retry with a more specific ref",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn not_supported(method: &str) -> Self {
Self::new(
ErrorCode::PlatformNotSupported,
format!("{method} is not supported on this platform"),
)
.with_suggestion(
"Use a platform/build that advertises this capability or choose a supported command",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn element_not_found(ref_id: &str) -> Self {
Self::new(
ErrorCode::ElementNotFound,
format!("Element {ref_id} could not be resolved"),
)
.with_suggestion("Re-run a snapshot to obtain fresh refs, then retry with the new ref")
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn timeout(message: impl Into<String>) -> Self {
Self::new(ErrorCode::Timeout, message)
.with_suggestion("The target application may be busy or unresponsive")
}
pub fn notification_not_found(index: usize) -> Self {
Self::new(
ErrorCode::NotificationNotFound,
format!("Notification at index {index} not found"),
)
.with_suggestion(
"Notification may have been dismissed or expired. \
Re-run a notification list to see current notifications \
(CLI: list-notifications; FFI: ad_list_notifications)",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn app_unresponsive(app: &str) -> Self {
Self::new(
ErrorCode::AppUnresponsive,
format!("Application '{app}' is not responding"),
)
.with_suggestion("Wait for the app to recover or force-quit it before retrying automation")
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn internal(message: impl Into<String>) -> Self {
Self::new(ErrorCode::Internal, message)
}
pub fn permission_denied() -> Self {
Self::new(
ErrorCode::PermDenied,
"Accessibility permission not granted",
)
.with_suggestion(
"Open System Settings > Privacy & Security > Accessibility and add the app that launches agent-desktop",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn snapshot_not_found(snapshot_id: &str) -> Self {
Self::new(
ErrorCode::SnapshotNotFound,
format!("Snapshot '{snapshot_id}' not found"),
)
.with_suggestion(
"Re-run a snapshot and retry with the returned snapshot_id \
(CLI: snapshot, then pass --snapshot <id>; FFI: ad_snapshot then supply snapshot_id to ad_execute_by_ref)",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn policy_denied(message: impl Into<String>) -> Self {
Self::new(ErrorCode::PolicyDenied, message)
.with_suggestion(
"Use an explicit mouse/focus command if physical interaction is intended",
)
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn policy_denied_for_policy(message: impl Into<String>, policy: InteractionPolicy) -> Self {
Self::new(ErrorCode::PolicyDenied, message)
.with_suggestion(policy_denied_suggestion(policy))
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn renderer_accessibility_activation_required(message: impl Into<String>) -> Self {
Self::new(ErrorCode::ActionNotSupported, message)
.with_details(serde_json::json!({
"renderer_accessibility_activation_required": true,
}))
.with_disposition(DeliverySemantics::not_delivered())
}
pub fn requires_renderer_accessibility_activation(&self) -> bool {
self.details
.as_ref()
.and_then(|details| details.get("renderer_accessibility_activation_required"))
.and_then(Value::as_bool)
== Some(true)
}
}
fn policy_denied_suggestion(policy: InteractionPolicy) -> &'static str {
if policy.allow_focus_steal && !policy.allow_cursor_move {
"Enable cursor movement in the interaction policy to permit cursor-driven actions \
(CLI: --headed; FFI: set allow_cursor_move in the policy), \
or use an explicit mouse command if physical input is intended"
} else if !policy.allow_focus_steal && !policy.allow_cursor_move {
"Headless mode allows only accessibility-backed actions; \
enable physical interaction in the policy (CLI: --headed) only if cursor/focus movement is intended, \
otherwise refresh the snapshot or target an element with the needed semantic action"
} else {
"Use an explicit mouse/focus command if physical interaction is intended"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retryability_is_typed_once_and_survives_diagnostic_enrichment() {
let error = AdapterError::new(ErrorCode::StaleRef, "stale")
.with_details(serde_json::json!({ "retryable": true }))
.with_details(serde_json::json!({ "phase": "resolve" }));
assert!(error.is_explicitly_retryable());
assert!(error.permits_retry_by_default());
}
#[test]
fn explicit_non_retryable_evidence_overrides_code_defaults() {
let error = AdapterError::new(ErrorCode::StaleRef, "stale")
.with_details(serde_json::json!({ "retryable": false }));
assert!(!error.is_explicitly_retryable());
assert!(!error.permits_retry_by_default());
}
#[test]
fn retryable_resolution_failure_characterizes_every_error_code() {
let all = [
ErrorCode::PermDenied,
ErrorCode::ElementNotFound,
ErrorCode::AppNotFound,
ErrorCode::ActionFailed,
ErrorCode::ActionNotSupported,
ErrorCode::StaleRef,
ErrorCode::AmbiguousTarget,
ErrorCode::WindowNotFound,
ErrorCode::PlatformNotSupported,
ErrorCode::Timeout,
ErrorCode::InvalidArgs,
ErrorCode::NotificationNotFound,
ErrorCode::SnapshotNotFound,
ErrorCode::PolicyDenied,
ErrorCode::AppUnresponsive,
ErrorCode::Internal,
];
let retryable = [
ErrorCode::StaleRef,
ErrorCode::AmbiguousTarget,
ErrorCode::Timeout,
ErrorCode::AppUnresponsive,
];
for code in all {
let expected = retryable.contains(&code);
let error = AdapterError::new(code.clone(), "failure")
.with_details(serde_json::json!({ "retryable": true }));
assert_eq!(error.is_retryable_resolution_failure(), expected);
assert!(!AdapterError::new(code, "failure").is_retryable_resolution_failure());
}
}
}

15
crates/core/src/adapter_session.rs Normal file
View file

@ -0,0 +1,15 @@
use crate::AdapterError;
/// A live adapter-native connection owned by a persistent host.
///
/// The session may retain platform connection state, such as a Windows COM
/// apartment or Linux D-Bus connection, but must not retain resolved element
/// handles. Elements remain command-scoped so stale identity cannot escape the
/// resolve-and-release boundary.
pub trait AdapterSession: Send + Sync {
fn close(self: Box<Self>) -> Result<(), AdapterError>;
}
#[cfg(test)]
#[path = "adapter_session_tests.rs"]
mod tests;

36
crates/core/src/adapter_session_tests.rs Normal file
View file

@ -0,0 +1,36 @@
use super::AdapterSession;
use crate::AdapterError;
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
struct FlagSession {
closed: Arc<AtomicBool>,
}
impl AdapterSession for FlagSession {
fn close(self: Box<Self>) -> Result<(), AdapterError> {
self.closed.store(true, Ordering::SeqCst);
Ok(())
}
}
#[test]
fn boxed_adapter_session_close_runs_through_dynamic_dispatch() {
let closed = Arc::new(AtomicBool::new(false));
let session: Box<dyn AdapterSession> = Box::new(FlagSession {
closed: Arc::clone(&closed),
});
session.close().unwrap();
assert!(closed.load(Ordering::SeqCst));
}
fn assert_send_sync<T: Send + Sync>() {}
#[test]
fn boxed_adapter_session_is_send_and_sync() {
assert_send_sync::<Box<dyn AdapterSession>>();
}

57
crates/core/src/app_error.rs Normal file
View file

@ -0,0 +1,57 @@
use thiserror::Error;
use crate::{AdapterError, ErrorCode};
#[derive(Debug, Error)]
pub enum AppError {
#[error(transparent)]
Adapter(#[from] AdapterError),
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("{0}")]
Internal(String),
}
impl AppError {
pub fn code(&self) -> &str {
match self {
Self::Adapter(error) => error.code.as_str(),
Self::Io(_) | Self::Json(_) | Self::Internal(_) => "INTERNAL",
}
}
pub fn suggestion(&self) -> Option<&str> {
match self {
Self::Adapter(error) => error.suggestion.as_deref(),
_ => None,
}
}
pub fn stale_ref(ref_id: &str) -> Self {
Self::Adapter(AdapterError::stale_ref(ref_id))
}
pub fn invalid_input(message: impl Into<String>) -> Self {
Self::Adapter(
AdapterError::new(ErrorCode::InvalidArgs, message)
.with_disposition(crate::DeliverySemantics::not_delivered()),
)
}
pub fn invalid_input_with_suggestion(
message: impl Into<String>,
suggestion: impl Into<String>,
) -> Self {
Self::Adapter(
AdapterError::new(ErrorCode::InvalidArgs, message)
.with_suggestion(suggestion)
.with_disposition(crate::DeliverySemantics::not_delivered()),
)
}
}
#[cfg(test)]
#[path = "error_tests.rs"]
mod tests;

13
crates/core/src/app_info.rs Normal file
View file

@ -0,0 +1,13 @@
use serde::{Deserialize, Serialize};
use crate::ProcessId;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AppInfo {
pub name: String,
pub pid: ProcessId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bundle_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub process_instance: Option<String>,
}

178
crates/core/src/app_lookup.rs Normal file
View file

@ -0,0 +1,178 @@
use serde_json::json;
use crate::{
AdapterError, AppError, AppInfo, Deadline, ErrorCode, ProcessIdentity,
adapter::{PlatformAdapter, WindowFilter},
};
pub(crate) fn resolve_app(
app: Option<&str>,
adapter: &dyn PlatformAdapter,
deadline: Deadline,
) -> Result<AppInfo, AppError> {
if let Some(name) = app {
let candidates = adapter.list_apps_scoped(name, None, deadline)?;
return select_unique_app(candidates, name);
}
let focused = adapter.list_windows(
&WindowFilter {
focused_only: true,
app: None,
},
deadline,
)?;
let window = match focused.as_slice() {
[] => {
return Err(AdapterError::new(
ErrorCode::AppNotFound,
"No focused application was found when --app was omitted",
)
.into());
}
[window] => window,
_ => {
return Err(AdapterError::ambiguous_target(
"A unique focused window is required when --app is omitted",
)
.with_details(json!({ "focused_window_count": focused.len() }))
.into());
}
};
let instance = window
.process_instance
.as_deref()
.filter(|instance| !instance.is_empty())
.ok_or_else(|| {
AdapterError::new(
ErrorCode::ActionNotSupported,
"Focused window has no process-instance identity",
)
})?;
let same_pid = adapter
.list_apps_scoped(&window.app, None, deadline)?
.into_iter()
.filter(|candidate| candidate.pid == window.pid)
.collect::<Vec<_>>();
reject_incomplete_app_identity(&same_pid, &window.app)?;
let candidates = same_pid
.into_iter()
.filter(|candidate| candidate.process_instance.as_deref() == Some(instance))
.collect();
select_unique_app(candidates, &window.app)
}
fn select_unique_app(mut candidates: Vec<AppInfo>, label: &str) -> Result<AppInfo, AppError> {
reject_incomplete_app_identity(&candidates, label)?;
match candidates.len() {
0 => Err(AdapterError::new(
ErrorCode::AppNotFound,
format!("Application '{label}' was not found with exact process identity"),
)
.into()),
1 => Ok(candidates.swap_remove(0)),
_ => {
let summaries = candidates
.iter()
.take(10)
.map(|candidate| {
json!({
"name": candidate.name,
"pid": candidate.pid,
"process_instance": candidate.process_instance,
})
})
.collect::<Vec<_>>();
Err(AdapterError::ambiguous_target(format!(
"Multiple application instances matched '{label}'"
))
.with_details(json!({
"candidate_count": candidates.len(),
"candidate_summaries_truncated": candidates.len() > summaries.len(),
"candidates": summaries,
}))
.into())
}
}
}
fn reject_incomplete_app_identity(candidates: &[AppInfo], label: &str) -> Result<(), AppError> {
let incomplete_count = candidates
.iter()
.filter(|candidate| {
candidate
.process_instance
.as_deref()
.is_none_or(str::is_empty)
})
.count();
if incomplete_count == 0 {
return Ok(());
}
Err(AdapterError::new(
ErrorCode::ActionNotSupported,
format!("Application inventory for '{label}' has incomplete process identity"),
)
.with_details(json!({
"candidate_count": candidates.len(),
"incomplete_identity_count": incomplete_count,
}))
.into())
}
pub(crate) fn process_identity(app: &AppInfo) -> Result<ProcessIdentity, AppError> {
let instance = app
.process_instance
.as_deref()
.filter(|instance| !instance.is_empty())
.ok_or_else(|| {
AdapterError::new(
ErrorCode::ActionNotSupported,
"Application has no process-instance identity",
)
})?;
Ok(ProcessIdentity::new(app.pid, instance))
}
pub(crate) fn revalidate_app_for_mutation(
adapter: &dyn PlatformAdapter,
expected: &AppInfo,
deadline: Deadline,
) -> Result<AppInfo, AppError> {
let expected_identity = process_identity(expected)?;
let same_pid = adapter
.list_apps_scoped(&expected.name, expected.bundle_id.as_deref(), deadline)?
.into_iter()
.filter(|candidate| candidate.pid == expected.pid)
.collect::<Vec<_>>();
reject_incomplete_app_identity(&same_pid, &expected.name)?;
let mut exact = same_pid
.into_iter()
.filter(|candidate| {
candidate.process_instance.as_deref() == Some(expected_identity.instance.as_str())
&& candidate.name.eq_ignore_ascii_case(&expected.name)
&& expected.bundle_id.as_deref().is_none_or(|expected_bundle| {
candidate
.bundle_id
.as_deref()
.is_some_and(|actual| actual.eq_ignore_ascii_case(expected_bundle))
})
})
.collect::<Vec<_>>();
match exact.len() {
1 => Ok(exact.swap_remove(0)),
0 => Err(AdapterError::new(
ErrorCode::StaleRef,
"Application identity changed before mutation",
)
.into()),
_ => Err(AdapterError::ambiguous_target(
"Multiple exact application identities remained before mutation",
)
.with_details(json!({ "candidate_count": exact.len() }))
.into()),
}
}
#[cfg(test)]
#[path = "app_lookup_tests.rs"]
mod tests;

250
crates/core/src/app_lookup_tests.rs Normal file
View file

@ -0,0 +1,250 @@
use super::*;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use std::sync::{
Mutex,
atomic::{AtomicUsize, Ordering},
};
struct InventoryAdapter {
apps: Vec<AppInfo>,
windows: Vec<crate::WindowInfo>,
}
impl ObservationOps for InventoryAdapter {
fn list_apps(&self, _deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> {
Ok(self.apps.clone())
}
fn list_windows(
&self,
_filter: &WindowFilter,
_deadline: Deadline,
) -> Result<Vec<crate::WindowInfo>, AdapterError> {
Ok(self.windows.clone())
}
}
impl ActionOps for InventoryAdapter {}
impl InputOps for InventoryAdapter {}
impl SystemOps for InventoryAdapter {}
fn app(instance: Option<&str>) -> AppInfo {
AppInfo {
name: "Example".into(),
pid: crate::ProcessId::new(42),
bundle_id: Some("com.example.app".into()),
process_instance: instance.map(str::to_string),
}
}
fn app_with_pid(pid: u32, instance: Option<&str>) -> AppInfo {
AppInfo {
pid: crate::ProcessId::new(pid),
..app(instance)
}
}
struct ScopedInventorySpy {
apps: Vec<AppInfo>,
global_calls: AtomicUsize,
scopes: Mutex<Vec<(String, Option<String>)>>,
}
impl ScopedInventorySpy {
fn new(apps: Vec<AppInfo>) -> Self {
Self {
apps,
global_calls: AtomicUsize::new(0),
scopes: Mutex::new(Vec::new()),
}
}
}
impl ObservationOps for ScopedInventorySpy {
fn list_apps(&self, _deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> {
self.global_calls.fetch_add(1, Ordering::Relaxed);
Err(AdapterError::new(
ErrorCode::AppUnresponsive,
"unrelated global inventory failure",
))
}
fn list_apps_scoped(
&self,
name: &str,
bundle_id: Option<&str>,
_deadline: Deadline,
) -> Result<Vec<AppInfo>, AdapterError> {
self.scopes
.lock()
.unwrap()
.push((name.to_string(), bundle_id.map(str::to_string)));
Ok(self.apps.clone())
}
}
impl ActionOps for ScopedInventorySpy {}
impl InputOps for ScopedInventorySpy {}
impl SystemOps for ScopedInventorySpy {}
struct ScopedPermissionAdapter;
impl ObservationOps for ScopedPermissionAdapter {
fn list_apps(&self, _deadline: Deadline) -> Result<Vec<AppInfo>, AdapterError> {
Err(AdapterError::new(
ErrorCode::AppUnresponsive,
"unrelated global inventory failure",
))
}
fn list_apps_scoped(
&self,
_name: &str,
_bundle_id: Option<&str>,
_deadline: Deadline,
) -> Result<Vec<AppInfo>, AdapterError> {
Err(AdapterError::new(
ErrorCode::PermDenied,
"permission denied reading requested process identity",
))
}
}
impl ActionOps for ScopedPermissionAdapter {}
impl InputOps for ScopedPermissionAdapter {}
impl SystemOps for ScopedPermissionAdapter {}
#[test]
fn named_resolution_uses_scoped_inventory_when_global_inventory_fails() {
let adapter = ScopedInventorySpy::new(vec![app(Some("generation-a"))]);
let resolved = resolve_app(Some("example"), &adapter, Deadline::standard().unwrap()).unwrap();
assert_eq!(resolved.pid, 42);
assert_eq!(adapter.global_calls.load(Ordering::Relaxed), 0);
assert_eq!(
*adapter.scopes.lock().unwrap(),
vec![("example".to_string(), None)]
);
}
#[test]
fn mutation_revalidation_uses_name_and_bundle_scope() {
let expected = app(Some("generation-a"));
let mut differently_cased = expected.clone();
differently_cased.name = "example".into();
differently_cased.bundle_id = Some("COM.EXAMPLE.APP".into());
let adapter = ScopedInventorySpy::new(vec![differently_cased]);
let resolved =
revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap()).unwrap();
assert_eq!(resolved.pid, 42);
assert_eq!(adapter.global_calls.load(Ordering::Relaxed), 0);
assert_eq!(
*adapter.scopes.lock().unwrap(),
vec![("Example".to_string(), Some("com.example.app".to_string()))]
);
}
#[test]
fn scoped_target_permission_failure_is_preserved() {
let error = resolve_app(
Some("Example"),
&ScopedPermissionAdapter,
Deadline::standard().unwrap(),
)
.unwrap_err();
assert_eq!(error.code(), "PERM_DENIED");
}
#[test]
fn default_scoped_inventory_filters_exact_name_and_bundle_case_insensitively() {
let mut wrong_name = app_with_pid(43, Some("generation-b"));
wrong_name.name = "Example Helper".into();
let mut wrong_bundle = app_with_pid(44, Some("generation-c"));
wrong_bundle.bundle_id = Some("com.example.other".into());
let adapter = InventoryAdapter {
apps: vec![app(Some("generation-a")), wrong_name, wrong_bundle],
windows: Vec::new(),
};
let scoped = adapter
.list_apps_scoped(
"example",
Some("COM.EXAMPLE.APP"),
Deadline::standard().unwrap(),
)
.unwrap();
assert_eq!(scoped.len(), 1);
assert_eq!(scoped[0].pid, 42);
}
#[test]
fn named_resolution_rejects_any_matching_candidate_without_generation() {
let adapter = InventoryAdapter {
apps: vec![app(Some("generation-a")), app(None)],
windows: Vec::new(),
};
let error = resolve_app(Some("Example"), &adapter, Deadline::standard().unwrap()).unwrap_err();
assert_eq!(error.code(), "ACTION_NOT_SUPPORTED");
}
#[test]
fn named_resolution_keeps_same_name_multiple_pids_ambiguous() {
let adapter = InventoryAdapter {
apps: vec![
app_with_pid(42, Some("generation-a")),
app_with_pid(43, Some("generation-b")),
],
windows: Vec::new(),
};
let error = resolve_app(Some("Example"), &adapter, Deadline::standard().unwrap()).unwrap_err();
assert_eq!(error.code(), "AMBIGUOUS_TARGET");
}
#[test]
fn omitted_app_with_no_focused_window_is_not_found() {
let adapter = InventoryAdapter {
apps: vec![app(Some("generation-a"))],
windows: Vec::new(),
};
let error = resolve_app(None, &adapter, Deadline::standard().unwrap()).unwrap_err();
assert_eq!(error.code(), "APP_NOT_FOUND");
}
#[test]
fn mutation_revalidation_rejects_pid_reuse() {
let expected = app(Some("generation-a"));
let adapter = InventoryAdapter {
apps: vec![app(Some("generation-b"))],
windows: Vec::new(),
};
let error = revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap())
.unwrap_err();
assert_eq!(error.code(), "STALE_REF");
}
#[test]
fn mutation_revalidation_rejects_missing_generation() {
let expected = app(Some("generation-a"));
let adapter = InventoryAdapter {
apps: vec![app(None)],
windows: Vec::new(),
};
let error = revalidate_app_for_mutation(&adapter, &expected, Deadline::standard().unwrap())
.unwrap_err();
assert_eq!(error.code(), "ACTION_NOT_SUPPORTED");
}

72
crates/core/src/capability.rs
View file

@ -33,34 +33,17 @@ pub fn for_action(action: &Action) -> &'static [&'static str] {
Action::Select(_) => &[SELECT, CLICK],
Action::Toggle => &[TOGGLE, CLICK],
Action::Check | Action::Uncheck => &[TOGGLE, CLICK],
Action::Scroll(_, _) => &[SCROLL, SCROLL_TO],
Action::Scroll(_, _) => &[SCROLL],
Action::ScrollTo => &[SCROLL_TO],
Action::PressKey(_) => &[PRESS_KEY],
Action::KeyDown(_) => &[KEY_DOWN],
Action::KeyUp(_) => &[KEY_UP],
Action::TypeText(_) => &[TYPE_TEXT, SET_VALUE],
Action::TypeText(_) => &[TYPE_TEXT],
Action::Hover => &[HOVER],
Action::Drag(_) => &[DRAG],
}
}
pub fn defaults_for_role(role: &str) -> Vec<String> {
let capabilities: &[&str] = match role {
"button" | "link" | "menuitem" | "tab" | "radiobutton" => &[CLICK],
"textfield" | "incrementor" => &[CLICK, SET_VALUE, SET_FOCUS],
"checkbox" => &[CLICK, TOGGLE],
"combobox" => &[CLICK, SELECT],
"treeitem" => &[CLICK, EXPAND, COLLAPSE],
"slider" => &[SET_VALUE],
"cell" => &[CLICK],
_ => &[CLICK],
};
capabilities
.iter()
.map(|capability| (*capability).to_string())
.collect()
}
pub fn contains(actions: &[String], capability: &str) -> bool {
actions.iter().any(|action| action == capability)
}
@ -71,10 +54,22 @@ pub fn contains_any(actions: &[String], capabilities: &[&str]) -> bool {
.any(|capability| contains(actions, capability))
}
pub(crate) fn supports_direct_semantic_pointer_delivery(
action: &Action,
available_actions: &[String],
) -> bool {
let capability = match action {
Action::Click => CLICK,
Action::RightClick => RIGHT_CLICK,
_ => return false,
};
contains(available_actions, capability)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::action::{Direction, KeyCombo};
use crate::{Direction, KeyCombo};
#[test]
fn action_capabilities_are_declared_in_one_place() {
@ -82,10 +77,7 @@ mod tests {
assert_eq!(for_action(&Action::RightClick), &[RIGHT_CLICK]);
assert_eq!(for_action(&Action::SetValue("x".into())), &[SET_VALUE]);
assert_eq!(for_action(&Action::Clear), &[SET_VALUE]);
assert_eq!(
for_action(&Action::Scroll(Direction::Down, 1)),
&[SCROLL, SCROLL_TO]
);
assert_eq!(for_action(&Action::Scroll(Direction::Down, 1)), &[SCROLL]);
assert_eq!(
for_action(&Action::PressKey(KeyCombo {
key: "A".into(),
@ -96,19 +88,25 @@ mod tests {
}
#[test]
fn role_defaults_are_declared_in_one_place() {
assert_eq!(defaults_for_role("button"), strings(&[CLICK]));
assert_eq!(
defaults_for_role("textfield"),
strings(&[CLICK, SET_VALUE, SET_FOCUS])
);
assert_eq!(
defaults_for_role("treeitem"),
strings(&[CLICK, EXPAND, COLLAPSE])
);
}
fn direct_semantic_pointer_delivery_requires_an_exact_capability() {
let click = vec![CLICK.to_string()];
let right_click = vec![RIGHT_CLICK.to_string()];
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| (*value).to_string()).collect()
assert!(supports_direct_semantic_pointer_delivery(
&Action::Click,
&click
));
assert!(supports_direct_semantic_pointer_delivery(
&Action::RightClick,
&right_click
));
assert!(!supports_direct_semantic_pointer_delivery(
&Action::DoubleClick,
&click
));
assert!(!supports_direct_semantic_pointer_delivery(
&Action::TripleClick,
&click
));
}
}

22
crates/core/src/clipboard_content.rs Normal file
View file

@ -0,0 +1,22 @@
use crate::{ClipboardFormat, ImageBuffer};
#[derive(Debug)]
pub enum ClipboardContent {
Text(String),
Image(ImageBuffer),
FileUrls(Vec<String>),
}
impl ClipboardContent {
pub fn format(&self) -> ClipboardFormat {
match self {
ClipboardContent::Text(_) => ClipboardFormat::Text,
ClipboardContent::Image(_) => ClipboardFormat::Image,
ClipboardContent::FileUrls(_) => ClipboardFormat::FileUrls,
}
}
}
#[cfg(test)]
#[path = "clipboard_content_tests.rs"]
mod tests;

34
crates/core/src/clipboard_content_tests.rs Normal file
View file

@ -0,0 +1,34 @@
use super::*;
#[test]
fn text_content_reports_text_format() {
let content = ClipboardContent::Text("hello".into());
assert_eq!(content.format(), ClipboardFormat::Text);
assert_eq!(ClipboardFormat::Text.as_str(), "text");
}
#[test]
fn image_content_reports_image_format() {
let content = ClipboardContent::Image(ImageBuffer {
data: vec![0u8; 4],
format: crate::ImageFormat::Png,
width: 2,
height: 2,
scale_factor: 1.0,
});
assert_eq!(content.format(), ClipboardFormat::Image);
assert_eq!(ClipboardFormat::Image.as_str(), "image");
}
#[test]
fn file_urls_content_reports_file_urls_format() {
let content = ClipboardContent::FileUrls(vec!["/tmp/a.txt".into()]);
assert_eq!(content.format(), ClipboardFormat::FileUrls);
assert_eq!(ClipboardFormat::FileUrls.as_str(), "file_urls");
}
#[test]
fn auto_format_tag_is_distinct_from_content_formats() {
assert_eq!(ClipboardFormat::Auto.as_str(), "auto");
assert_ne!(ClipboardFormat::Auto, ClipboardFormat::Text);
}

18
crates/core/src/clipboard_format.rs Normal file
View file

@ -0,0 +1,18 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClipboardFormat {
Auto,
Text,
Image,
FileUrls,
}
impl ClipboardFormat {
pub fn as_str(&self) -> &'static str {
match self {
ClipboardFormat::Auto => "auto",
ClipboardFormat::Text => "text",
ClipboardFormat::Image => "image",
ClipboardFormat::FileUrls => "file_urls",
}
}
}

2
crates/core/src/commands/batch.rs
View file

@ -1,4 +1,4 @@
use crate::error::AppError;
use crate::AppError;
use serde::Deserialize;
use serde_json::Value;

2
crates/core/src/commands/check.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

2
crates/core/src/commands/clear.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

2
crates/core/src/commands/click.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

5
crates/core/src/commands/clipboard_clear.rs
View file

@ -1,7 +1,8 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{AppError, adapter::PlatformAdapter};
use serde_json::{Value, json};
pub fn execute(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
adapter.clear_clipboard()?;
let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?;
adapter.clear_clipboard(&lease)?;
Ok(json!({ "cleared": true }))
}

102
crates/core/src/commands/clipboard_get.rs
View file

@ -1,7 +1,101 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{
AppError, ClipboardContent, ClipboardFormat, ImageBuffer,
adapter::PlatformAdapter,
context::CommandContext,
refs::{write_private_file, write_user_file},
refs_store::{
STALE_TMP_MAX_AGE,
prune::{is_orphaned_tmp_file, remove_stale_files_in_dir},
},
session,
};
use serde_json::{Value, json};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
pub fn execute(adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
let text = adapter.get_clipboard()?;
Ok(json!({ "text": text }))
pub struct ClipboardGetArgs {
pub format: Option<ClipboardFormat>,
pub out: Option<PathBuf>,
}
static IMAGE_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
const CLIPBOARD_IMAGE_MAX_AGE: Duration = Duration::from_secs(60 * 60);
pub fn execute(
args: ClipboardGetArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let format = args.format.unwrap_or(ClipboardFormat::Text);
let Some(content) = adapter.get_clipboard_content(format, crate::Deadline::standard()?)? else {
return Ok(json!({ "type": format.as_str(), "found": false }));
};
match content {
ClipboardContent::Text(text) => Ok(json!({ "type": "text", "text": text })),
ClipboardContent::FileUrls(file_urls) => {
Ok(json!({ "type": "file_urls", "file_urls": file_urls }))
}
ClipboardContent::Image(image) => write_image(image, args.out, context),
}
}
fn write_image(
image: ImageBuffer,
out: Option<PathBuf>,
context: &CommandContext,
) -> Result<Value, AppError> {
let path = match out {
Some(path) => {
write_user_file(&path, &image.data)?;
path
}
None => {
let path = default_clipboard_image_path(context)?;
write_private_file(&path, &image.data)?;
path
}
};
Ok(json!({
"type": "image",
"path": path.to_string_lossy(),
"width": image.width,
"height": image.height,
}))
}
fn default_clipboard_image_path(context: &CommandContext) -> Result<PathBuf, AppError> {
let dir = match context.session_id() {
Some(id) => session::session_dir(id)?.join("clipboard"),
None => {
let dir = session::agent_desktop_dir()?.join("tmp");
prune_sessionless_clipboard_tmp_dir(&dir);
dir
}
};
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let seq = IMAGE_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
Ok(dir.join(format!(
"clipboard-{}-{nanos}-{seq}.png",
std::process::id()
)))
}
fn prune_sessionless_clipboard_tmp_dir(dir: &Path) {
remove_stale_files_in_dir(dir, STALE_TMP_MAX_AGE, is_orphaned_tmp_file);
remove_stale_files_in_dir(dir, CLIPBOARD_IMAGE_MAX_AGE, is_clipboard_image_file);
}
fn is_clipboard_image_file(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.starts_with("clipboard-") && name.ends_with(".png"))
}
#[cfg(test)]
#[path = "clipboard_get_tests.rs"]
mod tests;

392
crates/core/src/commands/clipboard_get_tests.rs Normal file
View file

@ -0,0 +1,392 @@
use super::*;
use crate::AdapterError;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::refs_test_support::HomeGuard;
use std::sync::Mutex;
struct LocalDouble {
response: Mutex<Option<Result<Option<ClipboardContent>, AdapterError>>>,
seen_format: Mutex<Option<ClipboardFormat>>,
}
impl LocalDouble {
fn returning(result: Result<Option<ClipboardContent>, AdapterError>) -> Self {
Self {
response: Mutex::new(Some(result)),
seen_format: Mutex::new(None),
}
}
}
impl ObservationOps for LocalDouble {}
impl ActionOps for LocalDouble {}
impl SystemOps for LocalDouble {}
impl InputOps for LocalDouble {
fn get_clipboard_content(
&self,
format: ClipboardFormat,
_deadline: crate::Deadline,
) -> Result<Option<ClipboardContent>, AdapterError> {
*self.seen_format.lock().unwrap() = Some(format);
self.response
.lock()
.unwrap()
.take()
.expect("double invoked more than once in a single test")
}
}
fn no_session_context() -> CommandContext {
CommandContext::default()
}
#[test]
fn omitted_format_defaults_to_text_not_auto() {
let double = LocalDouble::returning(Ok(Some(ClipboardContent::Text("hi there".into()))));
let out = execute(
ClipboardGetArgs {
format: None,
out: None,
},
&double,
&no_session_context(),
)
.unwrap();
assert_eq!(
*double.seen_format.lock().unwrap(),
Some(ClipboardFormat::Text)
);
assert_eq!(out["type"], "text");
assert_eq!(out["text"], "hi there");
}
#[test]
fn missing_content_for_requested_format_is_structured_not_found_not_an_error() {
let double = LocalDouble::returning(Ok(None));
let out = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: None,
},
&double,
&no_session_context(),
)
.expect("must return Ok(..) not panic or bubble a hard error");
assert_eq!(out["type"], "image");
assert_eq!(out["found"], false);
}
#[test]
fn file_urls_variant_serializes_list() {
let double = LocalDouble::returning(Ok(Some(ClipboardContent::FileUrls(vec![
"/tmp/a.txt".into(),
"/tmp/b.txt".into(),
]))));
let out = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::FileUrls),
out: None,
},
&double,
&no_session_context(),
)
.unwrap();
assert_eq!(out["type"], "file_urls");
assert_eq!(
out["file_urls"],
serde_json::json!(["/tmp/a.txt", "/tmp/b.txt"])
);
}
#[test]
fn auto_format_request_is_relayed_to_adapter_unmodified() {
let double = LocalDouble::returning(Ok(Some(ClipboardContent::Text("auto text".into()))));
let _ = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Auto),
out: None,
},
&double,
&no_session_context(),
)
.unwrap();
assert_eq!(
*double.seen_format.lock().unwrap(),
Some(ClipboardFormat::Auto)
);
}
#[test]
fn image_variant_with_explicit_out_writes_that_path() {
let _home = HomeGuard::new();
let dir = std::env::temp_dir().join(format!(
"agent-desktop-clipboard-get-test-{}",
std::process::id()
));
std::fs::create_dir_all(&dir).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
}
let out_path = dir.join("explicit.png");
let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer {
data: vec![1, 2, 3, 4],
format: crate::ImageFormat::Png,
width: 10,
height: 5,
scale_factor: 1.0,
}))));
let out = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: Some(out_path.clone()),
},
&double,
&no_session_context(),
)
.unwrap();
assert_eq!(out["type"], "image");
assert_eq!(out["path"], out_path.to_string_lossy().into_owned());
assert_eq!(out["width"], 10);
assert_eq!(out["height"], 5);
assert_eq!(std::fs::read(&out_path).unwrap(), vec![1, 2, 3, 4]);
let _ = std::fs::remove_dir_all(&dir);
}
fn write_with_age(path: &std::path::Path, contents: &[u8], age: Duration) {
std::fs::write(path, contents).unwrap();
let file = std::fs::OpenOptions::new().write(true).open(path).unwrap();
file.set_modified(SystemTime::now() - age).unwrap();
}
#[test]
fn clipboard_image_predicate_matches_only_the_generated_shape() {
assert!(is_clipboard_image_file(Path::new(
"clipboard-123-456-0.png"
)));
assert!(!is_clipboard_image_file(Path::new(
"clipboard-123-456-0.png.tmp"
)));
assert!(!is_clipboard_image_file(Path::new("other.png")));
assert!(!is_clipboard_image_file(Path::new("clipboard-123.txt")));
}
#[test]
fn sessionless_prune_removes_stale_matches_keeps_fresh_and_unrelated() {
let _home = HomeGuard::new();
let dir = session::agent_desktop_dir().unwrap().join("tmp");
std::fs::create_dir_all(&dir).unwrap();
let stale_png = dir.join("clipboard-1-1-0.png");
let stale_tmp = dir.join("orphan-write.tmp");
let fresh_png = dir.join("clipboard-2-2-0.png");
let unrelated = dir.join("notes.txt");
write_with_age(
&stale_png,
b"old",
CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1),
);
write_with_age(
&stale_tmp,
b"old",
STALE_TMP_MAX_AGE + Duration::from_secs(1),
);
write_with_age(&fresh_png, b"new", Duration::from_secs(1));
write_with_age(
&unrelated,
b"keep me",
CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1),
);
prune_sessionless_clipboard_tmp_dir(&dir);
assert!(
!stale_png.exists(),
"clipboard png older than its TTL must be pruned"
);
assert!(
!stale_tmp.exists(),
"orphaned tmp file older than its TTL must be pruned"
);
assert!(
fresh_png.exists(),
"clipboard png within its TTL must survive the sweep"
);
assert!(
unrelated.exists(),
"a file outside the clipboard-*.png / *.tmp shapes must never be touched"
);
}
#[test]
fn sweep_never_removes_a_directory_even_if_it_matches_the_shape() {
let _home = HomeGuard::new();
let dir = session::agent_desktop_dir().unwrap().join("tmp");
let fake_dir = dir.join("clipboard-0-0-0.png");
std::fs::create_dir_all(&fake_dir).unwrap();
remove_stale_files_in_dir(&dir, Duration::ZERO, is_clipboard_image_file);
assert!(
fake_dir.is_dir(),
"the sweep must never remove a directory, even one matching the file shape"
);
}
#[test]
fn consecutive_sessionless_image_captures_keep_the_prior_capture() {
let _home = HomeGuard::new();
let first_double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer {
data: vec![1],
format: crate::ImageFormat::Png,
width: 1,
height: 1,
scale_factor: 1.0,
}))));
let first = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: None,
},
&first_double,
&no_session_context(),
)
.unwrap();
let first_path = PathBuf::from(first["path"].as_str().unwrap());
let second_double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer {
data: vec![2],
format: crate::ImageFormat::Png,
width: 1,
height: 1,
scale_factor: 1.0,
}))));
let _ = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: None,
},
&second_double,
&no_session_context(),
)
.unwrap();
assert!(
first_path.exists(),
"a clipboard image captured moments ago must not be pruned by the very next capture"
);
}
#[test]
fn image_variant_without_out_prunes_stale_sessionless_artifacts() {
let _home = HomeGuard::new();
let dir = session::agent_desktop_dir().unwrap().join("tmp");
std::fs::create_dir_all(&dir).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
for path in [dir.parent().unwrap(), dir.as_path()] {
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap();
}
}
let stale_png = dir.join("clipboard-9-9-0.png");
let stale_tmp = dir.join("orphan.tmp");
let unrelated = dir.join("keep.txt");
write_with_age(
&stale_png,
b"old capture",
CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1),
);
write_with_age(
&stale_tmp,
b"old",
STALE_TMP_MAX_AGE + Duration::from_secs(1),
);
write_with_age(
&unrelated,
b"keep",
CLIPBOARD_IMAGE_MAX_AGE + Duration::from_secs(1),
);
let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer {
data: vec![7, 7, 7],
format: crate::ImageFormat::Png,
width: 1,
height: 1,
scale_factor: 1.0,
}))));
let out = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: None,
},
&double,
&no_session_context(),
)
.unwrap();
let new_path = PathBuf::from(out["path"].as_str().unwrap());
assert!(
new_path.exists(),
"the image just written by this call must survive its own write"
);
assert!(
!stale_png.exists(),
"a stale clipboard png must be pruned by the next capture"
);
assert!(
!stale_tmp.exists(),
"a stale orphaned tmp file must be pruned by the next capture"
);
assert!(
unrelated.exists(),
"a non-matching file must never be touched"
);
}
#[cfg(unix)]
#[test]
fn image_variant_without_out_writes_private_0600_file_under_session_dir() {
use std::os::unix::fs::PermissionsExt;
let _home = HomeGuard::new();
let context = CommandContext::new(Some("clip-get-test-session".into()), None, false).unwrap();
let double = LocalDouble::returning(Ok(Some(ClipboardContent::Image(ImageBuffer {
data: vec![9, 9, 9],
format: crate::ImageFormat::Png,
width: 3,
height: 3,
scale_factor: 1.0,
}))));
let out = execute(
ClipboardGetArgs {
format: Some(ClipboardFormat::Image),
out: None,
},
&double,
&context,
)
.unwrap();
let path = out["path"].as_str().expect("path field").to_string();
let session_dir = session::session_dir("clip-get-test-session")
.unwrap()
.join("clipboard");
assert!(
std::path::Path::new(&path).starts_with(&session_dir),
"default image path {path} must live under the session's clipboard dir {}",
session_dir.display()
);
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
assert_eq!(
mode, 0o600,
"clipboard image temp file must be 0600, got {mode:o}"
);
}

84
crates/core/src/commands/clipboard_set.rs
View file

@ -1,7 +1,83 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{
AppError, ClipboardContent, ImageBuffer, ImageFormat, adapter::PlatformAdapter,
parse_png_dimensions,
};
use serde_json::{Value, json};
use std::path::PathBuf;
pub fn execute(text: String, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
adapter.set_clipboard(&text)?;
Ok(json!({ "ok": true }))
pub struct ClipboardSetArgs {
pub text: Option<String>,
pub image: Option<PathBuf>,
pub file_urls: Vec<String>,
}
pub fn execute(args: ClipboardSetArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
let content = build_content(args)?;
let format = content.format();
let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?;
adapter.set_clipboard_content(&content, &lease)?;
Ok(json!({ "ok": true, "type": format.as_str() }))
}
fn build_content(args: ClipboardSetArgs) -> Result<ClipboardContent, AppError> {
if !args.file_urls.is_empty() {
return Ok(ClipboardContent::FileUrls(validate_file_urls(
&args.file_urls,
)?));
}
if let Some(path) = args.image {
let data =
crate::private_file::read_regular_bounded(&path, crate::MAX_PNG_INPUT_BYTES as u64)
.map_err(image_read_error)?;
let Some((width, height)) = parse_png_dimensions(&data) else {
return Err(AppError::invalid_input("--image file is not a valid PNG"));
};
return Ok(ClipboardContent::Image(ImageBuffer {
data,
format: ImageFormat::Png,
width,
height,
scale_factor: 1.0,
}));
}
Ok(ClipboardContent::Text(args.text.unwrap_or_default()))
}
fn image_read_error(error: std::io::Error) -> AppError {
if matches!(
error.kind(),
std::io::ErrorKind::InvalidData
| std::io::ErrorKind::NotFound
| std::io::ErrorKind::PermissionDenied
) {
return AppError::invalid_input(format!(
"--image must be a local regular PNG no larger than {} MiB",
crate::MAX_PNG_INPUT_BYTES / (1024 * 1024)
));
}
AppError::Io(error)
}
/// Reports missing `--file-url` entries by count and index only — the
/// entries themselves may be sensitive paths and this error can reach a
/// trace file.
fn validate_file_urls(urls: &[String]) -> Result<Vec<String>, AppError> {
let missing: Vec<usize> = urls
.iter()
.enumerate()
.filter(|(_, path)| !std::path::Path::new(path).exists())
.map(|(index, _)| index)
.collect();
if !missing.is_empty() {
return Err(AppError::invalid_input(format!(
"{} of {} --file-url entries do not exist on disk (indexes: {missing:?})",
missing.len(),
urls.len()
)));
}
Ok(urls.to_vec())
}
#[cfg(test)]
#[path = "clipboard_set_tests.rs"]
mod tests;

246
crates/core/src/commands/clipboard_set_tests.rs Normal file
View file

@ -0,0 +1,246 @@
use super::*;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::{AdapterError, ErrorCode};
use std::sync::Mutex;
struct LocalDouble {
seen: Mutex<Option<ClipboardContent>>,
}
impl LocalDouble {
fn new() -> Self {
Self {
seen: Mutex::new(None),
}
}
}
impl ObservationOps for LocalDouble {}
impl ActionOps for LocalDouble {}
impl SystemOps for LocalDouble {
crate::adapter::guarded_interaction_lease!();
}
impl InputOps for LocalDouble {
fn set_clipboard_content(
&self,
content: &ClipboardContent,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
let stored = match content {
ClipboardContent::Text(text) => ClipboardContent::Text(text.clone()),
ClipboardContent::Image(image) => ClipboardContent::Image(ImageBuffer {
data: image.data.clone(),
format: crate::ImageFormat::Png,
width: image.width,
height: image.height,
scale_factor: image.scale_factor,
}),
ClipboardContent::FileUrls(urls) => ClipboardContent::FileUrls(urls.clone()),
};
*self.seen.lock().unwrap() = Some(stored);
Ok(())
}
}
#[test]
fn default_with_only_text_writes_text_content() {
let double = LocalDouble::new();
let out = execute(
ClipboardSetArgs {
text: Some("hello clipboard".into()),
image: None,
file_urls: vec![],
},
&double,
)
.unwrap();
assert_eq!(out["type"], "text");
match double.seen.lock().unwrap().as_ref().unwrap() {
ClipboardContent::Text(text) => assert_eq!(text, "hello clipboard"),
_ => panic!("expected Text content"),
}
}
#[test]
fn omitted_text_defaults_to_empty_string_not_error() {
let double = LocalDouble::new();
let out = execute(
ClipboardSetArgs {
text: None,
image: None,
file_urls: vec![],
},
&double,
)
.unwrap();
assert_eq!(out["type"], "text");
match double.seen.lock().unwrap().as_ref().unwrap() {
ClipboardContent::Text(text) => assert_eq!(text, ""),
_ => panic!("expected Text content"),
}
}
#[test]
fn missing_file_url_reports_invalid_args_with_count_and_index_only() {
let existing = std::env::temp_dir().join(format!(
"agent-desktop-clipboard-set-test-{}.txt",
std::process::id()
));
std::fs::write(&existing, b"present").unwrap();
let double = LocalDouble::new();
let err = execute(
ClipboardSetArgs {
text: None,
image: None,
file_urls: vec![
existing.to_string_lossy().into_owned(),
"/definitely/does/not/exist/secret-project-name.txt".into(),
],
},
&double,
)
.unwrap_err();
let _ = std::fs::remove_file(&existing);
assert_eq!(err.code(), ErrorCode::InvalidArgs.as_str());
let message = err.to_string();
assert!(
message.contains("1 of 2"),
"message should report count, got: {message}"
);
assert!(
message.contains("[1]"),
"message should report the missing entry's index (1), got: {message}"
);
assert!(
!message.contains("secret-project-name"),
"message must not leak the missing path's content, got: {message}"
);
assert!(
double.seen.lock().unwrap().is_none(),
"adapter must not be called on validation failure"
);
}
#[test]
fn image_flag_reads_bytes_and_dimensions_from_file() {
let path = std::env::temp_dir().join(format!(
"agent-desktop-clipboard-set-image-{}.png",
std::process::id()
));
let bytes = vec![
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 4,
0, 0, 0, 181, 28, 12, 2, 0, 0, 0, 11, 73, 68, 65, 84, 120, 218, 99, 100, 248, 15, 0, 1, 5,
1, 1, 39, 24, 227, 102, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
];
std::fs::write(&path, &bytes).unwrap();
let double = LocalDouble::new();
let out = execute(
ClipboardSetArgs {
text: None,
image: Some(path.clone()),
file_urls: vec![],
},
&double,
)
.unwrap();
let _ = std::fs::remove_file(&path);
assert_eq!(out["type"], "image");
match double.seen.lock().unwrap().as_ref().unwrap() {
ClipboardContent::Image(image) => {
assert_eq!(image.width, 1);
assert_eq!(image.height, 1);
assert_eq!(image.data, bytes);
}
_ => panic!("expected Image content"),
}
}
#[test]
fn missing_image_path_reports_invalid_args() {
let missing = std::env::temp_dir().join(format!(
"agent-desktop-missing-clipboard-image-{}.png",
crate::refs::new_snapshot_id()
));
let error = build_content(ClipboardSetArgs {
text: None,
image: Some(missing),
file_urls: Vec::new(),
})
.unwrap_err();
assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str());
}
#[test]
fn file_urls_take_priority_over_text_and_image() {
let existing = std::env::temp_dir().join(format!(
"agent-desktop-clipboard-set-priority-{}.txt",
std::process::id()
));
std::fs::write(&existing, b"present").unwrap();
let double = LocalDouble::new();
let out = execute(
ClipboardSetArgs {
text: Some("ignored".into()),
image: None,
file_urls: vec![existing.to_string_lossy().into_owned()],
},
&double,
)
.unwrap();
let _ = std::fs::remove_file(&existing);
assert_eq!(out["type"], "file_urls");
match double.seen.lock().unwrap().as_ref().unwrap() {
ClipboardContent::FileUrls(urls) => assert_eq!(urls.len(), 1),
_ => panic!("expected FileUrls content"),
}
}
#[cfg(unix)]
#[test]
fn image_rejects_fifo_without_blocking() {
use std::ffi::CString;
let path = std::env::temp_dir().join(format!(
"agent-desktop-clipboard-set-fifo-{}",
crate::refs::new_snapshot_id()
));
let c_path = CString::new(path.to_string_lossy().as_bytes()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0);
let started = std::time::Instant::now();
let error = build_content(ClipboardSetArgs {
text: None,
image: Some(path.clone()),
file_urls: Vec::new(),
})
.unwrap_err();
assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str());
assert!(started.elapsed() < std::time::Duration::from_secs(1));
std::fs::remove_file(path).unwrap();
}
#[cfg(unix)]
#[test]
fn image_rejects_device_files() {
let error = build_content(ClipboardSetArgs {
text: None,
image: Some(PathBuf::from("/dev/null")),
file_urls: Vec::new(),
})
.unwrap_err();
assert_eq!(error.code(), ErrorCode::InvalidArgs.as_str());
}

38
crates/core/src/commands/close_app.rs
View file

@ -1,4 +1,4 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{AppError, adapter::PlatformAdapter};
use serde_json::{Value, json};
pub struct CloseAppArgs {
@ -8,23 +8,41 @@ pub struct CloseAppArgs {
pub fn execute(args: CloseAppArgs, adapter: &dyn PlatformAdapter) -> Result<Value, AppError> {
if adapter.is_protected_process(&args.app) {
return Err(AppError::invalid_input_with_suggestion(
format!(
"'{}' is a protected system process and cannot be closed",
args.app
),
"Target a regular application; session-critical processes (loginwindow, WindowServer, Dock, Finder, launchd) are never closed.",
));
return Err(protected_process_error(&args.app));
}
adapter.close_app(&args.app, args.force)?;
let deadline = crate::Deadline::standard()?;
let expected = crate::commands::helpers::resolve_app(Some(&args.app), adapter, deadline)?;
let protected = adapter.is_protected_process(&args.app)
|| adapter.is_protected_process(&expected.name)
|| expected
.bundle_id
.as_deref()
.is_some_and(|bundle| adapter.is_protected_process(bundle));
if protected {
return Err(protected_process_error(&args.app));
}
let lease = adapter.acquire_interaction_lease(deadline)?;
let live = crate::commands::helpers::revalidate_app_for_mutation(
adapter,
&expected,
lease.deadline(),
)?;
adapter.close_app(&live, args.force, &lease)?;
Ok(json!({
"app": args.app,
"method": if args.force { "force" } else { "graceful" },
"requested": true,
"closed": args.force
"closed": true
}))
}
fn protected_process_error(app: &str) -> AppError {
AppError::invalid_input_with_suggestion(
format!("'{app}' is a protected system process and cannot be closed"),
"Target a regular application; session-critical processes (loginwindow, WindowServer, Dock, Finder, launchd) are never closed.",
)
}
#[cfg(test)]
#[path = "close_app_tests.rs"]
mod tests;

65
crates/core/src/commands/close_app_tests.rs
View file

@ -1,23 +1,67 @@
use super::*;
use crate::adapter::PlatformAdapter;
use crate::error::{AdapterError, ErrorCode};
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::{AdapterError, ErrorCode};
struct ProtectiveAdapter;
impl PlatformAdapter for ProtectiveAdapter {
impl ObservationOps for ProtectiveAdapter {
fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<crate::AppInfo>, AdapterError> {
Ok(vec![crate::AppInfo {
name: "TextEdit".into(),
pid: crate::ProcessId::new(42),
bundle_id: Some("com.apple.TextEdit".into()),
process_instance: Some("textedit-instance".into()),
}])
}
}
impl ActionOps for ProtectiveAdapter {}
impl InputOps for ProtectiveAdapter {}
impl SystemOps for ProtectiveAdapter {
crate::adapter::guarded_interaction_lease!();
fn is_protected_process(&self, identifier: &str) -> bool {
identifier.eq_ignore_ascii_case("CriticalThing")
}
fn close_app(&self, _id: &str, _force: bool) -> Result<(), crate::error::AdapterError> {
fn close_app(
&self,
_app: &crate::AppInfo,
_force: bool,
_lease: &crate::InteractionLease,
) -> Result<(), crate::AdapterError> {
Ok(())
}
}
struct FailingAdapter;
impl PlatformAdapter for FailingAdapter {
fn close_app(&self, _id: &str, _force: bool) -> Result<(), AdapterError> {
impl ObservationOps for FailingAdapter {
fn list_apps(&self, _deadline: crate::Deadline) -> Result<Vec<crate::AppInfo>, AdapterError> {
Ok(vec![crate::AppInfo {
name: "Ghost".into(),
pid: crate::ProcessId::new(77),
bundle_id: None,
process_instance: Some("ghost-instance".into()),
}])
}
}
impl ActionOps for FailingAdapter {}
impl InputOps for FailingAdapter {}
impl SystemOps for FailingAdapter {
crate::adapter::guarded_interaction_lease!();
fn close_app(
&self,
_app: &crate::AppInfo,
_force: bool,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Err(AdapterError::new(ErrorCode::AppNotFound, "no such app"))
}
}
@ -45,7 +89,7 @@ fn close_app_blocks_adapter_protected_process() {
}
#[test]
fn graceful_close_reports_request_without_claiming_termination() {
fn graceful_close_reports_verified_termination() {
let value = execute(
CloseAppArgs {
app: "TextEdit".into(),
@ -58,10 +102,7 @@ fn graceful_close_reports_request_without_claiming_termination() {
assert_eq!(value["app"], "TextEdit");
assert_eq!(value["method"], "graceful");
assert_eq!(value["requested"], true);
assert_eq!(
value["closed"], false,
"graceful close must not claim a termination it cannot confirm"
);
assert_eq!(value["closed"], true);
}
#[test]
@ -69,7 +110,7 @@ fn close_app_propagates_adapter_errors() {
let err = execute(
CloseAppArgs {
app: "Ghost".into(),
force: true,
force: false,
},
&FailingAdapter,
)

2
crates/core/src/commands/collapse.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

8
crates/core/src/commands/combo.rs
View file

@ -1,8 +1,4 @@
use crate::{
action::{KeyCombo, Modifier},
adapter::PlatformAdapter,
error::{AdapterError, AppError},
};
use crate::{AdapterError, AppError, KeyCombo, Modifier, adapter::PlatformAdapter};
/// Refuses `combo` when the platform adapter reports it as dangerous, unless
/// the caller forced it. Whether a combo is dangerous is the adapter's
@ -39,7 +35,7 @@ pub fn parse_combo(s: &str) -> Result<KeyCombo, AppError> {
for &part in &parts[..parts.len() - 1] {
let modifier = match part {
"cmd" | "command" => Modifier::Cmd,
"meta" | "cmd" | "command" => Modifier::Meta,
"ctrl" | "control" => Modifier::Ctrl,
"alt" | "option" => Modifier::Alt,
"shift" => Modifier::Shift,

13
crates/core/src/commands/combo_tests.rs
View file

@ -1,18 +1,18 @@
use super::parse_combo;
use crate::action::Modifier;
use crate::Modifier;
#[test]
fn parse_combo_single_modifier_and_key() {
let combo = parse_combo("cmd+k").expect("cmd+k is valid");
assert_eq!(combo.key, "k");
assert_eq!(combo.modifiers, vec![Modifier::Cmd]);
assert_eq!(combo.modifiers, vec![Modifier::Meta]);
}
#[test]
fn parse_combo_two_modifiers_preserved_in_declaration_order() {
let combo = parse_combo("cmd+shift+t").expect("cmd+shift+t is valid");
assert_eq!(combo.key, "t");
assert_eq!(combo.modifiers, vec![Modifier::Cmd, Modifier::Shift]);
assert_eq!(combo.modifiers, vec![Modifier::Meta, Modifier::Shift]);
}
#[test]
@ -25,7 +25,10 @@ fn parse_combo_bare_key_yields_empty_modifier_list() {
#[test]
fn parse_combo_accepts_long_form_modifier_aliases() {
let cmd = parse_combo("command+a").expect("command alias");
assert_eq!(cmd.modifiers, vec![Modifier::Cmd]);
assert_eq!(cmd.modifiers, vec![Modifier::Meta]);
let meta = parse_combo("meta+a").expect("portable meta name");
assert_eq!(meta.modifiers, vec![Modifier::Meta]);
let alt = parse_combo("option+x").expect("option alias");
assert_eq!(alt.modifiers, vec![Modifier::Alt]);
@ -58,5 +61,5 @@ fn parse_combo_key_is_preserved_verbatim_without_lowercasing() {
combo.key, "K",
"parse_combo must NOT lowercase the key — normalization is the caller's responsibility"
);
assert_eq!(combo.modifiers, vec![Modifier::Cmd]);
assert_eq!(combo.modifiers, vec![Modifier::Meta]);
}

13
crates/core/src/commands/dismiss_all_notifications.rs
View file

@ -1,4 +1,4 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{AppError, adapter::PlatformAdapter, context::CommandContext};
use serde_json::{Value, json};
pub struct DismissAllNotificationsArgs {
@ -8,8 +8,17 @@ pub struct DismissAllNotificationsArgs {
pub fn execute(
args: DismissAllNotificationsArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let (dismissed, failures) = adapter.dismiss_all_notifications(args.app.as_deref())?;
let policy = super::notification_policy::mutation_policy(context)?;
let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?;
let (dismissed, failures) = adapter.dismiss_all_notifications(
crate::DismissAllNotificationsRequest {
app_filter: args.app.as_deref(),
policy,
},
&lease,
)?;
let mut result = json!({
"dismissed_count": dismissed.len(),
});

19
crates/core/src/commands/dismiss_notification.rs
View file

@ -1,16 +1,31 @@
use crate::{adapter::PlatformAdapter, error::AppError};
use crate::{AppError, adapter::PlatformAdapter, context::CommandContext};
use serde_json::{Value, json};
pub struct DismissNotificationArgs {
pub index: usize,
pub app: Option<String>,
pub expected_app: Option<String>,
pub expected_title: Option<String>,
}
pub fn execute(
args: DismissNotificationArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let dismissed = adapter.dismiss_notification(args.index, args.app.as_deref())?;
let identity =
super::notification_identity::required_identity(args.expected_app, args.expected_title)?;
let policy = super::notification_policy::mutation_policy(context)?;
let lease = crate::commands::helpers::acquire_interaction_lease(adapter)?;
let dismissed = adapter.dismiss_notification(
crate::DismissNotificationRequest {
index: args.index,
app_filter: args.app.as_deref(),
identity: &identity,
policy,
},
&lease,
)?;
Ok(json!({
"dismissed": dismissed,
}))

2
crates/core/src/commands/double_click.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

131
crates/core/src/commands/drag.rs
View file

@ -1,23 +1,30 @@
use crate::{
action::DragParams,
AppError, DragParams,
adapter::PlatformAdapter,
commands::point_resolve::{
PointResolveArgs, focus_for_physical_input, require_cursor_policy,
resolve_point_from_ref_or_xy_with_context,
commands::{
helpers::{apply_post_action_wait, validate_post_action_wait},
point_resolve::{PointResolveArgs, require_cursor_policy},
pointer_action::{
PointResolveAttempt, ensure_point_deadline, focus_point_under_lease, point_deadline,
resolve_point_under_lease, retry_leased_point_phase, wait_for_point_with_deadline,
},
},
context::CommandContext,
error::AppError,
};
use serde_json::{Value, json};
pub struct DragEndpoint {
pub ref_id: Option<String>,
pub xy: Option<(f64, f64)>,
}
pub struct DragArgs {
pub from_ref: Option<String>,
pub from_xy: Option<(f64, f64)>,
pub to_ref: Option<String>,
pub to_xy: Option<(f64, f64)>,
pub from: DragEndpoint,
pub to: DragEndpoint,
pub snapshot_id: Option<String>,
pub duration_ms: Option<u64>,
pub drop_delay_ms: Option<u64>,
pub timeout_ms: Option<u64>,
}
pub fn execute(
@ -26,34 +33,93 @@ pub fn execute(
context: &CommandContext,
) -> Result<Value, AppError> {
require_cursor_policy(context, "drag")?;
let from = resolve_point_from_ref_or_xy_with_context(
PointResolveArgs {
ref_id: args.from_ref.as_deref(),
xy: args.from_xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide --from <ref> or --from-xy x,y",
},
adapter,
context,
validate_post_action_wait(context)?;
let deadline = point_deadline(args.timeout_ms)?;
let from_args = PointResolveArgs {
ref_id: args.from.ref_id.as_deref(),
xy: args.from.xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide --from <ref> or --from-xy x,y",
headed_requirement: crate::HeadedRequirement::FocusedWindowAndCursor,
};
let to_args = PointResolveArgs {
ref_id: args.to.ref_id.as_deref(),
xy: args.to.xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide --to <ref> or --to-xy x,y",
headed_requirement: crate::HeadedRequirement::None,
};
let auto_wait = args.timeout_ms.is_some_and(|timeout_ms| timeout_ms > 0);
if auto_wait {
wait_for_point_with_deadline(from_args, deadline, adapter, context)?;
wait_for_point_with_deadline(to_args, deadline, adapter, context)?;
}
let (from, to, lease) = retry_leased_point_phase(args.timeout_ms, deadline, || {
let lease = adapter.acquire_interaction_lease(deadline)?;
let focused = focus_point_under_lease(from_args, &lease, adapter, context)?;
let from = resolve_point_under_lease(
PointResolveAttempt {
args: from_args,
stability: None,
allow_scroll: !auto_wait,
},
deadline,
&lease,
adapter,
context,
)?;
let to = resolve_point_under_lease(
PointResolveAttempt {
args: to_args,
stability: None,
allow_scroll: !auto_wait,
},
deadline,
&lease,
adapter,
context,
)?;
let mut from = resolve_point_under_lease(
PointResolveAttempt {
args: from_args,
stability: Some(from.bounds_hash),
allow_scroll: false,
},
deadline,
&lease,
adapter,
context,
)?;
let to = resolve_point_under_lease(
PointResolveAttempt {
args: to_args,
stability: Some(to.bounds_hash),
allow_scroll: false,
},
deadline,
&lease,
adapter,
context,
)?;
from.focused = focused;
Ok((from, to, lease))
})?;
ensure_point_deadline(
deadline,
Some(json!({
"status": "actionable",
"from": { "x": from.point.x, "y": from.point.y },
"to": { "x": to.point.x, "y": to.point.y }
})),
)?;
let to = resolve_point_from_ref_or_xy_with_context(
PointResolveArgs {
ref_id: args.to_ref.as_deref(),
xy: args.to_xy,
snapshot_id: args.snapshot_id.as_deref(),
missing_input_message: "Provide --to <ref> or --to-xy x,y",
},
adapter,
context,
)?;
let focused = focus_for_physical_input(from.pid, adapter, context)?;
let params = DragParams {
from: from.point.clone(),
to: to.point.clone(),
duration_ms: args.duration_ms,
drop_delay_ms: args.drop_delay_ms,
};
adapter.drag(params)?;
params.validate(deadline)?;
adapter.drag(params, &lease)?;
let mut response = json!({
"dragged": true,
"from": { "x": from.point.x, "y": from.point.y },
@ -62,10 +128,11 @@ pub fn execute(
if let Some(drop_delay_ms) = args.drop_delay_ms {
response["drop_delay_ms"] = json!(drop_delay_ms);
}
if focused {
if from.focused {
response["focused"] = json!(true);
}
Ok(response)
drop(lease);
apply_post_action_wait(response, from.source_entry.as_ref(), adapter, context)
}
#[cfg(test)]

143
crates/core/src/commands/drag_occlusion_retry_tests.rs Normal file
View file

@ -0,0 +1,143 @@
use super::*;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
struct TransientOcclusionAdapter {
hit_tests: AtomicU32,
lease_acquisitions: AtomicU32,
lease_held: Arc<AtomicBool>,
captured: Mutex<Option<DragParams>>,
}
struct LeaseFlag(Arc<AtomicBool>);
impl Drop for LeaseFlag {
fn drop(&mut self) {
self.0.store(false, Ordering::SeqCst);
}
}
impl TransientOcclusionAdapter {
fn new() -> Self {
Self {
hit_tests: AtomicU32::new(0),
lease_acquisitions: AtomicU32::new(0),
lease_held: Arc::new(AtomicBool::new(false)),
captured: Mutex::new(None),
}
}
}
impl ObservationOps for TransientOcclusionAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
Ok(NativeHandle::null())
}
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
Ok(Some(Rect {
x: 10.0,
y: 20.0,
width: 40.0,
height: 60.0,
}))
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
if self.hit_tests.fetch_add(1, Ordering::SeqCst) == 0 {
Ok(HitTestResult::InterceptedBy {
role: Some("AXSheet".into()),
name: Some("Transient sheet".into()),
bounds: None,
})
} else {
Ok(HitTestResult::ReachesTarget)
}
}
}
impl ActionOps for TransientOcclusionAdapter {}
impl InputOps for TransientOcclusionAdapter {
fn drag(
&self,
params: DragParams,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
*self.captured.lock().unwrap() = Some(params);
Ok(())
}
}
impl SystemOps for TransientOcclusionAdapter {
fn acquire_interaction_lease(
&self,
deadline: crate::Deadline,
) -> Result<crate::InteractionLease, AdapterError> {
assert!(!self.lease_held.swap(true, Ordering::SeqCst));
self.lease_acquisitions.fetch_add(1, Ordering::SeqCst);
crate::InteractionLease::guarded(deadline, LeaseFlag(self.lease_held.clone()))
}
fn resolve_window_strict(
&self,
window: &crate::WindowInfo,
_deadline: crate::Deadline,
) -> Result<crate::WindowInfo, AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
_window: &crate::WindowInfo,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Ok(())
}
}
#[test]
fn drag_retries_transient_post_focus_occlusion_without_holding_lease() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = TransientOcclusionAdapter::new();
let mut args = cross_app_args(snapshot_id);
args.timeout_ms = Some(500);
let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap();
assert_eq!(value["dragged"], true);
assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 2);
assert!(!adapter.lease_held.load(Ordering::SeqCst));
assert!(adapter.captured.lock().unwrap().is_some());
}
#[test]
fn drag_single_shot_does_not_retry_post_focus_occlusion() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = TransientOcclusionAdapter::new();
let error = execute(
cross_app_args(snapshot_id),
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap_err();
assert_eq!(error.code(), "ACTION_FAILED");
assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 1);
assert!(adapter.captured.lock().unwrap().is_none());
}

378
crates/core/src/commands/drag_retry_tests.rs Normal file
View file

@ -0,0 +1,378 @@
use super::*;
struct StaleThenOkAdapter {
retry: StaleRetryCounter,
minimum_resolves_before_lease: u32,
lease_acquisitions: AtomicU32,
}
impl StaleThenOkAdapter {
fn new(fail_until: u32) -> Self {
Self {
retry: StaleRetryCounter::new(fail_until),
minimum_resolves_before_lease: 0,
lease_acquisitions: AtomicU32::new(0),
}
}
fn expect_poll_before_lease(mut self, minimum: u32) -> Self {
self.minimum_resolves_before_lease = minimum;
self
}
}
impl ObservationOps for StaleThenOkAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
self.retry.attempt()
}
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
Ok(Some(Rect {
x: 10.0,
y: 20.0,
width: 40.0,
height: 60.0,
}))
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
Ok(HitTestResult::ReachesTarget)
}
}
impl ActionOps for StaleThenOkAdapter {}
impl InputOps for StaleThenOkAdapter {
fn drag(
&self,
_params: DragParams,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Ok(())
}
}
impl SystemOps for StaleThenOkAdapter {
fn acquire_interaction_lease(
&self,
deadline: crate::Deadline,
) -> Result<crate::InteractionLease, AdapterError> {
assert!(self.retry.calls() >= self.minimum_resolves_before_lease);
self.lease_acquisitions.fetch_add(1, Ordering::SeqCst);
crate::InteractionLease::guarded(deadline, ())
}
fn resolve_window_strict(
&self,
window: &crate::WindowInfo,
_deadline: crate::Deadline,
) -> Result<crate::WindowInfo, AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
_window: &crate::WindowInfo,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Ok(())
}
}
#[test]
fn transient_stale_ref_retries_then_succeeds_when_timeout_wired() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = StaleThenOkAdapter::new(2).expect_poll_before_lease(3);
let value = execute(
DragArgs {
from: DragEndpoint {
ref_id: Some("@e1".into()),
xy: None,
},
to: DragEndpoint {
ref_id: Some("@e2".into()),
xy: None,
},
snapshot_id: Some(snapshot_id),
duration_ms: None,
drop_delay_ms: None,
timeout_ms: Some(5_000),
},
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap();
assert_eq!(value["dragged"], true);
assert!(adapter.retry.calls() >= 3);
assert_eq!(adapter.lease_acquisitions.load(Ordering::SeqCst), 1);
}
struct OccludedFromAdapter {
captured: Mutex<Option<DragParams>>,
}
impl ObservationOps for OccludedFromAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
Ok(NativeHandle::null())
}
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
Ok(Some(Rect {
x: 10.0,
y: 20.0,
width: 40.0,
height: 60.0,
}))
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
Ok(HitTestResult::InterceptedBy {
role: Some("AXSheet".into()),
name: Some("Save changes?".into()),
bounds: None,
})
}
}
impl ActionOps for OccludedFromAdapter {}
impl InputOps for OccludedFromAdapter {
fn drag(
&self,
params: DragParams,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
*self.captured.lock().unwrap() = Some(params);
Ok(())
}
}
impl SystemOps for OccludedFromAdapter {
crate::adapter::guarded_interaction_lease!();
fn resolve_window_strict(
&self,
window: &crate::WindowInfo,
_deadline: crate::Deadline,
) -> Result<crate::WindowInfo, AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
_window: &crate::WindowInfo,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Ok(())
}
}
#[test]
fn drag_from_occluded_ref_fails_preflight_before_dispatch() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = OccludedFromAdapter {
captured: Mutex::new(None),
};
let err = execute(
cross_app_args(snapshot_id),
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap_err();
assert_eq!(err.code(), "ACTION_FAILED");
let AppError::Adapter(adapter_error) = &err else {
panic!("expected adapter actionability failure")
};
assert!(
adapter_error
.details
.as_ref()
.is_some_and(|details| details.to_string().contains("AXSheet"))
);
assert!(adapter.captured.lock().unwrap().is_none());
}
struct MutuallyOffscreenAdapter {
visible_pid: AtomicU32,
dispatched: AtomicU32,
}
impl ObservationOps for MutuallyOffscreenAdapter {
fn resolve_element_strict(
&self,
entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
Ok(NativeHandle::new(entry.process.pid.get()))
}
fn get_element_bounds(
&self,
handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
let pid = *handle.downcast_ref::<u32>().unwrap();
if self.visible_pid.load(Ordering::SeqCst) != pid {
return Ok(None);
}
Ok(Some(Rect {
x: f64::from(pid) * 100.0,
y: 20.0,
width: 40.0,
height: 60.0,
}))
}
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
Ok(HitTestResult::ReachesTarget)
}
}
impl ActionOps for MutuallyOffscreenAdapter {
fn scroll_into_view(
&self,
handle: &NativeHandle,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
self.visible_pid
.store(*handle.downcast_ref::<u32>().unwrap(), Ordering::SeqCst);
Ok(())
}
}
impl InputOps for MutuallyOffscreenAdapter {
fn drag(
&self,
_params: DragParams,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
self.dispatched.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
impl SystemOps for MutuallyOffscreenAdapter {
crate::adapter::guarded_interaction_lease!();
fn resolve_window_strict(
&self,
window: &crate::WindowInfo,
_deadline: crate::Deadline,
) -> Result<crate::WindowInfo, AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
_window: &crate::WindowInfo,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
Ok(())
}
}
#[test]
fn drag_does_not_dispatch_when_endpoints_cannot_remain_visible_together() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = MutuallyOffscreenAdapter {
visible_pid: AtomicU32::new(1),
dispatched: AtomicU32::new(0),
};
let mut args = cross_app_args(snapshot_id);
args.timeout_ms = Some(500);
let error = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap_err();
assert_eq!(error.code(), "TIMEOUT");
assert_eq!(adapter.dispatched.load(Ordering::SeqCst), 0);
}
#[test]
fn timeout_none_is_single_shot() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = StaleThenOkAdapter::new(1);
let error = execute(
DragArgs {
from: DragEndpoint {
ref_id: Some("@e1".into()),
xy: None,
},
to: DragEndpoint {
ref_id: Some("@e2".into()),
xy: None,
},
snapshot_id: Some(snapshot_id),
duration_ms: None,
drop_delay_ms: None,
timeout_ms: None,
},
&adapter,
&CommandContext::default().with_headed(true),
)
.unwrap_err();
assert_eq!(error.code(), "STALE_REF");
assert_eq!(adapter.retry.calls(), 1);
}
#[test]
fn expired_xy_budget_never_dispatches_drag() {
let adapter = DragCaptureAdapter::new();
let mut args = xy_args(None);
args.timeout_ms = Some(0);
let err = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap_err();
assert_eq!(err.code(), "TIMEOUT");
let AppError::Adapter(adapter_error) = &err else {
panic!("expected adapter timeout")
};
assert_eq!(
adapter_error
.details
.as_ref()
.and_then(|details| details.get("kind")),
Some(&serde_json::json!("actionability_timeout"))
);
assert!(adapter.captured.lock().unwrap().is_none());
}

217
crates/core/src/commands/drag_tests.rs
View file

@ -1,19 +1,23 @@
use super::*;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::{
action::DragParams,
adapter::{NativeHandle, PlatformAdapter},
AdapterError, DragParams, Rect,
adapter::NativeHandle,
capability,
error::AdapterError,
node::Rect,
commands::stale_retry_test_support::StaleRetryCounter,
hit_test::HitTestResult,
refs::{RefEntry, RefMap},
refs_store::RefStore,
refs_test_support::HomeGuard,
};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
struct DragCaptureAdapter {
captured: Mutex<Option<DragParams>>,
focused_pids: Mutex<Vec<i32>>,
focused_pids: Mutex<Vec<crate::ProcessId>>,
resolve_calls: AtomicU32,
focused_bounds: Option<Rect>,
}
impl DragCaptureAdapter {
@ -21,16 +25,37 @@ impl DragCaptureAdapter {
Self {
captured: Mutex::new(None),
focused_pids: Mutex::new(Vec::new()),
resolve_calls: AtomicU32::new(0),
focused_bounds: None,
}
}
fn with_focused_bounds(mut self, bounds: Rect) -> Self {
self.focused_bounds = Some(bounds);
self
}
}
impl PlatformAdapter for DragCaptureAdapter {
fn resolve_element_strict(&self, _entry: &RefEntry) -> Result<NativeHandle, AdapterError> {
impl ObservationOps for DragCaptureAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
self.resolve_calls.fetch_add(1, Ordering::SeqCst);
Ok(NativeHandle::null())
}
fn get_element_bounds(&self, _handle: &NativeHandle) -> Result<Option<Rect>, AdapterError> {
fn get_element_bounds(
&self,
_handle: &NativeHandle,
_deadline: crate::Deadline,
) -> Result<Option<Rect>, AdapterError> {
if !self.focused_pids.lock().unwrap().is_empty()
&& let Some(bounds) = self.focused_bounds
{
return Ok(Some(bounds));
}
Ok(Some(Rect {
x: 10.0,
y: 20.0,
@ -39,26 +64,64 @@ impl PlatformAdapter for DragCaptureAdapter {
}))
}
fn focus_app(&self, pid: i32) -> Result<(), AdapterError> {
self.focused_pids.lock().unwrap().push(pid);
fn hit_test(
&self,
_handle: &NativeHandle,
_point: crate::Point,
_deadline: crate::Deadline,
) -> Result<HitTestResult, AdapterError> {
Ok(HitTestResult::ReachesTarget)
}
}
impl ActionOps for DragCaptureAdapter {}
impl InputOps for DragCaptureAdapter {
fn drag(
&self,
params: DragParams,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
*self.captured.lock().unwrap() = Some(params);
Ok(())
}
}
fn drag(&self, params: DragParams) -> Result<(), AdapterError> {
*self.captured.lock().unwrap() = Some(params);
impl SystemOps for DragCaptureAdapter {
crate::adapter::guarded_interaction_lease!();
fn resolve_window_strict(
&self,
window: &crate::WindowInfo,
_deadline: crate::Deadline,
) -> Result<crate::WindowInfo, AdapterError> {
Ok(window.clone())
}
fn focus_window(
&self,
window: &crate::WindowInfo,
_lease: &crate::InteractionLease,
) -> Result<(), AdapterError> {
self.focused_pids.lock().unwrap().push(window.pid);
Ok(())
}
}
fn xy_args(drop_delay_ms: Option<u64>) -> DragArgs {
DragArgs {
from_ref: None,
from_xy: Some((1.0, 2.0)),
to_ref: None,
to_xy: Some((3.0, 4.0)),
from: DragEndpoint {
ref_id: None,
xy: Some((1.0, 2.0)),
},
to: DragEndpoint {
ref_id: None,
xy: Some((3.0, 4.0)),
},
snapshot_id: None,
duration_ms: None,
drop_delay_ms,
timeout_ms: None,
}
}
@ -97,24 +160,39 @@ fn drop_delay_omitted_uses_adapter_default_and_no_response_field() {
assert_eq!(captured.drop_delay_ms, None);
}
fn ref_entry(pid: i32) -> RefEntry {
fn ref_entry(pid: u32) -> RefEntry {
RefEntry {
pid,
role: "button".into(),
name: Some("Item".into()),
value: None,
description: None,
states: vec![],
bounds: None,
bounds_hash: None,
available_actions: vec![capability::CLICK.into()],
source_app: None,
source_window_id: None,
source_window_title: None,
source_surface: crate::adapter::SnapshotSurface::Window,
root_ref: None,
path_is_absolute: false,
path: smallvec::SmallVec::new(),
process: crate::RefProcess {
pid: crate::ProcessId::new(pid),
process_instance: Some("test-instance".into()),
},
identity: crate::RefEntryIdentity {
role: "button".into(),
name: Some("Item".into()),
value: None,
description: None,
native_id: None,
},
geometry: crate::RefGeometry {
bounds: None,
bounds_hash: None,
},
capabilities: crate::RefCapabilities {
states: vec![],
available_actions: vec![capability::CLICK.into()],
},
source: crate::RefSource {
source_app: Some(format!("App {pid}")),
source_window_id: Some(format!("w-{pid}")),
source_window_title: Some(format!("Window {pid}")),
source_window_bounds_hash: None,
source_surface: crate::adapter::SnapshotSurface::Window,
},
scope: crate::RefScope {
root_ref: None,
path_is_absolute: false,
path: smallvec::SmallVec::new(),
},
}
}
@ -128,40 +206,21 @@ fn cross_app_snapshot() -> String {
fn cross_app_args(snapshot_id: String) -> DragArgs {
DragArgs {
from_ref: Some("@e1".into()),
from_xy: None,
to_ref: Some("@e2".into()),
to_xy: None,
from: DragEndpoint {
ref_id: Some("@e1".into()),
xy: None,
},
to: DragEndpoint {
ref_id: Some("@e2".into()),
xy: None,
},
snapshot_id: Some(snapshot_id),
duration_ms: None,
drop_delay_ms: None,
timeout_ms: None,
}
}
#[test]
fn drag_resolves_ref_bounds_to_center_point() {
let _guard = HomeGuard::new();
let store = RefStore::new().unwrap();
let mut refmap = RefMap::new();
refmap.allocate(ref_entry(1));
let snapshot_id = store.save_new_snapshot(&refmap).unwrap();
let adapter = DragCaptureAdapter::new();
let args = DragArgs {
from_ref: Some("@e1".into()),
from_xy: None,
to_ref: None,
to_xy: Some((100.0, 200.0)),
snapshot_id: Some(snapshot_id),
duration_ms: None,
drop_delay_ms: Some(300),
};
execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap();
let captured = adapter.captured.lock().unwrap().clone().unwrap();
assert_eq!((captured.from.x, captured.from.y), (30.0, 50.0));
}
#[test]
fn headless_ref_drag_is_policy_denied_before_cursor_move() {
let _guard = HomeGuard::new();
@ -184,7 +243,12 @@ fn headless_ref_drag_is_policy_denied_before_cursor_move() {
fn headed_ref_drag_focuses_only_the_from_app_once() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = DragCaptureAdapter::new();
let adapter = DragCaptureAdapter::new().with_focused_bounds(Rect {
x: 100.0,
y: 200.0,
width: 40.0,
height: 60.0,
});
let value = execute(
cross_app_args(snapshot_id),
@ -194,9 +258,32 @@ fn headed_ref_drag_focuses_only_the_from_app_once() {
.unwrap();
assert_eq!(*adapter.focused_pids.lock().unwrap(), vec![1]);
assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 4);
let captured = adapter.captured.lock().unwrap().clone().unwrap();
assert_eq!((captured.from.x, captured.from.y), (120.0, 230.0));
assert_eq!(value["focused"], true);
}
#[test]
fn positive_timeout_uses_stable_post_focus_drag_bounds() {
let _guard = HomeGuard::new();
let snapshot_id = cross_app_snapshot();
let adapter = DragCaptureAdapter::new().with_focused_bounds(Rect {
x: 100.0,
y: 200.0,
width: 40.0,
height: 60.0,
});
let mut args = cross_app_args(snapshot_id);
args.timeout_ms = Some(500);
let value = execute(args, &adapter, &CommandContext::default().with_headed(true)).unwrap();
assert_eq!(*adapter.focused_pids.lock().unwrap(), vec![1]);
assert_eq!(value["from"], json!({ "x": 120.0, "y": 230.0 }));
assert_eq!(value["to"], json!({ "x": 120.0, "y": 230.0 }));
}
#[test]
fn headed_xy_drag_never_steals_focus() {
let adapter = DragCaptureAdapter::new();
@ -211,3 +298,9 @@ fn headed_xy_drag_never_steals_focus() {
assert!(adapter.focused_pids.lock().unwrap().is_empty());
assert!(value.get("focused").is_none());
}
#[path = "drag_retry_tests.rs"]
mod retry_tests;
#[path = "drag_occlusion_retry_tests.rs"]
mod occlusion_retry_tests;

61
crates/core/src/commands/execute_by_ref.rs
View file

@ -1,50 +1,81 @@
use crate::{
AppError,
action::Action,
action_request::ActionRequest,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
commands::helpers::{RefArgs, execute_ref_action_with_context, normalize_action_timeout_ms},
context::CommandContext,
error::AppError,
interaction_policy::InteractionPolicy,
};
use serde_json::Value;
pub const DEFAULT_ACTION_TIMEOUT_MS: u64 = 5000;
/// Config struct bundling the caller-supplied inputs to `execute` /
/// `execute_with_timeout`, keeping both functions within the 5-parameter
/// limit (mirrors the `RefArgs`/`ActionRequest` config-struct pattern used
/// elsewhere in this module).
pub struct ExecuteByRefArgs<'a> {
pub ref_id: &'a str,
pub snapshot_id: Option<&'a str>,
pub action: Action,
pub caller_policy: InteractionPolicy,
}
/// Executes an action addressed by a snapshot ref through the canonical
/// ref-action pipeline: `RefStore` load → `RefMap` lookup → strict element
/// resolution → live actionability preflight → dispatch.
///
/// `snapshot_id` follows CLI `--snapshot` semantics: `None` pins to the
/// latest snapshot for the session; `Some(id)` pins to that specific snapshot.
/// A qualified `ref_id` embeds its snapshot and accepts `snapshot_id: None`.
/// A legacy bare `@eN` ref requires `Some(id)`. When both are supplied, the
/// explicit ID must match the qualified ref.
///
/// The effective `InteractionPolicy` is the join of `caller_policy` and the
/// action's CLI base policy, ensuring the result is always at least as
/// permissive as what the CLI would use for the same action, while allowing
/// FFI callers to opt in to higher-permission policies such as `headed`.
///
/// Note on PressKey: its base policy is `focus_fallback` (derived from
/// `Action::base_interaction_policy`, shared with `TypeText`) because a
/// ref-targeted key press may need the target focused for keystrokes to land.
/// action's base policy. Semantic actions, including `TypeText`, are strict
/// headless; explicit `PressKey` permits focus without cursor movement. FFI
/// callers may opt in to `focus_fallback` or `headed`.
pub fn execute(
ref_id: &str,
snapshot_id: Option<&str>,
action: Action,
caller_policy: InteractionPolicy,
args: ExecuteByRefArgs<'_>,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
execute_with_timeout(args, DEFAULT_ACTION_TIMEOUT_MS, adapter, context)
}
pub fn execute_with_timeout(
args: ExecuteByRefArgs<'_>,
timeout_ms: u64,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let ExecuteByRefArgs {
ref_id,
snapshot_id,
action,
caller_policy,
} = args;
let timeout_ms = normalize_action_timeout_ms(timeout_ms);
let base = action.base_interaction_policy();
let effective = base.join(caller_policy);
let request = ActionRequest {
action,
policy: effective,
timeout_ms,
verified_point: None,
expected_process: None,
};
execute_ref_action_with_context(
RefArgs {
ref_id: ref_id.to_owned(),
snapshot_id: snapshot_id.map(ToOwned::to_owned),
timeout_ms,
},
adapter,
request,
context,
)
}
#[cfg(test)]
#[path = "execute_by_ref_tests.rs"]
mod tests;

273
crates/core/src/commands/execute_by_ref_tests.rs Normal file
View file

@ -0,0 +1,273 @@
use super::*;
use crate::adapter::{ActionOps, InputOps, ObservationOps, SystemOps};
use crate::{
Action, AdapterError, ErrorCode, KeyCombo,
action_request::ActionRequest,
action_result::ActionResult,
adapter::NativeHandle,
refs::{RefEntry, RefMap},
refs_store::RefStore,
refs_test_support::HomeGuard,
};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
fn snapshot_with_ref(role: &str, available_actions: &[&str]) -> String {
let mut refmap = RefMap::new();
let bounds = crate::Rect {
x: 1.0,
y: 1.0,
width: 20.0,
height: 20.0,
};
refmap.allocate(RefEntry {
process: crate::RefProcess {
pid: crate::ProcessId::new(1),
process_instance: Some("test-instance".into()),
},
identity: crate::RefEntryIdentity {
role: role.into(),
name: Some("Target".into()),
value: None,
description: None,
native_id: None,
},
geometry: crate::RefGeometry {
bounds: Some(bounds),
bounds_hash: bounds.bounds_hash(),
},
capabilities: crate::RefCapabilities {
states: vec![],
available_actions: available_actions.iter().map(|a| (*a).to_string()).collect(),
},
source: crate::RefSource {
source_app: Some("Fixture".into()),
source_window_id: Some("w-1".into()),
source_window_title: Some("Fixture".into()),
source_window_bounds_hash: None,
source_surface: crate::adapter::SnapshotSurface::Window,
},
scope: crate::RefScope {
root_ref: None,
path_is_absolute: false,
path: smallvec::SmallVec::new(),
},
});
RefStore::new().unwrap().save_new_snapshot(&refmap).unwrap()
}
struct StaleThenOkAdapter {
resolve_calls: AtomicU32,
fail_until: u32,
}
impl StaleThenOkAdapter {
fn new(fail_until: u32) -> Self {
Self {
resolve_calls: AtomicU32::new(0),
fail_until,
}
}
}
impl ObservationOps for StaleThenOkAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
let n = self.resolve_calls.fetch_add(1, Ordering::SeqCst) + 1;
if n <= self.fail_until {
return Err(AdapterError::new(ErrorCode::StaleRef, "not yet resolvable")
.with_details(serde_json::json!({ "retryable": true })));
}
Ok(NativeHandle::null())
}
crate::adapter::complete_live_observation!(
"textfield",
"Target",
[
crate::capability::CLICK,
crate::capability::PRESS_KEY,
crate::capability::SET_VALUE
]
);
}
impl ActionOps for StaleThenOkAdapter {
fn execute_action(
&self,
_handle: &NativeHandle,
_request: ActionRequest,
_lease: &crate::InteractionLease,
) -> Result<ActionResult, AdapterError> {
Ok(ActionResult::delivered_unverified("click"))
}
}
impl InputOps for StaleThenOkAdapter {}
impl SystemOps for StaleThenOkAdapter {
crate::adapter::guarded_interaction_lease!();
}
struct PolicyCaptureAdapter {
captured: Mutex<Option<ActionRequest>>,
}
impl PolicyCaptureAdapter {
fn new() -> Self {
Self {
captured: Mutex::new(None),
}
}
}
impl ObservationOps for PolicyCaptureAdapter {
fn resolve_element_strict(
&self,
_entry: &RefEntry,
_deadline: crate::Deadline,
) -> Result<NativeHandle, AdapterError> {
Ok(NativeHandle::null())
}
crate::adapter::complete_live_observation!(
"textfield",
"Target",
[
crate::capability::CLICK,
crate::capability::PRESS_KEY,
crate::capability::SET_VALUE
]
);
}
impl ActionOps for PolicyCaptureAdapter {
fn execute_action(
&self,
_handle: &NativeHandle,
request: ActionRequest,
_lease: &crate::InteractionLease,
) -> Result<ActionResult, AdapterError> {
let name = request.action.name().to_string();
*self.captured.lock().unwrap() = Some(request);
Ok(ActionResult::delivered_unverified(name))
}
}
impl InputOps for PolicyCaptureAdapter {}
impl SystemOps for PolicyCaptureAdapter {
crate::adapter::guarded_interaction_lease!();
crate::adapter::exact_window_focus!();
}
#[test]
fn default_action_timeout_ms_is_five_seconds() {
assert_eq!(DEFAULT_ACTION_TIMEOUT_MS, 5000);
}
/// `execute` must forward `DEFAULT_ACTION_TIMEOUT_MS` (not `None` or `0`) into
/// the ref-action retry budget, exactly as a direct
/// `execute_with_timeout(args, DEFAULT_ACTION_TIMEOUT_MS, ..)` call would.
/// A transient `STALE_REF` that clears within the budget must therefore be
/// retried until it resolves rather than surfacing on the first attempt.
#[test]
fn execute_forwards_default_timeout_and_retries_transient_stale_ref() {
let _guard = HomeGuard::new();
let snapshot_id = snapshot_with_ref("textfield", &["Click"]);
let adapter = StaleThenOkAdapter::new(2);
let value = execute(
ExecuteByRefArgs {
ref_id: "@e1",
snapshot_id: Some(&snapshot_id),
action: Action::Click,
caller_policy: InteractionPolicy::headless(),
},
&adapter,
&CommandContext::default(),
)
.unwrap();
assert_eq!(value["action"], "click");
assert!(adapter.resolve_calls.load(Ordering::SeqCst) >= 3);
}
/// `execute_with_timeout` must run `normalize_action_timeout_ms` on the raw
/// timeout it is given, so a `0` (the CLI's "no retry budget" sentinel)
/// collapses to `None` and the ref-action pipeline makes exactly one resolve
/// attempt instead of silently retrying anyway.
#[test]
fn execute_with_timeout_zero_normalizes_to_single_attempt() {
let _guard = HomeGuard::new();
let snapshot_id = snapshot_with_ref("textfield", &["Click"]);
let adapter = StaleThenOkAdapter::new(1);
let err = execute_with_timeout(
ExecuteByRefArgs {
ref_id: "@e1",
snapshot_id: Some(&snapshot_id),
action: Action::Click,
caller_policy: InteractionPolicy::headless(),
},
0,
&adapter,
&CommandContext::default(),
)
.unwrap_err();
assert_eq!(err.code(), "STALE_REF");
assert_eq!(adapter.resolve_calls.load(Ordering::SeqCst), 1);
}
#[test]
fn explicit_press_key_keeps_its_focus_fallback_policy() {
let _guard = HomeGuard::new();
let snapshot_id = snapshot_with_ref("textfield", &["PressKey"]);
let adapter = PolicyCaptureAdapter::new();
execute(
ExecuteByRefArgs {
ref_id: "@e1",
snapshot_id: Some(&snapshot_id),
action: Action::PressKey(KeyCombo {
key: "A".into(),
modifiers: vec![],
}),
caller_policy: InteractionPolicy::headless(),
},
&adapter,
&CommandContext::default(),
)
.unwrap();
let captured = adapter.captured.lock().unwrap();
let policy = captured.as_ref().unwrap().policy;
assert!(policy.allow_focus_steal);
assert!(!policy.allow_cursor_move);
}
#[test]
fn effective_policy_honors_caller_policy_above_action_base() {
let _guard = HomeGuard::new();
let snapshot_id = snapshot_with_ref("textfield", &["SetValue"]);
let adapter = PolicyCaptureAdapter::new();
execute(
ExecuteByRefArgs {
ref_id: "@e1",
snapshot_id: Some(&snapshot_id),
action: Action::SetValue("value".into()),
caller_policy: InteractionPolicy::headed(),
},
&adapter,
&CommandContext::default(),
)
.unwrap();
let captured = adapter.captured.lock().unwrap();
let policy = captured.as_ref().unwrap().policy;
assert!(policy.allow_focus_steal);
assert!(policy.allow_cursor_move);
}

2
crates/core/src/commands/expand.rs
View file

@ -1,9 +1,9 @@
use crate::{
AppError,
action::Action,
adapter::PlatformAdapter,
commands::helpers::{RefArgs, execute_ref_action_with_context},
context::CommandContext,
error::AppError,
};
use serde_json::Value;

215
crates/core/src/commands/find.rs
View file

@ -1,20 +1,34 @@
#[cfg(test)]
use crate::AccessibilityNode;
#[cfg(test)]
use crate::commands::query;
use crate::{
adapter::PlatformAdapter, commands::query, context::CommandContext, error::AppError,
node::AccessibilityNode, roles, search_text, snapshot,
AppError, IdentityPredicate, LocatorQuery, StatePredicate, adapter::PlatformAdapter,
context::CommandContext, search_text,
};
use serde_json::{Value, json};
use serde_json::Value;
#[cfg(test)]
use serde_json::json;
#[cfg(test)]
use std::collections::BTreeSet;
const DEFAULT_LIMIT: usize = 50;
pub use query::FindQuery;
pub struct FindArgs {
pub app: Option<String>,
/// Match-criteria fields: how a candidate element is identified. Grouped out
/// of [`FindArgs`] to keep it under the repo's field-count limit.
pub struct FindFilterArgs {
pub role: Option<String>,
pub name: Option<String>,
pub description: Option<String>,
pub native_id: Option<String>,
pub value: Option<String>,
pub text: Option<String>,
pub exact: bool,
}
/// Result-shaping fields: which of the matches to return. Mutually exclusive
/// at the CLI/batch layer (enforced by [`validate_find_mode`]).
pub struct FindSelectionArgs {
pub count: bool,
pub first: bool,
pub last: bool,
@ -22,75 +36,77 @@ pub struct FindArgs {
pub limit: Option<usize>,
}
pub struct FindArgs {
pub app: Option<String>,
pub window_id: Option<String>,
pub filter: FindFilterArgs,
pub states: Vec<StatePredicate>,
pub selection: FindSelectionArgs,
}
pub fn execute(
args: FindArgs,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
validate_find_mode(&args)?;
let query = FindQuery::from_args(&args);
let opts = crate::adapter::TreeOptions::default();
let result = if args.count {
snapshot::build(adapter, &opts, args.app.as_deref(), None)?
} else {
snapshot::run_with_context(adapter, &opts, args.app.as_deref(), None, context)?
};
let query = locator_query_from_args(&args)?;
query.validate_states().map_err(AppError::Adapter)?;
if args.count {
return Ok(json!({ "count": count_matches(&result.tree, &query) }));
}
let mut matches = Vec::new();
let max_matches = max_matches_for_args(&args);
search_tree(
&result.tree,
&query,
&mut Vec::new(),
&mut matches,
max_matches,
);
if args.first {
return Ok(single_match_response(
matches.into_iter().next(),
&query,
&result.tree,
));
}
if args.last {
return Ok(single_match_response(
matches.into_iter().last(),
&query,
&result.tree,
));
}
if let Some(n) = args.nth {
return Ok(single_match_response(
matches.into_iter().nth(n),
&query,
&result.tree,
));
}
let mut response = json!({ "matches": matches });
attach_roles_present_hint(&mut response, matches.is_empty(), &query, &result.tree);
Ok(response)
live::execute(&args, &query, adapter, context)
}
/// When a role-filtered search returns nothing, the caller cannot tell
/// "no elements of this role are on screen" from "this role name does not
/// exist." Listing the roles actually present in the searched tree answers
/// that from live data — no hardcoded vocabulary, so a role any adapter
/// newly emits shows up here automatically.
pub fn parse_state_flag(raw: &str) -> Result<StatePredicate, AppError> {
let (token, expected) = match raw.split_once('=') {
Some((token, value)) => {
let expected = match value {
"true" | "1" => Some(true),
"false" | "0" => Some(false),
_ => {
return Err(AppError::invalid_input_with_suggestion(
format!("Invalid state flag value '{value}' in '{raw}'"),
"Use TOKEN, TOKEN=true, or TOKEN=false.",
));
}
};
(token, expected)
}
None => (raw, None),
};
Ok(StatePredicate {
token: token.to_string(),
expected,
})
}
fn locator_query_from_args(args: &FindArgs) -> Result<LocatorQuery, AppError> {
Ok(LocatorQuery {
identity: IdentityPredicate {
role: args.filter.role.clone(),
name: args.filter.name.as_deref().map(search_text::normalize),
description: args
.filter
.description
.as_deref()
.map(search_text::normalize),
native_id: args.filter.native_id.clone(),
value: args.filter.value.as_deref().map(search_text::normalize),
},
has_text: args.filter.text.as_deref().map(search_text::normalize),
exact: args.filter.exact,
states: args.states.clone(),
..LocatorQuery::default()
})
}
#[cfg(test)]
fn attach_roles_present_hint(
response: &mut Value,
is_empty: bool,
query: &FindQuery,
query: &LocatorQuery,
tree: &AccessibilityNode,
) {
if !is_empty || query.role.is_none() {
if !is_empty || query.identity.role.is_none() {
return;
}
let mut present = BTreeSet::new();
@ -103,9 +119,10 @@ fn attach_roles_present_hint(
}
}
#[cfg(test)]
fn single_match_response(
found: Option<Value>,
query: &FindQuery,
query: &LocatorQuery,
tree: &AccessibilityNode,
) -> Value {
let is_empty = found.is_none();
@ -114,6 +131,7 @@ fn single_match_response(
response
}
#[cfg(test)]
fn collect_roles(node: &AccessibilityNode, roles: &mut BTreeSet<String>) {
roles.insert(node.role.clone());
for child in &node.children {
@ -121,28 +139,17 @@ fn collect_roles(node: &AccessibilityNode, roles: &mut BTreeSet<String>) {
}
}
fn max_matches_for_args(args: &FindArgs) -> Option<usize> {
if args.count || args.last {
return None;
}
if args.first {
return Some(1);
}
if let Some(n) = args.nth {
return Some(n.saturating_add(1));
}
match args.limit.unwrap_or(DEFAULT_LIMIT) {
0 => None,
limit => Some(limit),
}
}
fn validate_find_mode(args: &FindArgs) -> Result<(), AppError> {
let selector_count = [args.count, args.first, args.last, args.nth.is_some()]
.into_iter()
.filter(|selected| *selected)
.count();
if selector_count > 1 || (selector_count == 1 && args.limit.is_some()) {
let selector_count = [
args.selection.count,
args.selection.first,
args.selection.last,
args.selection.nth.is_some(),
]
.into_iter()
.filter(|selected| *selected)
.count();
if selector_count > 1 || (selector_count == 1 && args.selection.limit.is_some()) {
return Err(AppError::invalid_input_with_suggestion(
"find accepts only one result-shaping mode",
"Use one of --count, --first, --last, --nth, or --limit.",
@ -151,20 +158,10 @@ fn validate_find_mode(args: &FindArgs) -> Result<(), AppError> {
Ok(())
}
impl FindQuery {
fn from_args(args: &FindArgs) -> Self {
Self {
role: args.role.as_deref().map(roles::normalize_role_query),
name: args.name.as_deref().map(search_text::normalize),
value: args.value.as_deref().map(search_text::normalize),
text: args.text.as_deref().map(search_text::normalize),
}
}
}
fn search_tree(
#[cfg(test)]
fn collect_snapshot_matches(
node: &AccessibilityNode,
query: &FindQuery,
query: &LocatorQuery,
path: &mut Vec<String>,
matches: &mut Vec<Value>,
max_matches: Option<usize>,
@ -175,18 +172,19 @@ fn search_tree(
if query::node_matches(node, query) {
let interactive = node.ref_id.is_some();
let display_name = node
.identity
.name
.as_deref()
.or(node.value.as_deref())
.or(node.description.as_deref())
.or(node.identity.value.as_deref())
.or(node.identity.description.as_deref())
.map(String::from)
.unwrap_or_else(|| format!("(unnamed {})", node.role));
matches.push(json!({
"ref_id": node.ref_id,
"role": node.role,
"name": display_name,
"value": node.value,
"states": node.states,
"value": node.identity.value,
"states": node.presentation.states,
"interactive": interactive,
"path": path.clone()
}));
@ -196,15 +194,16 @@ fn search_tree(
}
let label = node
.identity
.name
.as_deref()
.or(node.value.as_deref())
.or(node.identity.value.as_deref())
.map(|label| format!("{}:{label}", node.role))
.unwrap_or_else(|| node.role.clone());
path.push(label);
for child in &node.children {
if search_tree(child, query, path, matches, max_matches) {
if collect_snapshot_matches(child, query, path, matches, max_matches) {
path.pop();
return true;
}
@ -214,7 +213,8 @@ fn search_tree(
false
}
fn count_matches(node: &AccessibilityNode, query: &FindQuery) -> usize {
#[cfg(test)]
fn count_matches(node: &AccessibilityNode, query: &LocatorQuery) -> usize {
usize::from(query::node_matches(node, query))
+ node
.children
@ -223,6 +223,13 @@ fn count_matches(node: &AccessibilityNode, query: &FindQuery) -> usize {
.sum::<usize>()
}
#[path = "find_live.rs"]
mod live;
#[cfg(test)]
#[path = "find_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "find_live_tests.rs"]
mod live_tests;

243
crates/core/src/commands/find_live.rs Normal file
View file

@ -0,0 +1,243 @@
use super::{DEFAULT_LIMIT, FindArgs};
use crate::{
AdapterError, AppError, LocatorQuery,
adapter::PlatformAdapter,
context::CommandContext,
live_locator::{
LocatorMaterialization, LocatorResolution, LocatorResolveRequest, LocatorSelection,
ObservationRoot, resolve_query,
},
refs::RefMap,
refs_store::RefStore,
snapshot, trace_artifacts,
};
use serde_json::{Value, json};
use std::time::Duration;
const LOCATOR_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_RAW_DEPTH: u8 = 50;
pub(super) fn execute(
args: &FindArgs,
query: &LocatorQuery,
adapter: &dyn PlatformAdapter,
context: &CommandContext,
) -> Result<Value, AppError> {
let deadline = crate::Deadline::from_duration(LOCATOR_TIMEOUT)?;
let window = snapshot::resolve_window(
adapter,
args.app.as_deref(),
args.window_id.as_deref(),
deadline,
)?;
let request = resolve_request(args, deadline);
let mut resolution = resolve_query(adapter, query, ObservationRoot::Window(&window), &request)?;
require_complete(&resolution)?;
let ref_count = resolution.refmap.as_ref().map(RefMap::len);
let snapshot_id = match resolution.refmap.take() {
Some(refmap) => Some(persist_refmap(context, &refmap)?),
None => None,
};
if let Some(snapshot_id) = snapshot_id.as_deref() {
for found in &mut resolution.matches {
if let Some(local_ref) = found.data.ref_id.as_deref() {
found.data.ref_id = Some(crate::ref_token::qualify_ref_id(snapshot_id, local_ref));
}
}
}
emit_resolution(context, &resolution, snapshot_id.as_deref(), ref_count)?;
format_response(args, query, resolution, snapshot_id.as_deref())
}
fn resolve_request(args: &FindArgs, deadline: crate::Deadline) -> LocatorResolveRequest {
let selection = if args.selection.count {
LocatorSelection::Count
} else if args.selection.first {
LocatorSelection::First
} else if args.selection.last {
LocatorSelection::Last
} else if let Some(index) = args.selection.nth {
LocatorSelection::Nth(u32::try_from(index).unwrap_or(u32::MAX))
} else {
LocatorSelection::All {
limit: args
.selection
.limit
.map_or(Some(DEFAULT_LIMIT as u32), |limit| {
(limit != 0).then(|| u32::try_from(limit).unwrap_or(u32::MAX))
}),
}
};
LocatorResolveRequest {
selection,
deadline,
max_raw_depth: MAX_RAW_DEPTH,
materialization: if args.selection.count {
LocatorMaterialization::None
} else {
LocatorMaterialization::SelectedMatches
},
}
}
fn require_complete(resolution: &LocatorResolution) -> Result<(), AppError> {
if resolution.meta.selection_complete {
return Ok(());
}
Err(
AdapterError::timeout("Locator traversal did not produce an authoritative result")
.with_details(json!({
"kind": "locator_incomplete",
"observed_matches": resolution.meta.total_matches,
"query_stats": resolution.stats,
"roles_present": resolution.meta.roles_present,
}))
.into(),
)
}
fn persist_refmap(context: &CommandContext, refmap: &RefMap) -> Result<String, AppError> {
let store = RefStore::for_session(context.session_id())?;
let snapshot_id = store.save_new_snapshot(refmap)?;
trace_artifacts::copy_refmap_if_full(context, &store, &snapshot_id, refmap)?;
Ok(snapshot_id)
}
fn emit_resolution(
context: &CommandContext,
resolution: &LocatorResolution,
snapshot_id: Option<&str>,
ref_count: Option<usize>,
) -> Result<(), AppError> {
context.trace_lazy("locator.resolve", || {
json!({
"complete": resolution.meta.complete,
"match_count": resolution.meta.total_matches,
"query_stats": resolution.stats,
"ref_count": ref_count,
"snapshot_id": snapshot_id,
"truncated": resolution.meta.truncated,
})
})
}
fn format_response(
args: &FindArgs,
query: &LocatorQuery,
resolution: LocatorResolution,
snapshot_id: Option<&str>,
) -> Result<Value, AppError> {
if args.selection.count {
return Ok(json!({ "count": resolution.meta.total_matches }));
}
let total_matches = resolution.meta.total_matches;
let truncated = resolution.meta.truncated;
let roles_present = resolution.meta.roles_present;
let matches = resolution
.matches
.into_iter()
.map(|found| serde_json::to_value(found.data))
.collect::<Result<Vec<_>, _>>()?;
if args.selection.first || args.selection.last || args.selection.nth.is_some() {
let mut response = single_match_response(matches.into_iter().next(), query, roles_present);
attach_snapshot_id(&mut response, snapshot_id);
return Ok(response);
}
let mut response = json!({
"matches": matches,
"total_matches": total_matches,
"truncated": truncated,
});
let is_empty = response["matches"].as_array().is_some_and(Vec::is_empty);
attach_roles_present(&mut response, is_empty, query, roles_present);
attach_snapshot_id(&mut response, snapshot_id);
Ok(response)
}
fn attach_snapshot_id(response: &mut Value, snapshot_id: Option<&str>) {
if let (Some(object), Some(snapshot_id)) = (response.as_object_mut(), snapshot_id) {
object.insert("snapshot_id".into(), json!(snapshot_id));
}
}
fn single_match_response(
found: Option<Value>,
query: &LocatorQuery,
roles_present: Vec<String>,
) -> Value {
let is_empty = found.is_none();
let mut response = json!({ "match": found });
attach_roles_present(&mut response, is_empty, query, roles_present);
response
}
fn attach_roles_present(
response: &mut Value,
is_empty: bool,
query: &LocatorQuery,
roles_present: Vec<String>,
) {
if !is_empty || query.identity.role.is_none() {
return;
}
if let Some(object) = response.as_object_mut() {
object.insert("roles_present".into(), json!(roles_present));
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::live_locator::{LocatorResolutionMeta, LocatorStats};
fn args(limit: Option<usize>) -> FindArgs {
FindArgs {
app: None,
window_id: None,
filter: crate::commands::find::FindFilterArgs {
role: None,
name: None,
description: None,
native_id: None,
value: None,
text: None,
exact: false,
},
states: Vec::new(),
selection: crate::commands::find::FindSelectionArgs {
count: false,
first: false,
last: false,
nth: None,
limit,
},
}
}
#[test]
fn zero_limit_requests_all_matches() {
let request = resolve_request(&args(Some(0)), crate::Deadline::standard().unwrap());
assert_eq!(request.selection, LocatorSelection::All { limit: None });
}
#[test]
fn all_response_exposes_total_and_truncation() {
let resolution = LocatorResolution {
matches: Vec::new(),
refmap: None,
stats: LocatorStats::default(),
meta: LocatorResolutionMeta {
total_matches: 72,
complete: true,
selection_complete: true,
truncated: true,
roles_present: vec!["button".into()],
},
};
let response =
format_response(&args(Some(50)), &LocatorQuery::default(), resolution, None).unwrap();
assert_eq!(response["total_matches"], 72);
assert_eq!(response["truncated"], true);
}
}

Some files were not shown because too many files have changed in this diff Show more